Add automated client smoke tests. (#2678)

.github/workflows/build.yml

@@ -34,3 +34,23 @@ jobs:
         with:
           name: Maven Local
           path: /root/.m2/repository
+
+  client_test:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-java@v3
+        with:
+          distribution: 'microsoft'
+          java-version: '17'
+      - name: Run Auto test Client
+        uses: modmuss50/xvfb-action@v1
+        with:
+          run: ./gradlew runProductionAutoTestClient --stacktrace --warning-mode=fail
+      - uses: actions/upload-artifact@v2
+        if: always()
+        with:
+          name: Test Screenshots
+          path: run/screenshots

build.gradle

@@ -354,15 +354,66 @@ loom {
 		}
 		autoTestServer {
 			inherit testmodServer
-
 			name "Auto Test Server"
-
+			vmArg "-Dfabric.autoTest"
+		}
+		autoTestClient {
+			inherit testmodClient
+			name "Auto Test Client"
 			vmArg "-Dfabric.autoTest"
 		}
 	}
 }
 test.dependsOn runGametest
 
+configurations {
+	productionRuntime {
+		extendsFrom configurations.minecraftLibraries
+		extendsFrom configurations.loaderLibraries
+		extendsFrom configurations.minecraftRuntimeOnlyLibraries
+	}
+}
+
+dependencies {
+	productionRuntime "net.fabricmc:fabric-loader:${project.loader_version}"
+	productionRuntime "net.fabricmc:intermediary:${project.minecraft_version}"
+}
+
+import net.fabricmc.loom.util.OperatingSystem
+
+// This is very far beyond loom's API if you copy this, you're on your own.
+task runProductionAutoTestClient(type: JavaExec, dependsOn: [remapJar, remapTestmodJar]) {
+	classpath.from configurations.productionRuntime
+	mainClass = "net.fabricmc.loader.impl.launch.knot.KnotClient"
+	workingDir = file("run")
+
+	afterEvaluate {
+		dependsOn downloadAssets
+	}
+
+	doFirst {
+		classpath.from loom.minecraftProvider.minecraftClientJar
+		workingDir.mkdirs()
+
+		args(
+			"--assetIndex", loom.minecraftProvider.versionInfo.assetIndex().fabricId(loom.minecraftProvider.minecraftVersion()),
+			"--assetsDir", new File(loom.files.userCache, "assets").absolutePath,
+			"--gameDir", workingDir.absolutePath
+		)
+
+		if (OperatingSystem.CURRENT_OS == OperatingSystem.MAC_OS) {
+			jvmArgs(
+				"-XstartOnFirstThread"
+			)
+		}
+
+		jvmArgs(
+			"-Dfabric.addMods=${remapJar.archiveFile.get().asFile.absolutePath}${File.pathSeparator}${remapTestmodJar.archiveFile.get().asFile.absolutePath}",
+			"-Dfabric.autoTest"
+		)
+	}
+}
+
 subprojects {
 	if (it.name == "deprecated") return
 

fabric-api-base/build.gradle

@@ -3,5 +3,6 @@ version = getSubprojectVersion(project)
 
 testDependencies(project, [
 	':fabric-command-api-v2',
-	':fabric-lifecycle-events-v1'
+	':fabric-lifecycle-events-v1',
+	':fabric-screen-api-v1'
 ])

fabric-api-base/src/testmod/java/net/fabricmc/fabric/test/base/FabricApiAutoTestClient.java

@@ -0,0 +1,134 @@
+/*
+ * Copyright (c) 2016, 2017, 2018, 2019 FabricMC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.fabricmc.fabric.test.base;
+
+import static net.fabricmc.fabric.test.base.FabricClientTestHelper.clickScreenButton;
+import static net.fabricmc.fabric.test.base.FabricClientTestHelper.closeScreen;
+import static net.fabricmc.fabric.test.base.FabricClientTestHelper.enableDebugHud;
+import static net.fabricmc.fabric.test.base.FabricClientTestHelper.openGameMenu;
+import static net.fabricmc.fabric.test.base.FabricClientTestHelper.openInventory;
+import static net.fabricmc.fabric.test.base.FabricClientTestHelper.setPerspective;
+import static net.fabricmc.fabric.test.base.FabricClientTestHelper.takeScreenshot;
+import static net.fabricmc.fabric.test.base.FabricClientTestHelper.waitForLoadingComplete;
+import static net.fabricmc.fabric.test.base.FabricClientTestHelper.waitForScreen;
+import static net.fabricmc.fabric.test.base.FabricClientTestHelper.waitForWorldTicks;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+import org.spongepowered.asm.mixin.MixinEnvironment;
+
+import net.minecraft.client.gui.screen.ConfirmScreen;
+import net.minecraft.client.gui.screen.TitleScreen;
+import net.minecraft.client.gui.screen.world.CreateWorldScreen;
+import net.minecraft.client.gui.screen.world.SelectWorldScreen;
+import net.minecraft.client.option.Perspective;
+
+import net.fabricmc.api.ClientModInitializer;
+import net.fabricmc.loader.api.FabricLoader;
+
+public class FabricApiAutoTestClient implements ClientModInitializer {
+	@Override
+	public void onInitializeClient() {
+		if (System.getProperty("fabric.autoTest") == null) {
+			return;
+		}
+
+		var thread = new Thread(() -> {
+			try {
+				runTest();
+			} catch (Throwable t) {
+				t.printStackTrace();
+				System.exit(1);
+			}
+		});
+		thread.setName("Fabric Auto Test");
+		thread.start();
+	}
+
+	private void runTest() {
+		waitForLoadingComplete();
+
+		{
+			waitForScreen(TitleScreen.class);
+			takeScreenshot("title_screen");
+			clickScreenButton("menu.singleplayer");
+		}
+
+		if (!isDirEmpty(FabricLoader.getInstance().getGameDir().resolve("saves"))) {
+			waitForScreen(SelectWorldScreen.class);
+			takeScreenshot("select_world_screen");
+			clickScreenButton("selectWorld.create");
+		}
+
+		{
+			waitForScreen(CreateWorldScreen.class);
+			clickScreenButton("selectWorld.gameMode");
+			clickScreenButton("selectWorld.gameMode");
+			takeScreenshot("create_world_screen");
+			clickScreenButton("selectWorld.create");
+		}
+
+		{
+			// API test mods use experimental features
+			waitForScreen(ConfirmScreen.class);
+			clickScreenButton("gui.yes");
+		}
+
+		{
+			enableDebugHud();
+			waitForWorldTicks(200);
+			takeScreenshot("in_game_overworld");
+		}
+
+		MixinEnvironment.getCurrentEnvironment().audit();
+
+		{
+			// See if the player render events are working.
+			setPerspective(Perspective.THIRD_PERSON_BACK);
+			takeScreenshot("in_game_overworld_third_person");
+		}
+
+		{
+			openInventory();
+			takeScreenshot("in_game_inventory");
+			closeScreen();
+		}
+
+		{
+			openGameMenu();
+			takeScreenshot("game_menu");
+			clickScreenButton("menu.returnToMenu");
+		}
+
+		{
+			waitForScreen(TitleScreen.class);
+			clickScreenButton("menu.quit");
+		}
+	}
+
+	private boolean isDirEmpty(Path path) {
+		try (DirectoryStream<Path> directory = Files.newDirectoryStream(path)) {
+			return !directory.iterator().hasNext();
+		} catch (IOException e) {
+			throw new UncheckedIOException(e);
+		}
+	}
+}

fabric-api-base/src/testmod/java/net/fabricmc/fabric/test/base/FabricApiAutoTestServer.java

@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2016, 2017, 2018, 2019 FabricMC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.fabricmc.fabric.test.base;
+
+import org.spongepowered.asm.mixin.MixinEnvironment;
+
+import net.fabricmc.api.DedicatedServerModInitializer;
+import net.fabricmc.fabric.api.event.lifecycle.v1.ServerTickEvents;
+
+public class FabricApiAutoTestServer implements DedicatedServerModInitializer {
+	private int ticks = 0;
+
+	@Override
+	public void onInitializeServer() {
+		if (System.getProperty("fabric.autoTest") != null) {
+			ServerTickEvents.END_SERVER_TICK.register(server -> {
+				ticks++;
+
+				if (ticks == 50) {
+					MixinEnvironment.getCurrentEnvironment().audit();
+					server.stop(false);
+				}
+			});
+		}
+	}
+}

fabric-api-base/src/testmod/java/net/fabricmc/fabric/test/base/FabricApiBaseTestInit.java

@@ -24,24 +24,10 @@ import net.minecraft.text.Text;
 
 import net.fabricmc.api.ModInitializer;
 import net.fabricmc.fabric.api.command.v2.CommandRegistrationCallback;
-import net.fabricmc.fabric.api.event.lifecycle.v1.ServerTickEvents;
 
 public class FabricApiBaseTestInit implements ModInitializer {
-	private int ticks = 0;
-
 	@Override
 	public void onInitialize() {
-		if (System.getProperty("fabric.autoTest") != null) {
-			ServerTickEvents.END_SERVER_TICK.register(server -> {
-				ticks++;
-
-				if (ticks == 50) {
-					MixinEnvironment.getCurrentEnvironment().audit();
-					server.stop(false);
-				}
-			});
-		}
-
 		// Command to call audit the mixin environment
 		CommandRegistrationCallback.EVENT.register((dispatcher, registryAccess, environment) -> {
 			dispatcher.register(literal("audit_mixins").executes(context -> {

fabric-api-base/src/testmod/java/net/fabricmc/fabric/test/base/FabricClientTestHelper.java

@@ -0,0 +1,199 @@
+/*
+ * Copyright (c) 2016, 2017, 2018, 2019 FabricMC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.fabricmc.fabric.test.base;
+
+import java.time.Duration;
+import java.time.LocalDateTime;
+import java.util.Objects;
+import java.util.concurrent.CompletableFuture;
+import java.util.function.Function;
+import java.util.function.Predicate;
+
+import net.minecraft.client.MinecraftClient;
+import net.minecraft.client.gui.Drawable;
+import net.minecraft.client.gui.Element;
+import net.minecraft.client.gui.screen.GameMenuScreen;
+import net.minecraft.client.gui.screen.LevelLoadingScreen;
+import net.minecraft.client.gui.screen.Screen;
+import net.minecraft.client.gui.screen.ingame.CreativeInventoryScreen;
+import net.minecraft.client.gui.screen.ingame.InventoryScreen;
+import net.minecraft.client.gui.widget.ButtonWidget;
+import net.minecraft.client.gui.widget.CyclingButtonWidget;
+import net.minecraft.client.gui.widget.GridWidget;
+import net.minecraft.client.gui.widget.PressableWidget;
+import net.minecraft.client.option.Perspective;
+import net.minecraft.client.util.ScreenshotRecorder;
+import net.minecraft.text.Text;
+
+import net.fabricmc.fabric.test.base.mixin.CyclingButtonWidgetAccessor;
+import net.fabricmc.fabric.test.base.mixin.ScreenAccessor;
+import net.fabricmc.loader.api.FabricLoader;
+
+// Provides thread safe utils for interacting with a running game.
+public final class FabricClientTestHelper {
+	public static void waitForLoadingComplete() {
+		waitFor("Loading to complete", client -> client.getOverlay() == null, Duration.ofMinutes(5));
+	}
+
+	public static void waitForScreen(Class<? extends Screen> screenClass) {
+		waitFor("Screen %s".formatted(screenClass.getName()), client -> client.currentScreen != null && client.currentScreen.getClass() == screenClass);
+	}
+
+	public static void openGameMenu() {
+		setScreen((client) -> new GameMenuScreen(true));
+		waitForScreen(GameMenuScreen.class);
+	}
+
+	public static void openInventory() {
+		setScreen((client) -> new InventoryScreen(Objects.requireNonNull(client.player)));
+
+		boolean creative = submitAndWait(client -> Objects.requireNonNull(client.player).isCreative());
+		waitForScreen(creative ? CreativeInventoryScreen.class : InventoryScreen.class);
+	}
+
+	public static void closeScreen() {
+		setScreen((client) -> null);
+	}
+
+	private static void setScreen(Function<MinecraftClient, Screen> screenSupplier) {
+		submit(client -> {
+			client.setScreen(screenSupplier.apply(client));
+			return null;
+		});
+	}
+
+	public static void takeScreenshot(String name) {
+		// Allow time for any screens to open
+		waitFor(Duration.ofSeconds(1));
+
+		submitAndWait(client -> {
+			ScreenshotRecorder.saveScreenshot(FabricLoader.getInstance().getGameDir().toFile(), name + ".png", client.getFramebuffer(), (message) -> {
+			});
+			return null;
+		});
+	}
+
+	public static void clickScreenButton(String translationKey) {
+		final String buttonText = Text.translatable(translationKey).getString();
+
+		waitFor("Click button" + buttonText, client -> {
+			final Screen screen = client.currentScreen;
+
+			if (screen == null) {
+				return false;
+			}
+
+			final ScreenAccessor screenAccessor = (ScreenAccessor) screen;
+
+			for (Drawable drawable : screenAccessor.getDrawables()) {
+				if (drawable instanceof PressableWidget pressableWidget && pressMatchingButton(pressableWidget, buttonText)) {
+					return true;
+				}
+
+				if (drawable instanceof GridWidget gridWidget) {
+					for (Element child : gridWidget.children()) {
+						if (child instanceof PressableWidget pressableWidget && pressMatchingButton(pressableWidget, buttonText)) {
+							return true;
+						}
+					}
+				}
+			}
+
+			// Was unable to find the button to press
+			return false;
+		});
+	}
+
+	private static boolean pressMatchingButton(PressableWidget widget, String text) {
+		if (widget instanceof ButtonWidget buttonWidget) {
+			if (text.equals(buttonWidget.getMessage().getString())) {
+				buttonWidget.onPress();
+				return true;
+			}
+		}
+
+		if (widget instanceof CyclingButtonWidget<?> buttonWidget) {
+			CyclingButtonWidgetAccessor accessor = (CyclingButtonWidgetAccessor) buttonWidget;
+
+			if (text.equals(accessor.getOptionText().getString())) {
+				buttonWidget.onPress();
+				return true;
+			}
+		}
+
+		return false;
+	}
+
+	public static void waitForWorldTicks(long ticks) {
+		// Wait for the world to be loaded and get the start ticks
+		waitFor("World load", client -> client.world != null && !(client.currentScreen instanceof LevelLoadingScreen), Duration.ofMinutes(30));
+		final long startTicks = submitAndWait(client -> client.world.getTime());
+		waitFor("World load", client -> Objects.requireNonNull(client.world).getTime() > startTicks + ticks, Duration.ofMinutes(10));
+	}
+
+	public static void enableDebugHud() {
+		submitAndWait(client -> {
+			client.options.debugEnabled = true;
+			return null;
+		});
+	}
+
+	public static void setPerspective(Perspective perspective) {
+		submitAndWait(client -> {
+			client.options.setPerspective(perspective);
+			return null;
+		});
+	}
+
+	private static void waitFor(String what, Predicate<MinecraftClient> predicate) {
+		waitFor(what, predicate, Duration.ofSeconds(10));
+	}
+
+	private static void waitFor(String what, Predicate<MinecraftClient> predicate, Duration timeout) {
+		final LocalDateTime end = LocalDateTime.now().plus(timeout);
+
+		while (true) {
+			boolean result = submitAndWait(predicate::test);
+
+			if (result) {
+				break;
+			}
+
+			if (LocalDateTime.now().isAfter(end)) {
+				throw new RuntimeException("Timed out waiting for " + what);
+			}
+
+			waitFor(Duration.ofSeconds(1));
+		}
+	}
+
+	private static void waitFor(Duration duration) {
+		try {
+			Thread.sleep(duration.toMillis());
+		} catch (InterruptedException e) {
+			throw new RuntimeException(e);
+		}
+	}
+
+	private static <T> CompletableFuture<T> submit(Function<MinecraftClient, T> function) {
+		return MinecraftClient.getInstance().submit(() -> function.apply(MinecraftClient.getInstance()));
+	}
+
+	private static <T> T submitAndWait(Function<MinecraftClient, T> function) {
+		return submit(function).join();
+	}
+}

fabric-api-base/src/testmod/java/net/fabricmc/fabric/test/base/mixin/CyclingButtonWidgetAccessor.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2016, 2017, 2018, 2019 FabricMC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.fabricmc.fabric.test.base.mixin;
+
+import org.spongepowered.asm.mixin.Mixin;
+import org.spongepowered.asm.mixin.gen.Accessor;
+
+import net.minecraft.client.gui.widget.CyclingButtonWidget;
+import net.minecraft.text.Text;
+
+@Mixin(CyclingButtonWidget.class)
+public interface CyclingButtonWidgetAccessor {
+	@Accessor
+	Text getOptionText();
+}

fabric-api-base/src/testmod/java/net/fabricmc/fabric/test/base/mixin/ScreenAccessor.java

@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2016, 2017, 2018, 2019 FabricMC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.fabricmc.fabric.test.base.mixin;
+
+import java.util.List;
+
+import org.spongepowered.asm.mixin.Mixin;
+import org.spongepowered.asm.mixin.gen.Accessor;
+
+import net.minecraft.client.gui.Drawable;
+import net.minecraft.client.gui.screen.Screen;
+
+@Mixin(Screen.class)
+public interface ScreenAccessor {
+	@Accessor
+	List<Drawable> getDrawables();
+}

fabric-api-base/src/testmod/resources/fabric-api-base-testmod.mixins.json

@@ -0,0 +1,12 @@
+{
+  "required": true,
+  "package": "net.fabricmc.fabric.test.base.mixin",
+  "compatibilityLevel": "JAVA_17",
+  "mixins": [
+    "CyclingButtonWidgetAccessor",
+    "ScreenAccessor"
+  ],
+  "injectors": {
+    "defaultRequire": 1
+  }
+}

fabric-api-base/src/testmod/resources/fabric.mod.json

@@ -9,8 +9,17 @@
     "main": [
       "net.fabricmc.fabric.test.base.FabricApiBaseTestInit"
     ],
+    "client": [
+      "net.fabricmc.fabric.test.base.FabricApiAutoTestClient"
+    ],
+    "server": [
+      "net.fabricmc.fabric.test.base.FabricApiAutoTestServer"
+    ],
     "fabric-gametest" : [
       "net.fabricmc.fabric.test.base.FabricApiBaseGameTest"
     ]
-  }
+  },
+  "mixins": [
+    "fabric-api-base-testmod.mixins.json"
+  ]
 }

fabric-transfer-api-v1/src/testmod/java/net/fabricmc/fabric/test/transfer/ingame/client/FluidVariantRenderTest.java

@@ -54,6 +54,8 @@ public class FluidVariantRenderTest implements ClientModInitializer {
 			PlayerEntity player = MinecraftClient.getInstance().player;
 			if (player == null) return;
 
+			if (MinecraftClient.getInstance().options.debugEnabled) return;
+
 			int renderY = 0;
 			List<FluidVariant> variants = List.of(FluidVariant.of(Fluids.WATER), FluidVariant.of(Fluids.LAVA));