Merge pull request #52 from tiktok/feat-build-test-sparo-output

build-tests/sparo-output-test/.eslintrc.js

@@ -0,0 +1,19 @@
+// This is a workaround for https://github.com/eslint/eslint/issues/3458
+require('@rushstack/heft-node-rig/profiles/default/includes/eslint/patch/modern-module-resolution');
+// This is a workaround for https://github.com/microsoft/rushstack/issues/3021
+require('@rushstack/heft-node-rig/profiles/default/includes/eslint/patch/custom-config-package-names');
+
+module.exports = {
+  extends: [
+    '@rushstack/heft-node-rig/profiles/default/includes/eslint/profile/node-trusted-tool',
+    '@rushstack/heft-node-rig/profiles/default/includes/eslint/mixins/friendly-locals'
+  ],
+  parserOptions: { tsconfigRootDir: __dirname },
+
+  overrides: [
+    {
+      files: ['*.ts', '*.tsx'],
+      rules: {}
+    }
+  ]
+};

build-tests/sparo-output-test/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) TikTok Pte. Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

build-tests/sparo-output-test/README.md

@@ -0,0 +1,11 @@
+# sparo-output-test
+
+Building this project tests sparo command outputs
+
+# Details
+
+`lib/start-test.js` is run after building the project. This scripts generate the output text files under `temp/etc`. In local builds, those files are copied to `etc` folder. During a CI build, the files under these two folders are compared and the CI build fails if they are different. This ensures that files under `etc` folder must be up to date in the PR, and people who review the PR must approve any changes.
+
+# How to fix the build errors
+
+Run `rush build -t sparo-output-test` to regenerate files under `etc` folder and commit them into Git.

build-tests/sparo-output-test/config/heft.json

@@ -0,0 +1,24 @@
+{
+  "$schema": "https://developer.microsoft.com/json-schemas/heft/v0/heft.schema.json",
+
+  "extends": "@rushstack/heft-node-rig/profiles/default/config/heft.json",
+
+  "phasesByName": {
+    "build": {
+      "tasksByName": {
+        "post-compile": {
+          "taskDependencies": ["typescript"],
+
+          "taskPlugin": {
+            "pluginName": "run-script-plugin",
+            "pluginPackage": "@rushstack/heft",
+
+            "options": {
+              "scriptPath": "lib/start-test.js"
+            }
+          }
+        }
+      }
+    }
+  }
+}

build-tests/sparo-output-test/config/rig.json

@@ -0,0 +1,5 @@
+{
+  "$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json",
+
+  "rigPackageName": "@rushstack/heft-node-rig"
+}

build-tests/sparo-output-test/etc/checkout-help.txt

@@ -0,0 +1,23 @@
+Running "sparo checkout --help":
+
+Sparo accelerator for Git __VERSION__ - https://tiktok.github.io/sparo/
+Node.js version is __VERSION__ (LTS)
+Git version is __VERSION__
+
+sparo checkout [branch] [start-point]
+
+Updates files in the working tree to match the version in the index or the
+specified tree. If no pathspec was given, git checkout will also update HEAD to
+set the specified branch as the current branch.
+
+Positionals:
+  branch                                                                [string]
+  start-point                                                           [string]
+
+Options:
+  -h, --help         Show help                                         [boolean]
+  -b                 Create a new branch and start it at <start-point> [boolean]
+  -B                 Create a new branch and start it at <start-point>; if it
+                     already exists, reset it to <start-point>         [boolean]
+      --profile                                            [array] [default: []]
+      --add-profile                                        [array] [default: []]

build-tests/sparo-output-test/etc/clone-help.txt

@@ -0,0 +1,25 @@
+Running "sparo clone --help":
+
+Sparo accelerator for Git __VERSION__ - https://tiktok.github.io/sparo/
+Node.js version is __VERSION__ (LTS)
+Git version is __VERSION__
+
+sparo clone <repository> [directory]
+
+Positionals:
+  repository  The remote repository to clone from.           [string] [required]
+  directory   The name of a new directory to clone into. The "humanish" part of
+              the source repository is used if no directory is explicitly given
+              (repo for /path/to/repo.gitService and foo for
+              host.xz:foo/.gitService). Cloning into an existing directory is
+              only allowed if the directory is empty                    [string]
+
+Options:
+  -h, --help             Show help                                     [boolean]
+  -s, --skip-git-config  By default, Sparo automatically configures the
+                         recommended git settings for the repository you are
+                         about to clone. If you prefer not to include this step,
+                         you can use the input parameter --skip-git-config
+                                                      [boolean] [default: false]
+  -b, --branch           Specify a branch to clone                      [string]
+      --profile                                            [array] [default: []]

build-tests/sparo-output-test/etc/top-level-help.txt

@@ -0,0 +1,30 @@
+Running "sparo --help":
+
+Sparo accelerator for Git __VERSION__ - https://tiktok.github.io/sparo/
+Node.js version is __VERSION__ (LTS)
+Git version is __VERSION__
+
+sparo [command]
+
+Commands:
+  sparo auto-config                      Automatic setup optimized git config
+  sparo list-profiles                    List all available profiles or query
+                                         profiles that contain the specified
+                                         project name
+  sparo init-profile                     Initialize a new profile.
+  sparo clone <repository> [directory]
+  sparo checkout [branch] [start-point]  Updates files in the working tree to
+                                         match the version in the index or the
+                                         specified tree. If no pathspec was
+                                         given, git checkout will also update
+                                         HEAD to set the specified branch as the
+                                         current branch.
+  sparo fetch [remote] [branch]          fetch remote branch to local
+  sparo git-clone                        original git clone command
+  sparo git-checkout                     original git checkout command
+  sparo git-fetch                        original git fetch command
+  sparo git-pull                         original git pull command
+
+Options:
+  -h, --help     Show help                                             [boolean]
+      --version  Show version number                                   [boolean]

build-tests/sparo-output-test/etc/top-level-nonexistent-command.txt

@@ -0,0 +1,10 @@
+Running "sparo nonexistent-command":
+
+Sparo accelerator for Git __VERSION__ - https://tiktok.github.io/sparo/
+Node.js version is __VERSION__ (LTS)
+Git version is __VERSION__
+
+
+--[ git nonexistent-command ]--------------------------------------------------
+-------------------------------------------------------------------------------
+

build-tests/sparo-output-test/package.json

@@ -0,0 +1,25 @@
+{
+  "name": "sparo-output-test",
+  "description": "Building this project tests sparo command outputs",
+  "version": "1.0.0",
+  "private": true,
+  "scripts": {
+    "_phase:build": "heft run --only build -- --clean",
+    "build": "heft build --clean"
+  },
+  "dependencies": {
+    "@rushstack/node-core-library": "~3.64.2",
+    "sparo": "workspace:*",
+    "jest-diff": "~29.7.0"
+  },
+  "devDependencies": {
+    "@rushstack/heft": "0.64.3",
+    "@rushstack/heft-node-rig": "2.4.5",
+    "@types/heft-jest": "1.0.6",
+    "@types/node": "20.11.16",
+    "eslint": "8.56.0",
+    "typescript": "~5.3.3"
+  },
+  "keywords": [],
+  "license": "MIT"
+}

build-tests/sparo-output-test/src/start-test.ts

@@ -0,0 +1,213 @@
+import * as path from 'path';
+import { Async, Executable, FileSystem, type FolderItem, Text } from '@rushstack/node-core-library';
+import { diff } from 'jest-diff';
+import type { SpawnSyncReturns } from 'child_process';
+import type { IRunScriptOptions } from '@rushstack/heft';
+
+interface IScenarioDefinition {
+  /**
+   * The scenario name. It is used to generate the output file name.
+   *
+   * For example, if the name is "top-level-help", the output file name will be "top-level-help.txt".
+   */
+  name: string;
+  /**
+   * The command line arguments to run. This doesn't include the command name itself.
+   *
+   * For example, if the command is "sparo clone --help", the args will be ["clone", "--help"].
+   */
+  args: string[];
+}
+
+/**
+ * This build test is highly inspired by the build test for api-extractor in rushstack.
+ */
+export async function runAsync(runScriptOptions: IRunScriptOptions): Promise<void> {
+  const {
+    heftTaskSession: {
+      logger,
+      parameters: { production }
+    },
+    heftConfiguration: { buildFolderPath }
+  } = runScriptOptions;
+  const binPath: string = path.join(buildFolderPath, 'node_modules', '.bin', 'sparo');
+  const tempFolder: string = path.join(buildFolderPath, 'temp', 'etc');
+
+  const scenarios: IScenarioDefinition[] = [
+    {
+      name: 'top-level-help',
+      args: ['--help']
+    },
+    {
+      name: 'clone-help',
+      args: ['clone', '--help']
+    },
+    {
+      name: 'checkout-help',
+      args: ['checkout', '--help']
+    },
+    {
+      name: 'top-level-nonexistent-command',
+      args: ['nonexistent-command']
+    }
+    // FIXME: This is currently broken -- it simply ignores the unrecognized parameter
+    // {
+    //   name: 'checkout-nonexistent-parameter',
+    //   args: ['checkout', '--nonexistent-parameter']
+    // }
+  ];
+
+  /**
+   * Run each scenario and generate outputs
+   */
+  await FileSystem.ensureEmptyFolderAsync(tempFolder);
+  for (const scenario of scenarios) {
+    const { name, args } = scenario;
+    const result: SpawnSyncReturns<string> = Executable.spawnSync(binPath, args, {
+      environment: {
+        ...process.env,
+        // Always use color for the output
+        FORCE_COLOR: 'true'
+      }
+    });
+
+    if (result.status !== 0) {
+      throw new Error(
+        `Failed to run "sparo ${args.join(' ')}" with exit code ${result.status}\n${result.stderr}`
+      );
+    }
+
+    const outputPath: string = path.join(tempFolder, `${name}.txt`);
+    FileSystem.writeFile(
+      outputPath,
+      `Running "sparo ${args.join(' ')}":\n${processVersionString(result.stdout)}`
+    );
+  }
+
+  /**
+   * Files under outFolderPath are tracked by Git, files under inFolderPath are temporary files. During a local build,
+   * --production is false, temporary files are copied to outFolderPath. During a CI build, --production is true, the
+   * files with same name under these two folders are compared and CI build fails if they are different.
+   *
+   * This ensures that the temporary files must be up to date in the PR, and people who review the PR must approve any
+   * changes.
+   */
+  const inFolderPath: string = tempFolder;
+  const outFolderPath: string = `${buildFolderPath}/etc`;
+  await FileSystem.ensureFolderAsync(outFolderPath);
+
+  const inFolderPaths: AsyncIterable<string> = enumerateFolderPaths(inFolderPath, '');
+  const outFolderPaths: AsyncIterable<string> = enumerateFolderPaths(outFolderPath, '');
+  const outFolderPathsSet: Set<string> = new Set<string>();
+
+  for await (const outFolderPath of outFolderPaths) {
+    outFolderPathsSet.add(outFolderPath);
+  }
+
+  const nonMatchingFiles: string[] = [];
+  const nonMatchingFileErrorMessages: Map<string, string> = new Map<string, string>();
+  await Async.forEachAsync(
+    inFolderPaths,
+    async (folderItemPath: string) => {
+      outFolderPathsSet.delete(folderItemPath);
+
+      const sourceFileContents: string = await FileSystem.readFileAsync(inFolderPath + folderItemPath);
+      const outFilePath: string = outFolderPath + folderItemPath;
+
+      let outFileContents: string | undefined;
+      try {
+        outFileContents = await FileSystem.readFileAsync(outFilePath);
+      } catch (e) {
+        if (!FileSystem.isNotExistError(e)) {
+          throw e;
+        }
+      }
+
+      const normalizedSourceFileContents: string = Text.convertToLf(sourceFileContents);
+      const normalizedOutFileContents: string | undefined = outFileContents
+        ? Text.convertToLf(outFileContents)
+        : undefined;
+
+      if (normalizedSourceFileContents !== normalizedOutFileContents) {
+        nonMatchingFiles.push(outFilePath);
+        if (production) {
+          // Display diff only when running in production mode, mostly for CI build
+          nonMatchingFileErrorMessages.set(
+            outFilePath,
+            diff(normalizedOutFileContents, normalizedSourceFileContents) || ''
+          );
+        }
+        if (!production) {
+          await FileSystem.writeFileAsync(outFilePath, normalizedSourceFileContents, {
+            ensureFolderExists: true
+          });
+        }
+      }
+    },
+    {
+      concurrency: 10
+    }
+  );
+
+  if (outFolderPathsSet.size > 0) {
+    nonMatchingFiles.push(...outFolderPathsSet);
+    if (!production) {
+      await Async.forEachAsync(
+        outFolderPathsSet,
+        async (outFolderPath) => {
+          await FileSystem.deleteFileAsync(`${outFolderPath}/${outFolderPath}`);
+        },
+        { concurrency: 10 }
+      );
+    }
+  }
+
+  if (nonMatchingFiles.length > 0) {
+    const errorLines: string[] = [];
+    for (const nonMatchingFile of nonMatchingFiles.sort()) {
+      errorLines.push(`  ${nonMatchingFile}`);
+      const errorMessage: string | undefined = nonMatchingFileErrorMessages.get(nonMatchingFile);
+      if (errorMessage) {
+        errorLines.push(`${errorMessage}`);
+      }
+    }
+
+    if (production) {
+      logger.emitError(
+        new Error(
+          'The following file(s) do not match the expected output. Build this project in non-production ' +
+            `mode and commit the changes:\n${errorLines.join('\n')}`
+        )
+      );
+    } else {
+      logger.emitWarning(
+        new Error(
+          `The following file(s) do not match the expected output and must be committed to Git:\n` +
+            errorLines.join('\n')
+        )
+      );
+    }
+  }
+}
+
+/**
+ * Replace all x.y.z version strings with __VERSION__.
+ */
+function processVersionString(text: string): string {
+  return text.replace(/\d+\.\d+\.\d+/g, '__VERSION__');
+}
+
+async function* enumerateFolderPaths(
+  absoluteFolderPath: string,
+  relativeFolderPath: string
+): AsyncIterable<string> {
+  const folderItems: FolderItem[] = await FileSystem.readFolderItemsAsync(absoluteFolderPath);
+  for (const folderItem of folderItems) {
+    const childRelativeFolderPath: string = `${relativeFolderPath}/${folderItem.name}`;
+    if (folderItem.isDirectory()) {
+      yield* enumerateFolderPaths(`${absoluteFolderPath}/${folderItem.name}`, childRelativeFolderPath);
+    } else {
+      yield childRelativeFolderPath;
+    }
+  }
+}

build-tests/sparo-output-test/tsconfig.json

@@ -0,0 +1,8 @@
+{
+  "$schema": "http://json.schemastore.org/tsconfig",
+
+  "extends": "./node_modules/@rushstack/heft-node-rig/profiles/default/tsconfig-base.json",
+  "compilerOptions": {
+    "types": ["node", "heft-jest"]
+  }
+}

common/config/rush/pnpm-lock.yaml

@@ -149,6 +149,37 @@ importers:
         specifier: ~5.3.3
         version: 5.3.3
 
+  ../../build-tests/sparo-output-test:
+    dependencies:
+      '@rushstack/node-core-library':
+        specifier: ~3.64.2
+        version: 3.64.2(@types/node@20.11.16)
+      jest-diff:
+        specifier: ~29.7.0
+        version: 29.7.0
+      sparo:
+        specifier: workspace:*
+        version: link:../../apps/sparo
+    devDependencies:
+      '@rushstack/heft':
+        specifier: 0.64.3
+        version: 0.64.3(@types/node@20.11.16)
+      '@rushstack/heft-node-rig':
+        specifier: 2.4.5
+        version: 2.4.5(@rushstack/heft@0.64.3)(@types/node@20.11.16)
+      '@types/heft-jest':
+        specifier: 1.0.6
+        version: 1.0.6
+      '@types/node':
+        specifier: 20.11.16
+        version: 20.11.16
+      eslint:
+        specifier: 8.56.0
+        version: 8.56.0
+      typescript:
+        specifier: ~5.3.3
+        version: 5.3.3
+
 packages:
 
   /@aashutoshrathi/word-wrap@1.2.6:
@@ -2548,7 +2579,7 @@ packages:
       '@jest/console': 29.7.0
       '@jest/reporters': 29.5.0
       '@jest/test-result': 29.7.0
-      '@jest/transform': 29.5.0
+      '@jest/transform': 29.7.0
       '@jest/types': 29.6.3
       '@types/node': 20.11.16
       ansi-escapes: 4.3.2
@@ -2561,11 +2592,11 @@ packages:
       jest-haste-map: 29.7.0
       jest-message-util: 29.7.0
       jest-regex-util: 29.6.3
-      jest-resolve: 29.5.0
+      jest-resolve: 29.7.0
       jest-resolve-dependencies: 29.7.0
       jest-runner: 29.7.0
       jest-runtime: 29.7.0
-      jest-snapshot: 29.5.0
+      jest-snapshot: 29.7.0
       jest-util: 29.7.0
       jest-validate: 29.7.0
       jest-watcher: 29.7.0
@@ -2642,7 +2673,7 @@ packages:
       '@bcoe/v8-coverage': 0.2.3
       '@jest/console': 29.7.0
       '@jest/test-result': 29.7.0
-      '@jest/transform': 29.5.0
+      '@jest/transform': 29.7.0
       '@jest/types': 29.6.3
       '@jridgewell/trace-mapping': 0.3.23
       '@types/node': 20.11.16
@@ -4177,7 +4208,6 @@ packages:
   /ansi-styles@5.2.0:
     resolution: {integrity: sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==}
     engines: {node: '>=10'}
-    dev: true
 
   /ansi-styles@6.2.1:
     resolution: {integrity: sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==}
@@ -5484,7 +5514,6 @@ packages:
   /diff-sequences@29.6.3:
     resolution: {integrity: sha512-EjePK1srD3P08o2j4f0ExnylqRs5B9tJjcp9t1krH2qRi8CCdsYfwe9JgSLurFBWwq4uOlipzfk5fHNvwFKr8Q==}
     engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0}
-    dev: true
 
   /dir-glob@3.0.1:
     resolution: {integrity: sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==}
@@ -7722,10 +7751,10 @@ packages:
       glob: 7.2.3
       graceful-fs: 4.2.11
       jest-circus: 29.7.0
-      jest-environment-node: 29.5.0
+      jest-environment-node: 29.7.0
       jest-get-type: 29.6.3
       jest-regex-util: 29.6.3
-      jest-resolve: 29.5.0
+      jest-resolve: 29.7.0
       jest-runner: 29.7.0
       jest-util: 29.7.0
       jest-validate: 29.7.0
@@ -7747,7 +7776,6 @@ packages:
       diff-sequences: 29.6.3
       jest-get-type: 29.6.3
       pretty-format: 29.7.0
-    dev: true
 
   /jest-docblock@29.7.0:
     resolution: {integrity: sha512-q617Auw3A612guyaFgsbFeYpNP5t2aoUNLwBUbc/0kD1R4t9ixDbyFTHd1nok4epoVFpr7PmeWHrhvuV3XaJ4g==}
@@ -7794,7 +7822,6 @@ packages:
   /jest-get-type@29.6.3:
     resolution: {integrity: sha512-zrteXnqYxfQh7l5FHyL38jL39di8H8rHoecLH3JNxH3BwOrBsNeabdap5e0I23lD4HHI8W5VFBZqG4Eaq5LNcw==}
     engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0}
-    dev: true
 
   /jest-haste-map@29.7.0:
     resolution: {integrity: sha512-fP8u2pyfqx0K1rGn1R9pyE0/KTn+G7PxktWidOBTqFPLYX0b9ksaMFkhK5vrS3DVun09pckLdlx90QthlW7AmA==}
@@ -7996,7 +8023,7 @@ packages:
       '@babel/traverse': 7.23.9
       '@babel/types': 7.23.9
       '@jest/expect-utils': 29.7.0
-      '@jest/transform': 29.5.0
+      '@jest/transform': 29.7.0
       '@jest/types': 29.6.3
       '@types/babel__traverse': 7.20.5
       '@types/prettier': 2.7.3
@@ -10003,7 +10030,6 @@ packages:
       '@jest/schemas': 29.6.3
       ansi-styles: 5.2.0
       react-is: 18.2.0
-    dev: true
 
   /pretty-time@1.1.0:
     resolution: {integrity: sha512-28iF6xPQrP8Oa6uxE6a1biz+lWeTOAPKggvjB8HAs6nVMKZwf5bG++632Dx614hIWgUPkgivRfG+a8uAXGTIbA==}
@@ -10235,7 +10261,6 @@ packages:
 
   /react-is@18.2.0:
     resolution: {integrity: sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w==}
-    dev: true
 
   /react-json-view-lite@1.2.1(react@18.2.0):
     resolution: {integrity: sha512-Itc0g86fytOmKZoIoJyGgvNqohWSbh3NXIKNgH6W6FT9PC1ck4xas1tT3Rr/b3UlFXyA9Jjaw9QSXdZy2JwGMQ==}
@@ -12275,6 +12300,7 @@ time:
   /eslint@8.56.0: '2023-12-15T22:55:11.833Z'
   /git-repo-info@2.1.1: '2019-10-18T15:12:09.674Z'
   /inversify@6.0.2: '2023-10-20T23:35:39.918Z'
+  /jest-diff@29.7.0: '2023-09-12T06:43:43.883Z'
   /lunr@2.3.9: '2020-08-19T20:30:07.948Z'
   /prism-react-renderer@2.3.1: '2023-12-18T14:23:38.265Z'
   /prismjs@1.29.0: '2022-08-23T10:42:14.395Z'

rush.json

@@ -434,6 +434,14 @@
     //   "reviewCategory": "tools",
     //   "tags": [ "tools" ]
     // }
+
+    // Build tests
+    {
+      "packageName": "sparo-output-test",
+      "projectFolder": "build-tests/sparo-output-test"
+    },
+
+    // Sparo
     {
       "packageName": "sparo",
       "projectFolder": "apps/sparo",
@@ -446,6 +454,8 @@
       "versionPolicyName": "sparo",
       "shouldPublish": true
     },
+
+    // Document site
     {
       "packageName": "website",
       "projectFolder": "apps/website"