feat: build test for sparo outputs

build-tests/sparo-outputs/.eslintrc.js

@@ -0,0 +1,19 @@
+// This is a workaround for https://github.com/eslint/eslint/issues/3458
+require('@rushstack/heft-node-rig/profiles/default/includes/eslint/patch/modern-module-resolution');
+// This is a workaround for https://github.com/microsoft/rushstack/issues/3021
+require('@rushstack/heft-node-rig/profiles/default/includes/eslint/patch/custom-config-package-names');
+
+module.exports = {
+  extends: [
+    '@rushstack/heft-node-rig/profiles/default/includes/eslint/profile/node-trusted-tool',
+    '@rushstack/heft-node-rig/profiles/default/includes/eslint/mixins/friendly-locals'
+  ],
+  parserOptions: { tsconfigRootDir: __dirname },
+
+  overrides: [
+    {
+      files: ['*.ts', '*.tsx'],
+      rules: {}
+    }
+  ]
+};

build-tests/sparo-outputs/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) TikTok Pte. Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

build-tests/sparo-outputs/README.md

@@ -0,0 +1,43 @@
+# Sparo
+
+<div>
+  <br />
+  <a href="https://tiktok.github.io/sparo/">
+    <img width="380" src="https://tiktok.github.io/sparo/images/site/sparo-logo.svg" alt="Sparo" />
+  </a>
+  <p />
+</div>
+
+## Is Git too slow in your frontend monorepo?
+
+By default `git clone` will download every file in your Git repository, as well as the complete history of every file. For small repositories, that's no big deal. But as your monorepo accumulates projects and years of history, Git operations become slower and slower, until one day `git status` is taking 10 seconds or more. What to do?
+
+<!-- Text below this line should stay in sync with the website index.md -->
+<!-- ------------------------------------------------------------------ -->
+
+## Clone faster!
+
+Sparo optimizes performance of Git operations for your large frontend monorepo.
+
+## Key features
+
+- **Familiar interface:** The `sparo` command-line interface (CLI) wrapper offers **better defaults** and **performance suggestions** without altering the familiar `git` syntax. (The native `git` CLI is also supported.)
+- **A proven solution:** Git provides [quite a lot of ingredients](https://tiktok.github.io/sparo/pages/reference/git_optimization/) for optimizing very large repos; Sparo is your recipe for combining these features intelligently.
+- **Simplified sparse checkout:** Work with sparse checkout [profiles](https://tiktok.github.io/sparo/pages/guide/sparo_profiles/) instead of confusing "cones" and globs
+- **Frontend integration:** Sparo leverages [Rush](https://rushjs.io/) and [PNPM](https://pnpm.io/) workspace configurations, including the ability to automatically checkout project dependencies
+- **Dual workflows:** The `sparo-ci` tool implements a specialized checkout model optimized for continuous integration (CI) pipelines
+- **Extra safeguards**: Avoid common Git mistakes such as checkouts with staged files outside the active view
+- **Go beyond Git hooks:** Optionally collect anonymized Git timing metrics in your monorepo, enabling your build team to set data-driven goals for _local_ developer experience (not just CI!)
+
+  _(Metrics are transmitted to your own service and are not accessible by any other party.)_
+
+<!-- ------------------------------------------------------------------ -->
+<!-- Text above this line should stay in sync with the website index.md -->
+
+## Links
+
+- [Quick demo](https://tiktok.github.io/sparo/#quick-demo): See for yourself in 3 minutes!
+- [Getting Started](https://tiktok.github.io/sparo/pages/guide/getting_started/): Step by step instructions
+- [CHANGELOG.md](
+  https://github.com/tiktok/sparo/blob/main/apps/sparo/CHANGELOG.md): Find
+  out what's new in the latest version

build-tests/sparo-outputs/config/heft.json

@@ -0,0 +1,24 @@
+{
+  "$schema": "https://developer.microsoft.com/json-schemas/heft/v0/heft.schema.json",
+
+  "extends": "@rushstack/heft-node-rig/profiles/default/config/heft.json",
+
+  "phasesByName": {
+    "build": {
+      "tasksByName": {
+        "post-compile": {
+          "taskDependencies": ["typescript"],
+
+          "taskPlugin": {
+            "pluginName": "run-script-plugin",
+            "pluginPackage": "@rushstack/heft",
+
+            "options": {
+              "scriptPath": "lib/generateOutputs.js"
+            }
+          }
+        }
+      }
+    }
+  }
+}

build-tests/sparo-outputs/config/jest.config.json

@@ -0,0 +1,3 @@
+{
+  "extends": "@rushstack/heft-node-rig/profiles/default/config/jest.config.json"
+}

build-tests/sparo-outputs/config/rig.json

@@ -0,0 +1,5 @@
+{
+  "$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json",
+
+  "rigPackageName": "@rushstack/heft-node-rig"
+}

build-tests/sparo-outputs/etc/checkout-help.txt

@@ -0,0 +1,23 @@
+Running "sparo checkout --help":
+
+Sparo accelerator for Git __VERSION__ - https://tiktok.github.io/sparo/
+Node.js version is __VERSION__ (LTS)
+Git version is __VERSION__
+
+sparo checkout [branch] [start-point]
+
+Updates files in the working tree to match the version in the index or the
+specified tree. If no pathspec was given, git checkout will also update HEAD to
+set the specified branch as the current branch.
+
+Positionals:
+  branch                                                                [string]
+  start-point                                                           [string]
+
+Options:
+  -h, --help         Show help                                         [boolean]
+  -b                 Create a new branch and start it at <start-point> [boolean]
+  -B                 Create a new branch and start it at <start-point>; if it
+                     already exists, reset it to <start-point>         [boolean]
+      --profile                                            [array] [default: []]
+      --add-profile                                        [array] [default: []]

build-tests/sparo-outputs/etc/clone-help.txt

@@ -0,0 +1,25 @@
+Running "sparo clone --help":
+
+Sparo accelerator for Git __VERSION__ - https://tiktok.github.io/sparo/
+Node.js version is __VERSION__ (LTS)
+Git version is __VERSION__
+
+sparo clone <repository> [directory]
+
+Positionals:
+  repository  The remote repository to clone from.           [string] [required]
+  directory   The name of a new directory to clone into. The "humanish" part of
+              the source repository is used if no directory is explicitly given
+              (repo for /path/to/repo.gitService and foo for
+              host.xz:foo/.gitService). Cloning into an existing directory is
+              only allowed if the directory is empty                    [string]
+
+Options:
+  -h, --help             Show help                                     [boolean]
+  -s, --skip-git-config  By default, Sparo automatically configures the
+                         recommended git settings for the repository you are
+                         about to clone. If you prefer not to include this step,
+                         you can use the input parameter --skip-git-config
+                                                      [boolean] [default: false]
+  -b, --branch           Specify a branch to clone                      [string]
+      --profile                                            [array] [default: []]

build-tests/sparo-outputs/etc/top-level-help.txt

@@ -0,0 +1,30 @@
+Running "sparo --help":
+
+Sparo accelerator for Git __VERSION__ - https://tiktok.github.io/sparo/
+Node.js version is __VERSION__ (LTS)
+Git version is __VERSION__
+
+sparo [command]
+
+Commands:
+  sparo auto-config                      Automatic setup optimized git config
+  sparo list-profiles                    List all available profiles or query
+                                         profiles that contain the specified
+                                         project name
+  sparo init-profile                     Initialize a new profile.
+  sparo clone <repository> [directory]
+  sparo checkout [branch] [start-point]  Updates files in the working tree to
+                                         match the version in the index or the
+                                         specified tree. If no pathspec was
+                                         given, git checkout will also update
+                                         HEAD to set the specified branch as the
+                                         current branch.
+  sparo fetch [remote] [branch]          fetch remote branch to local
+  sparo git-clone                        original git clone command
+  sparo git-checkout                     original git checkout command
+  sparo git-fetch                        original git fetch command
+  sparo git-pull                         original git pull command
+
+Options:
+  -h, --help     Show help                                             [boolean]
+      --version  Show version number                                   [boolean]

build-tests/sparo-outputs/etc/top-level-nonexistent-command.txt

@@ -0,0 +1,10 @@
+Running "sparo nonexistent-command":
+
+Sparo accelerator for Git __VERSION__ - https://tiktok.github.io/sparo/
+Node.js version is __VERSION__ (LTS)
+Git version is __VERSION__
+
+
+--[ git nonexistent-command ]--------------------------------------------------
+-------------------------------------------------------------------------------
+

build-tests/sparo-outputs/package.json

@@ -0,0 +1,26 @@
+{
+  "name": "sparo-outputs",
+  "description": "Building this project tests sparo command outputs",
+  "version": "1.0.0",
+  "private": true,
+  "scripts": {
+    "_phase:build": "heft run --only build -- --clean",
+    "_phase:test": "heft run --only test -- --clean",
+    "test": "heft test --clean",
+    "build": "heft build --clean"
+  },
+  "dependencies": {
+    "@rushstack/node-core-library": "~3.64.2",
+    "sparo": "workspace:*"
+  },
+  "devDependencies": {
+    "@rushstack/heft": "0.64.3",
+    "@rushstack/heft-node-rig": "2.4.5",
+    "@types/heft-jest": "1.0.6",
+    "@types/node": "20.11.16",
+    "eslint": "8.56.0",
+    "typescript": "~5.3.3"
+  },
+  "keywords": [],
+  "license": "MIT"
+}

build-tests/sparo-outputs/src/generateOutputs.ts

@@ -0,0 +1,202 @@
+import * as path from 'path';
+import { Async, Executable, FileSystem, type FolderItem, Text } from '@rushstack/node-core-library';
+import type { SpawnSyncReturns } from 'child_process';
+import type { IRunScriptOptions } from '@rushstack/heft';
+
+interface IScenarioDefinition {
+  /**
+   * The scenario name. It is used to generate the output file name.
+   *
+   * For example, if the name is "top-level-help", the output file name will be "top-level-help.txt".
+   */
+  name: string;
+  /**
+   * The command line arguments to run. This doesn't include the command name itself.
+   *
+   * For example, if the command is "sparo clone --help", the args will be ["clone", "--help"].
+   */
+  args: string[];
+}
+
+/**
+ * This build test is highly inspired by the build test for api-extractor in rushstack.
+ */
+export async function runAsync(runScriptOptions: IRunScriptOptions): Promise<void> {
+  const {
+    heftTaskSession: {
+      logger,
+      parameters: { production }
+    },
+    heftConfiguration: { buildFolderPath }
+  } = runScriptOptions;
+  const binPath: string = path.join(buildFolderPath, 'node_modules', '.bin', 'sparo');
+  const tempFolder: string = path.join(buildFolderPath, 'temp', 'etc');
+
+  const scenarios: IScenarioDefinition[] = [
+    {
+      name: 'top-level-help',
+      args: ['--help']
+    },
+    {
+      name: 'clone-help',
+      args: ['clone', '--help']
+    },
+    {
+      name: 'checkout-help',
+      args: ['checkout', '--help']
+    },
+    {
+      name: 'top-level-nonexistent-command',
+      args: ['nonexistent-command']
+    }
+    // FIXME: This is currently broken -- it simply ignores the unrecognized parameter
+    // {
+    //   name: 'checkout-nonexistent-parameter',
+    //   args: ['checkout', '--nonexistent-parameter']
+    // }
+  ];
+
+  /**
+   * Run each scenario and generate outputs in parallel
+   *
+   * NOTE: please make sure the commands can be run in parallel
+   */
+  await FileSystem.ensureEmptyFolderAsync(tempFolder);
+  await Async.forEachAsync(
+    scenarios,
+    async (scenario: IScenarioDefinition) => {
+      const { name, args } = scenario;
+      const result: SpawnSyncReturns<string> = Executable.spawnSync(binPath, args);
+
+      if (result.status !== 0) {
+        throw new Error(
+          `Failed to run "sparo ${args.join(' ')}" with exit code ${result.status}\n${result.stderr}`
+        );
+      }
+
+      const outputPath: string = path.join(tempFolder, `${name}.txt`);
+      FileSystem.writeFile(
+        outputPath,
+        `Running "sparo ${args.join(' ')}":\n${processVersionString(result.stdout)}`
+      );
+    },
+    {
+      concurrency: 10
+    }
+  );
+
+  /**
+   * Files under outFolderPath are tracked by Git, files under inFolderPath are temporary files. During a local build,
+   * --production is false, temporary files are copied to outFolderPath. During a CI build, --production is true, the
+   * files with same name under these two folders are compared and CI build fails if they are different.
+   *
+   * This ensures that the temporary files must be up to date in the PR, and people who review the PR must approve any
+   * changes.
+   */
+  const inFolderPath: string = tempFolder;
+  const outFolderPath: string = `${buildFolderPath}/etc`;
+  await FileSystem.ensureFolderAsync(outFolderPath);
+
+  const inFolderPaths: AsyncIterable<string> = enumerateFolderPaths(inFolderPath, '');
+  const outFolderPaths: AsyncIterable<string> = enumerateFolderPaths(outFolderPath, '');
+  const outFolderPathsSet: Set<string> = new Set<string>();
+
+  for await (const outFolderPath of outFolderPaths) {
+    outFolderPathsSet.add(outFolderPath);
+  }
+
+  const nonMatchingFiles: string[] = [];
+  await Async.forEachAsync(
+    inFolderPaths,
+    async (folderItemPath: string) => {
+      outFolderPathsSet.delete(folderItemPath);
+
+      const sourceFileContents: string = await FileSystem.readFileAsync(inFolderPath + folderItemPath);
+      const outFilePath: string = outFolderPath + folderItemPath;
+
+      let outFileContents: string | undefined;
+      try {
+        outFileContents = await FileSystem.readFileAsync(outFilePath);
+      } catch (e) {
+        if (!FileSystem.isNotExistError(e)) {
+          throw e;
+        }
+      }
+
+      const normalizedSourceFileContents: string = Text.convertToLf(sourceFileContents);
+      const normalizedOutFileContents: string | undefined = outFileContents
+        ? Text.convertToLf(outFileContents)
+        : undefined;
+
+      if (normalizedSourceFileContents !== normalizedOutFileContents) {
+        nonMatchingFiles.push(outFilePath);
+        if (!production) {
+          await FileSystem.writeFileAsync(outFilePath, normalizedSourceFileContents, {
+            ensureFolderExists: true
+          });
+        }
+      }
+    },
+    {
+      concurrency: 10
+    }
+  );
+
+  if (outFolderPathsSet.size > 0) {
+    nonMatchingFiles.push(...outFolderPathsSet);
+    if (!production) {
+      await Async.forEachAsync(
+        outFolderPathsSet,
+        async (outFolderPath) => {
+          await FileSystem.deleteFileAsync(`${outFolderPath}/${outFolderPath}`);
+        },
+        { concurrency: 10 }
+      );
+    }
+  }
+
+  if (nonMatchingFiles.length > 0) {
+    const errorLines: string[] = [];
+    for (const nonMatchingFile of nonMatchingFiles.sort()) {
+      errorLines.push(`  ${nonMatchingFile}`);
+    }
+
+    if (production) {
+      logger.emitError(
+        new Error(
+          'The following file(s) do not match the expected output. Build this project in non-production ' +
+            `mode and commit the changes:\n${errorLines.join('\n')}`
+        )
+      );
+    } else {
+      logger.emitWarning(
+        new Error(
+          `The following file(s) do not match the expected output and must be committed to Git:\n` +
+            errorLines.join('\n')
+        )
+      );
+    }
+  }
+}
+
+/**
+ * Replace all x.y.z version strings with __VERSION__.
+ */
+function processVersionString(text: string): string {
+  return text.replace(/\d+\.\d+\.\d+/g, '__VERSION__');
+}
+
+async function* enumerateFolderPaths(
+  absoluteFolderPath: string,
+  relativeFolderPath: string
+): AsyncIterable<string> {
+  const folderItems: FolderItem[] = await FileSystem.readFolderItemsAsync(absoluteFolderPath);
+  for (const folderItem of folderItems) {
+    const childRelativeFolderPath: string = `${relativeFolderPath}/${folderItem.name}`;
+    if (folderItem.isDirectory()) {
+      yield* enumerateFolderPaths(`${absoluteFolderPath}/${folderItem.name}`, childRelativeFolderPath);
+    } else {
+      yield childRelativeFolderPath;
+    }
+  }
+}

build-tests/sparo-outputs/tsconfig.json

@@ -0,0 +1,8 @@
+{
+  "$schema": "http://json.schemastore.org/tsconfig",
+
+  "extends": "./node_modules/@rushstack/heft-node-rig/profiles/default/tsconfig-base.json",
+  "compilerOptions": {
+    "types": ["node", "heft-jest"]
+  }
+}

common/config/rush/pnpm-lock.yaml

@@ -149,6 +149,34 @@ importers:
         specifier: ~5.3.3
         version: 5.3.3
 
+  ../../build-tests/sparo-outputs:
+    dependencies:
+      '@rushstack/node-core-library':
+        specifier: ~3.64.2
+        version: 3.64.2(@types/node@20.11.16)
+      sparo:
+        specifier: workspace:*
+        version: link:../../apps/sparo
+    devDependencies:
+      '@rushstack/heft':
+        specifier: 0.64.3
+        version: 0.64.3(@types/node@20.11.16)
+      '@rushstack/heft-node-rig':
+        specifier: 2.4.5
+        version: 2.4.5(@rushstack/heft@0.64.3)(@types/node@20.11.16)
+      '@types/heft-jest':
+        specifier: 1.0.6
+        version: 1.0.6
+      '@types/node':
+        specifier: 20.11.16
+        version: 20.11.16
+      eslint:
+        specifier: 8.56.0
+        version: 8.56.0
+      typescript:
+        specifier: ~5.3.3
+        version: 5.3.3
+
 packages:
 
   /@aashutoshrathi/word-wrap@1.2.6:
@@ -2548,7 +2576,7 @@ packages:
       '@jest/console': 29.7.0
       '@jest/reporters': 29.5.0
       '@jest/test-result': 29.7.0
-      '@jest/transform': 29.5.0
+      '@jest/transform': 29.7.0
       '@jest/types': 29.6.3
       '@types/node': 20.11.16
       ansi-escapes: 4.3.2
@@ -2561,11 +2589,11 @@ packages:
       jest-haste-map: 29.7.0
       jest-message-util: 29.7.0
       jest-regex-util: 29.6.3
-      jest-resolve: 29.5.0
+      jest-resolve: 29.7.0
       jest-resolve-dependencies: 29.7.0
       jest-runner: 29.7.0
       jest-runtime: 29.7.0
-      jest-snapshot: 29.5.0
+      jest-snapshot: 29.7.0
       jest-util: 29.7.0
       jest-validate: 29.7.0
       jest-watcher: 29.7.0
@@ -2642,7 +2670,7 @@ packages:
       '@bcoe/v8-coverage': 0.2.3
       '@jest/console': 29.7.0
       '@jest/test-result': 29.7.0
-      '@jest/transform': 29.5.0
+      '@jest/transform': 29.7.0
       '@jest/types': 29.6.3
       '@jridgewell/trace-mapping': 0.3.23
       '@types/node': 20.11.16
@@ -7722,10 +7750,10 @@ packages:
       glob: 7.2.3
       graceful-fs: 4.2.11
       jest-circus: 29.7.0
-      jest-environment-node: 29.5.0
+      jest-environment-node: 29.7.0
       jest-get-type: 29.6.3
       jest-regex-util: 29.6.3
-      jest-resolve: 29.5.0
+      jest-resolve: 29.7.0
       jest-runner: 29.7.0
       jest-util: 29.7.0
       jest-validate: 29.7.0
@@ -7996,7 +8024,7 @@ packages:
       '@babel/traverse': 7.23.9
       '@babel/types': 7.23.9
       '@jest/expect-utils': 29.7.0
-      '@jest/transform': 29.5.0
+      '@jest/transform': 29.7.0
       '@jest/types': 29.6.3
       '@types/babel__traverse': 7.20.5
       '@types/prettier': 2.7.3

rush.json

@@ -434,6 +434,14 @@
     //   "reviewCategory": "tools",
     //   "tags": [ "tools" ]
     // }
+
+    // Build tests
+    {
+      "packageName": "sparo-outputs",
+      "projectFolder": "build-tests/sparo-outputs"
+    },
+
+    // Sparo
     {
       "packageName": "sparo",
       "projectFolder": "apps/sparo",
@@ -446,6 +454,8 @@
       "versionPolicyName": "sparo",
       "shouldPublish": true
     },
+
+    // Document site
     {
       "packageName": "website",
       "projectFolder": "apps/website"