fix(core): optimize warm cache performance for task execution

Batch daemon calls, add bulk cache resolution fast path, and parallelize
output hash checking to dramatically improve warm cache hit performance.

- Batch daemon calls for recordOutputsHash and outputsHashesMatch
- Add resolveCachedTasksBulk fast path for bulk cache resolution
- Cache readProjectsConfigurationFromProjectGraph in getExecutorForTask
- Add verified match cache in daemon to skip redundant filesystem scans
- Add Rayon-parallel get_files_for_outputs_batch in Rust
- Batch-hash unhashed tasks per topological level
- Skip recordOutputsHash for local-cache-kept-existing

Author: FrozenPandaz

astro-docs/tsconfig.json

@@ -20,6 +20,9 @@
     "strict": true
   },
   "references": [
+    {
+      "path": "../packages/nx"
+    },
     {
       "path": "../nx-dev/ui-markdoc"
     },

benchmarks/goals.json

@@ -10,5 +10,8 @@
   },
   "copy-warm": {
     "max": 1.0
+  },
+  "build-warm": {
+    "max": 5.0
   }
 }

benchmarks/package.json

@@ -11,7 +11,7 @@
     "bench:show-projects": "NX_NO_CLOUD=true hyperfine --show-output --setup 'nx reset' --warmup 1 --min-runs 5 --export-json results-show-projects.json 'nx show projects --tui=false'",
     "bench:cat-warm": "NX_NO_CLOUD=true hyperfine --show-output --setup 'nx reset' --warmup 1 --min-runs 5 --export-json results-cat-warm.json 'nx run-many -t cat --tui=false'",
     "bench:copy-warm": "NX_NO_CLOUD=true hyperfine --show-output --setup 'nx reset' --warmup 1 --min-runs 5 --export-json results-copy-warm.json 'nx run-many -t copy --tui=false'",
-    "disabled:bench:build-warm": "NX_NO_CLOUD=true hyperfine --show-output --setup 'nx reset' --warmup 1 --min-runs 5 --export-json results-build-warm.json 'nx run-many -t build --tui=true --tui-auto-exit'"
+    "bench:build-warm": "NX_NO_CLOUD=true hyperfine --ignore-failure --setup 'nx reset' --warmup 1 --min-runs 5 --export-json results-build-warm.json 'nx run-many -t build --tui=true --tui-auto-exit'"
   },
   "nx": {
     "targets": {
@@ -20,7 +20,8 @@
           "bench:version",
           "bench:show-projects",
           "bench:cat-warm",
-          "bench:copy-warm"
+          "bench:copy-warm",
+          "bench:build-warm"
         ]
       },
       "bench:version": {
@@ -47,7 +48,7 @@
         ],
         "parallelism": false
       },
-      "disabled:bench:build-warm": {
+      "bench:build-warm": {
         "dependsOn": [
           "^build"
         ],

e2e/nx-init/tsconfig.json

@@ -6,6 +6,9 @@
   "include": [],
   "files": [],
   "references": [
+    {
+      "path": "../../packages/nx"
+    },
     {
       "path": "../utils"
     },

nx-dev/ui-markdoc/tsconfig.lib.json

@@ -28,6 +28,9 @@
   ],
   "include": ["**/*.js", "**/*.jsx", "**/*.ts", "**/*.tsx"],
   "references": [
+    {
+      "path": "../../packages/nx/tsconfig.lib.json"
+    },
     {
       "path": "../feature-analytics/tsconfig.lib.json"
     },

packages/create-nx-workspace/tsconfig.lib.json

@@ -14,5 +14,9 @@
     "jest.config.ts"
   ],
   "include": ["**/*.ts", "*.json", "migrations.json"],
-  "references": []
+  "references": [
+    {
+      "path": "../nx/tsconfig.lib.json"
+    }
+  ]
 }

packages/nest/tsconfig.lib.json

@@ -17,6 +17,9 @@
   ],
   "include": ["**/*.ts"],
   "references": [
+    {
+      "path": "../nx/tsconfig.lib.json"
+    },
     {
       "path": "../node/tsconfig.lib.json"
     },

packages/nx/src/daemon/client/client.ts

@@ -795,6 +795,15 @@ export class DaemonClient {
     });
   }
 
+  recordOutputsHashBatch(
+    entries: { outputs: string[]; hash: string }[]
+  ): Promise<any> {
+    return this.sendToDaemonViaQueue({
+      type: 'RECORD_OUTPUTS_HASH_BATCH',
+      data: entries,
+    });
+  }
+
   outputsHashesMatch(outputs: string[], hash: string): Promise<any> {
     return this.sendToDaemonViaQueue({
       type: 'OUTPUTS_HASHES_MATCH',
@@ -805,6 +814,15 @@ export class DaemonClient {
     });
   }
 
+  outputsHashesMatchBatch(
+    entries: { outputs: string[]; hash: string }[]
+  ): Promise<boolean[]> {
+    return this.sendToDaemonViaQueue({
+      type: 'OUTPUTS_HASHES_MATCH_BATCH',
+      data: entries,
+    });
+  }
+
   glob(globs: string[], exclude?: string[]): Promise<string[]> {
     const message: HandleGlobMessage = {
       type: 'GLOB',

packages/nx/src/daemon/server/handle-outputs-tracking.ts

@@ -1,5 +1,10 @@
 import { HandlerResult } from './server';
-import { outputsHashesMatch, recordOutputsHash } from './outputs-tracking';
+import {
+  outputsHashesMatch,
+  outputsHashesMatchBatch,
+  recordOutputsHash,
+  recordOutputsHashBatch,
+} from './outputs-tracking';
 
 export async function handleRecordOutputsHash(payload: {
   type: string;
@@ -21,6 +26,26 @@ export async function handleRecordOutputsHash(payload: {
   }
 }
 
+export async function handleRecordOutputsHashBatch(payload: {
+  type: string;
+  data: { outputs: string[]; hash: string }[];
+}): Promise<HandlerResult> {
+  try {
+    recordOutputsHashBatch(payload.data);
+    return {
+      description: 'recordOutputsHashBatch',
+      response: '{}',
+    };
+  } catch (e) {
+    return {
+      description: 'recordOutputsHashBatch failed',
+      error: new Error(
+        `Critical error when recording metadata about outputs: '${e.message}'.`
+      ),
+    };
+  }
+}
+
 export async function handleOutputsHashesMatch(payload: {
   type: string;
   data: { outputs: string[]; hash: string };
@@ -43,3 +68,23 @@ export async function handleOutputsHashesMatch(payload: {
     };
   }
 }
+
+export async function handleOutputsHashesMatchBatch(payload: {
+  type: string;
+  data: { outputs: string[]; hash: string }[];
+}): Promise<HandlerResult> {
+  try {
+    const results = outputsHashesMatchBatch(payload.data);
+    return {
+      response: results,
+      description: 'outputsHashesMatchBatch',
+    };
+  } catch (e) {
+    return {
+      description: 'outputsHashesMatchBatch failed',
+      error: new Error(
+        `Critical error when verifying the contents of the outputs haven't changed: '${e.message}'.`
+      ),
+    };
+  }
+}

packages/nx/src/daemon/server/outputs-tracking.ts

@@ -1,5 +1,9 @@
 import { dirname } from 'path';
-import { WatchEvent, getFilesForOutputs } from '../../native';
+import {
+  WatchEvent,
+  getFilesForOutputs,
+  getFilesForOutputsBatch,
+} from '../../native';
 import { collapseExpandedOutputs } from '../../utils/collapse-expanded-outputs';
 import { workspaceRoot } from '../../utils/workspace-root';
 
@@ -77,6 +81,7 @@ export function processFileChangesInOutputs(
     if (dirsContainingOutputs[current]) {
       dirsContainingOutputs[current].forEach((output) => {
         if (now - timestamps[output] > 2000) {
+          const hash = recordedHashes[output];
           recordedHashes[output] = undefined;
         }
       });
@@ -94,6 +99,46 @@ export function processFileChangesInOutputs(
   }
 }
 
+/**
+ * Batch version of outputsHashesMatch that uses Rayon-parallel
+ * filesystem scanning for uncached entries.
+ */
+export function outputsHashesMatchBatch(
+  entries: { outputs: string[]; hash: string }[]
+): boolean[] {
+  if (disabled) return entries.map(() => false);
+
+  // Batch filesystem scan in parallel via Rust/Rayon
+  const outputsBatch = entries.map((e) => e.outputs);
+  const expandedBatch = getFilesForOutputsBatch(workspaceRoot, outputsBatch);
+
+  const results: boolean[] = [];
+  for (let i = 0; i < entries.length; i++) {
+    const expanded = collapseExpandedOutputs(expandedBatch[i]);
+    results.push(_outputsHashesMatch(expanded, entries[i].hash));
+  }
+
+  return results;
+}
+
+/**
+ * Batch version of recordOutputsHash that uses Rayon-parallel
+ * filesystem scanning.
+ */
+export function recordOutputsHashBatch(
+  entries: { outputs: string[]; hash: string }[]
+) {
+  if (disabled) return;
+
+  const outputsBatch = entries.map((e) => e.outputs);
+  const expandedBatch = getFilesForOutputsBatch(workspaceRoot, outputsBatch);
+
+  for (let i = 0; i < entries.length; i++) {
+    const expanded = collapseExpandedOutputs(expandedBatch[i]);
+    _recordOutputsHash(expanded, entries[i].hash);
+  }
+}
+
 export function disableOutputsTracking() {
   disabled = true;
 }