[ci] add dist hash diffing and targeted cache purge flow

.github/workflows/ci.yml

@@ -57,6 +57,77 @@ jobs:
       - name: Build assets
         run: npm run build
 
+      - name: Generate dist hash state
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+        run: |
+          mkdir -p artifacts
+          node scripts/hash-dist.mjs --dist dist --output artifacts/dist-hashes-current.json
+
+      - name: Fetch prior dist hash state
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+        env:
+          DIST_HASH_STATE_URL: ${{ secrets.DIST_HASH_STATE_URL }}
+        run: |
+          if [ -n "$DIST_HASH_STATE_URL" ]; then
+            curl --fail --silent --show-error "$DIST_HASH_STATE_URL" -o artifacts/dist-hashes-previous.json
+            echo "Loaded prior dist hash state from authoritative store."
+          else
+            echo '{"combined":"","files":[]}' > artifacts/dist-hashes-previous.json
+            echo "No DIST_HASH_STATE_URL configured; using empty baseline."
+          fi
+
+      - name: Diff dist hash states
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+        run: |
+          node scripts/diff-dist-hashes.mjs \
+            --current artifacts/dist-hashes-current.json \
+            --previous artifacts/dist-hashes-previous.json > artifacts/dist-hashes-changed.txt
+          echo "Changed dist paths:"
+          cat artifacts/dist-hashes-changed.txt || true
+
+      - name: Plan cache purge scope
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+        run: |
+          node scripts/plan-cache-purge.mjs \
+            --changed-file artifacts/dist-hashes-changed.txt \
+            --output-file artifacts/dist-purge-targets.txt > artifacts/dist-purge-plan.json
+          cat artifacts/dist-purge-plan.json
+
+      - name: Purge changed Cloudflare URLs
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+        env:
+          CLOUDFLARE_ZONE_ID: ${{ secrets.CLOUDFLARE_ZONE_ID }}
+          CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
+          CLOUDFLARE_PURGE_BASE_URL: ${{ secrets.CLOUDFLARE_PURGE_BASE_URL }}
+        run: |
+          if [ ! -s artifacts/dist-purge-targets.txt ]; then
+            echo "No purge targets detected. Skipping purge."
+            exit 0
+          fi
+
+          if [ -z "$CLOUDFLARE_ZONE_ID" ] || [ -z "$CLOUDFLARE_API_TOKEN" ] || [ -z "$CLOUDFLARE_PURGE_BASE_URL" ]; then
+            echo "Cloudflare secrets not configured; skipping targeted purge."
+            exit 0
+          fi
+
+          node scripts/purge-cloudflare-changed.mjs \
+            --changed-file artifacts/dist-purge-targets.txt \
+            --base-url "$CLOUDFLARE_PURGE_BASE_URL"
+
+      - name: Record authoritative dist hash state (post-deploy)
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+        env:
+          DIST_HASH_STATE_UPLOAD_URL: ${{ secrets.DIST_HASH_STATE_UPLOAD_URL }}
+        run: |
+          if [ -z "$DIST_HASH_STATE_UPLOAD_URL" ]; then
+            echo "No DIST_HASH_STATE_UPLOAD_URL configured; skipping authoritative hash-state update."
+            exit 0
+          fi
+
+          node scripts/store-dist-hash-state.mjs \
+            --input artifacts/dist-hashes-current.json \
+            --upload-url "$DIST_HASH_STATE_UPLOAD_URL"
+
       - name: Install Playwright browsers
         run: npx playwright install --with-deps chromium
 

package.json

@@ -39,7 +39,10 @@
     "test:unit:shard3": "cross-env UNIT_TEST_SHARD=3/3 node scripts/run-unit-tests.mjs",
     "telemetry:aggregate": "node scripts/agent/telemetry-aggregator.mjs",
     "lint:assets": "node scripts/check-asset-references.mjs",
-    "lint:sw-compat": "node scripts/check-sw-compat.mjs"
+    "lint:sw-compat": "node scripts/check-sw-compat.mjs",
+    "hash:dist": "node scripts/hash-dist.mjs --dist dist --output artifacts/dist-hashes-current.json",
+    "diff:dist-hashes": "node scripts/diff-dist-hashes.mjs",
+    "purge:cloudflare:changed": "node scripts/purge-cloudflare-changed.mjs"
   },
   "devDependencies": {
     "@playwright/test": "^1.58.2",

scripts/diff-dist-hashes.mjs

@@ -0,0 +1,91 @@
+#!/usr/bin/env node
+import fs from 'node:fs';
+import path from 'node:path';
+
+function parseArgs(argv) {
+  const args = {
+    current: null,
+    previous: null,
+    outputJson: false
+  };
+
+  for (let i = 0; i < argv.length; i += 1) {
+    const arg = argv[i];
+
+    if (arg === '--current') {
+      args.current = argv[i + 1] ?? args.current;
+      i += 1;
+      continue;
+    }
+
+    if (arg === '--previous') {
+      args.previous = argv[i + 1] ?? args.previous;
+      i += 1;
+      continue;
+    }
+
+    if (arg === '--json') {
+      args.outputJson = true;
+      continue;
+    }
+
+    if (arg === '--help' || arg === '-h') {
+      console.log('Usage: node scripts/diff-dist-hashes.mjs --current <json> --previous <json> [--json]');
+      process.exit(0);
+    }
+
+    throw new Error(`Unknown argument: ${arg}`);
+  }
+
+  if (!args.current || !args.previous) {
+    throw new Error('Both --current and --previous are required.');
+  }
+
+  return args;
+}
+
+function readHashState(filePath) {
+  const resolved = path.resolve(filePath);
+  if (!fs.existsSync(resolved)) {
+    throw new Error(`Hash state file not found: ${resolved}`);
+  }
+
+  const payload = JSON.parse(fs.readFileSync(resolved, 'utf8'));
+  if (!Array.isArray(payload.files)) {
+    throw new Error(`Invalid hash state format in ${resolved}: missing files[] array.`);
+  }
+
+  return payload;
+}
+
+function toMap(state) {
+  return new Map(state.files.map((entry) => [entry.path, entry.sha256]));
+}
+
+function getChangedPaths(currentState, previousState) {
+  const current = toMap(currentState);
+  const previous = toMap(previousState);
+  const allPaths = new Set([...current.keys(), ...previous.keys()]);
+
+  return [...allPaths]
+    .filter((filePath) => current.get(filePath) !== previous.get(filePath))
+    .sort((a, b) => a.localeCompare(b));
+}
+
+function main() {
+  const args = parseArgs(process.argv.slice(2));
+  const currentState = readHashState(args.current);
+  const previousState = readHashState(args.previous);
+  const changedPaths = getChangedPaths(currentState, previousState);
+
+  if (args.outputJson) {
+    console.log(JSON.stringify({ changed: changedPaths }, null, 2));
+    return;
+  }
+
+  for (const changedPath of changedPaths) {
+    console.log(changedPath);
+  }
+}
+
+main();

scripts/hash-dist.mjs

@@ -0,0 +1,115 @@
+#!/usr/bin/env node
+import fs from 'node:fs';
+import path from 'node:path';
+import crypto from 'node:crypto';
+
+function parseArgs(argv) {
+  const args = {
+    distDir: 'dist',
+    output: null,
+    pretty: true
+  };
+
+  for (let i = 0; i < argv.length; i += 1) {
+    const arg = argv[i];
+    if (arg === '--dist') {
+      args.distDir = argv[i + 1] ?? args.distDir;
+      i += 1;
+      continue;
+    }
+
+    if (arg === '--output') {
+      args.output = argv[i + 1] ?? args.output;
+      i += 1;
+      continue;
+    }
+
+    if (arg === '--compact') {
+      args.pretty = false;
+      continue;
+    }
+
+    if (arg === '--help' || arg === '-h') {
+      console.log('Usage: node scripts/hash-dist.mjs [--dist <dir>] [--output <file>] [--compact]');
+      process.exit(0);
+    }
+
+    throw new Error(`Unknown argument: ${arg}`);
+  }
+
+  return args;
+}
+
+function toPosixRelative(baseDir, filePath) {
+  return path.relative(baseDir, filePath).split(path.sep).join('/');
+}
+
+function walkFiles(rootDir) {
+  const files = [];
+  const stack = [rootDir];
+
+  while (stack.length > 0) {
+    const currentDir = stack.pop();
+    const entries = fs.readdirSync(currentDir, { withFileTypes: true })
+      .sort((a, b) => a.name.localeCompare(b.name));
+
+    for (const entry of entries) {
+      const fullPath = path.join(currentDir, entry.name);
+      if (entry.isDirectory()) {
+        stack.push(fullPath);
+        continue;
+      }
+      if (entry.isFile()) {
+        files.push(fullPath);
+      }
+    }
+  }
+
+  return files.sort((a, b) => toPosixRelative(rootDir, a).localeCompare(toPosixRelative(rootDir, b)));
+}
+
+function hashBuffer(buffer) {
+  return crypto.createHash('sha256').update(buffer).digest('hex');
+}
+
+function buildHashState(distDir) {
+  const resolvedDistDir = path.resolve(distDir);
+  if (!fs.existsSync(resolvedDistDir)) {
+    throw new Error(`Dist directory does not exist: ${resolvedDistDir}`);
+  }
+
+  const files = walkFiles(resolvedDistDir).map((filePath) => {
+    const contents = fs.readFileSync(filePath);
+    return {
+      path: toPosixRelative(resolvedDistDir, filePath),
+      sha256: hashBuffer(contents)
+    };
+  });
+
+  const combinedInput = files
+    .map((entry) => `${entry.path}:${entry.sha256}`)
+    .join('\n');
+
+  return {
+    combined: hashBuffer(Buffer.from(combinedInput, 'utf8')),
+    files
+  };
+}
+
+function main() {
+  const args = parseArgs(process.argv.slice(2));
+  const hashState = buildHashState(args.distDir);
+  const serialized = JSON.stringify(hashState, null, args.pretty ? 2 : 0);
+
+  if (args.output) {
+    const outputPath = path.resolve(args.output);
+    fs.mkdirSync(path.dirname(outputPath), { recursive: true });
+    fs.writeFileSync(outputPath, `${serialized}\n`);
+    console.log(`Wrote hash state to ${outputPath}`);
+    return;
+  }
+
+  console.log(serialized);
+}
+
+main();

scripts/plan-cache-purge.mjs

@@ -0,0 +1,104 @@
+#!/usr/bin/env node
+import fs from 'node:fs';
+
+const ALWAYS_PUBLISH_PATHS = new Set(['index.html', 'embed.html', 'sw.min.js', 'site.webmanifest']);
+const NON_ASSET_CHANGED_PATHS = new Set(['asset-manifest.json', '_headers', '_redirects']);
+
+function parseArgs(argv) {
+  const args = {
+    changedFile: null,
+    outputFile: null
+  };
+
+  for (let i = 0; i < argv.length; i += 1) {
+    const arg = argv[i];
+
+    if (arg === '--changed-file') {
+      args.changedFile = argv[i + 1] ?? null;
+      i += 1;
+      continue;
+    }
+
+    if (arg === '--output-file') {
+      args.outputFile = argv[i + 1] ?? null;
+      i += 1;
+      continue;
+    }
+
+    if (arg === '--help' || arg === '-h') {
+      console.log('Usage: node scripts/plan-cache-purge.mjs --changed-file <paths.txt> [--output-file <paths.txt>]');
+      process.exit(0);
+    }
+
+    throw new Error(`Unknown argument: ${arg}`);
+  }
+
+  if (!args.changedFile) {
+    throw new Error('--changed-file is required.');
+  }
+
+  return args;
+}
+
+function readLines(filePath) {
+  if (!fs.existsSync(filePath)) {
+    return [];
+  }
+  return fs.readFileSync(filePath, 'utf8').split(/\r?\n/).map((line) => line.trim()).filter(Boolean);
+}
+
+function isHashedFilename(filePath) {
+  return /\.[a-f0-9]{16}\.[^./]+$/i.test(filePath);
+}
+
+function isHtmlOrServiceWorkerPath(filePath) {
+  return ALWAYS_PUBLISH_PATHS.has(filePath);
+}
+
+function isNonAssetMetadataPath(filePath) {
+  return NON_ASSET_CHANGED_PATHS.has(filePath);
+}
+
+function isUnhashedAssetPath(filePath) {
+  if (isHtmlOrServiceWorkerPath(filePath) || isNonAssetMetadataPath(filePath)) {
+    return false;
+  }
+
+  if (isHashedFilename(filePath)) {
+    return false;
+  }
+
+  return /^(css|js|assets|vendor|components|views|content)\//.test(filePath);
+}
+
+function uniqueSorted(items) {
+  return [...new Set(items)].sort((a, b) => a.localeCompare(b));
+}
+
+function main() {
+  const args = parseArgs(process.argv.slice(2));
+  const changedPaths = uniqueSorted(readLines(args.changedFile));
+  const unhashedAssetPaths = changedPaths.filter(isUnhashedAssetPath);
+  const hashedModeFullyActive = unhashedAssetPaths.length === 0;
+
+  const purgePaths = hashedModeFullyActive
+    ? changedPaths.filter(isHtmlOrServiceWorkerPath)
+    : changedPaths;
+
+  const uniquePurgePaths = uniqueSorted(purgePaths);
+
+  if (args.outputFile) {
+    fs.writeFileSync(args.outputFile, uniquePurgePaths.join('\n') + (uniquePurgePaths.length > 0 ? '\n' : ''));
+  }
+
+  const summary = {
+    mode: hashedModeFullyActive ? 'hashed_full' : 'unhashed_detected',
+    changedCount: changedPaths.length,
+    purgeCount: uniquePurgePaths.length,
+    unhashedAssetPaths
+  };
+
+  console.log(JSON.stringify(summary, null, 2));
+}
+
+main();

scripts/purge-cloudflare-changed.mjs

@@ -0,0 +1,144 @@
+#!/usr/bin/env node
+import fs from 'node:fs';
+
+const MAX_URLS_PER_REQUEST = 30;
+
+function parseArgs(argv) {
+  const args = {
+    changedPathsFile: null,
+    changedPaths: [],
+    baseUrl: process.env.CLOUDFLARE_PURGE_BASE_URL ?? '',
+    allowPurgeAll: false,
+    dryRun: false
+  };
+
+  for (let i = 0; i < argv.length; i += 1) {
+    const arg = argv[i];
+
+    if (arg === '--changed-file') {
+      args.changedPathsFile = argv[i + 1] ?? args.changedPathsFile;
+      i += 1;
+      continue;
+    }
+
+    if (arg === '--changed') {
+      args.changedPaths.push(argv[i + 1] ?? '');
+      i += 1;
+      continue;
+    }
+
+    if (arg === '--base-url') {
+      args.baseUrl = argv[i + 1] ?? args.baseUrl;
+      i += 1;
+      continue;
+    }
+
+    if (arg === '--allow-purge-all') {
+      args.allowPurgeAll = true;
+      continue;
+    }
+
+    if (arg === '--dry-run') {
+      args.dryRun = true;
+      continue;
+    }
+
+    if (arg === '--help' || arg === '-h') {
+      console.log(
+        'Usage: node scripts/purge-cloudflare-changed.mjs --changed-file <paths.txt> [--base-url https://example.com] [--dry-run]'
+      );
+      process.exit(0);
+    }
+
+    throw new Error(`Unknown argument: ${arg}`);
+  }
+
+  return args;
+}
+
+function readChangedPaths(args) {
+  const output = [];
+
+  if (args.changedPathsFile) {
+    const fileContents = fs.readFileSync(args.changedPathsFile, 'utf8');
+    output.push(...fileContents.split(/\r?\n/));
+  }
+
+  output.push(...args.changedPaths);
+
+  return [...new Set(output.map((value) => value.trim()).filter(Boolean))].sort((a, b) => a.localeCompare(b));
+}
+
+function toAbsoluteUrl(baseUrl, relativePath) {
+  if (!baseUrl) {
+    return relativePath.startsWith('/') ? relativePath : `/${relativePath}`;
+  }
+
+  const normalizedBase = baseUrl.endsWith('/') ? baseUrl.slice(0, -1) : baseUrl;
+  const normalizedPath = relativePath.startsWith('/') ? relativePath : `/${relativePath}`;
+  return `${normalizedBase}${normalizedPath}`;
+}
+
+function chunk(array, size) {
+  const parts = [];
+  for (let i = 0; i < array.length; i += size) {
+    parts.push(array.slice(i, i + size));
+  }
+  return parts;
+}
+
+async function purgeBatch({ zoneId, apiToken, urls, dryRun }) {
+  if (dryRun) {
+    console.log(`[dry-run] Would purge ${urls.length} URLs`);
+    return;
+  }
+
+  const response = await fetch(`https://api.cloudflare.com/client/v4/zones/${zoneId}/purge_cache`, {
+    method: 'POST',
+    headers: {
+      Authorization: `Bearer ${apiToken}`,
+      'Content-Type': 'application/json'
+    },
+    body: JSON.stringify({ files: urls })
+  });
+
+  const payload = await response.json();
+  if (!response.ok || payload.success !== true) {
+    throw new Error(`Cloudflare purge failed: ${JSON.stringify(payload)}`);
+  }
+}
+
+async function main() {
+  const args = parseArgs(process.argv.slice(2));
+  const changedPaths = readChangedPaths(args);
+
+  if (changedPaths.length === 0) {
+    console.log('No changed paths supplied. Skipping Cloudflare purge.');
+    return;
+  }
+
+  if (changedPaths.includes('*') && !args.allowPurgeAll) {
+    throw new Error('Refusing to purge all cache by default. Remove wildcard path or pass --allow-purge-all explicitly.');
+  }
+
+  const urls = changedPaths.map((relativePath) => toAbsoluteUrl(args.baseUrl, relativePath));
+
+  const zoneId = process.env.CLOUDFLARE_ZONE_ID;
+  const apiToken = process.env.CLOUDFLARE_API_TOKEN;
+  if (!args.dryRun && (!zoneId || !apiToken)) {
+    throw new Error('Missing CLOUDFLARE_ZONE_ID or CLOUDFLARE_API_TOKEN environment variables.');
+  }
+
+  const batches = chunk(urls, MAX_URLS_PER_REQUEST);
+  for (const [index, batch] of batches.entries()) {
+    console.log(`Purging batch ${index + 1}/${batches.length} (${batch.length} URLs)`);
+    await purgeBatch({ zoneId, apiToken, urls: batch, dryRun: args.dryRun });
+  }
+
+  console.log(`Cloudflare targeted purge complete. URLs purged: ${urls.length}`);
+}
+
+main().catch((error) => {
+  console.error(error.message);
+  process.exit(1);
+});

scripts/store-dist-hash-state.mjs

@@ -0,0 +1,69 @@
+#!/usr/bin/env node
+import fs from 'node:fs';
+
+function parseArgs(argv) {
+  const args = {
+    input: null,
+    uploadUrl: process.env.DIST_HASH_STATE_UPLOAD_URL ?? '',
+    method: process.env.DIST_HASH_STATE_UPLOAD_METHOD ?? 'PUT'
+  };
+
+  for (let i = 0; i < argv.length; i += 1) {
+    const arg = argv[i];
+    if (arg === '--input') {
+      args.input = argv[i + 1] ?? null;
+      i += 1;
+      continue;
+    }
+    if (arg === '--upload-url') {
+      args.uploadUrl = argv[i + 1] ?? '';
+      i += 1;
+      continue;
+    }
+    if (arg === '--method') {
+      args.method = argv[i + 1] ?? args.method;
+      i += 1;
+      continue;
+    }
+    if (arg === '--help' || arg === '-h') {
+      console.log('Usage: node scripts/store-dist-hash-state.mjs --input <hash-state.json> [--upload-url <url>] [--method PUT]');
+      process.exit(0);
+    }
+    throw new Error(`Unknown argument: ${arg}`);
+  }
+
+  if (!args.input) {
+    throw new Error('--input is required.');
+  }
+  if (!args.uploadUrl) {
+    throw new Error('Missing upload URL. Provide --upload-url or DIST_HASH_STATE_UPLOAD_URL.');
+  }
+
+  return args;
+}
+
+async function main() {
+  const args = parseArgs(process.argv.slice(2));
+  const body = fs.readFileSync(args.input);
+
+  const response = await fetch(args.uploadUrl, {
+    method: args.method,
+    headers: {
+      'Content-Type': 'application/json',
+      'Cache-Control': 'no-store'
+    },
+    body
+  });
+
+  if (!response.ok) {
+    const errorText = await response.text();
+    throw new Error(`Failed to upload hash state (${response.status}): ${errorText}`);
+  }
+
+  console.log(`Uploaded dist hash state to ${args.uploadUrl}`);
+}
+
+main().catch((error) => {
+  console.error(error.message);
+  process.exit(1);
+});