From 17f7282a868a5592e7d150eb3220b2e4220f4e7a Mon Sep 17 00:00:00 2001 From: thePR0M3TH3AN <53631862+PR0M3TH3AN@users.noreply.github.com> Date: Tue, 10 Feb 2026 01:00:42 -0500 Subject: [PATCH] [ci] add dist hash diffing and targeted cache purge flow --- .github/workflows/ci.yml | 71 +++++++++++++ package.json | 5 +- scripts/diff-dist-hashes.mjs | 91 +++++++++++++++++ scripts/hash-dist.mjs | 115 +++++++++++++++++++++ scripts/plan-cache-purge.mjs | 104 +++++++++++++++++++ scripts/purge-cloudflare-changed.mjs | 144 +++++++++++++++++++++++++++ scripts/store-dist-hash-state.mjs | 69 +++++++++++++ 7 files changed, 598 insertions(+), 1 deletion(-) create mode 100644 scripts/diff-dist-hashes.mjs create mode 100644 scripts/hash-dist.mjs create mode 100644 scripts/plan-cache-purge.mjs create mode 100644 scripts/purge-cloudflare-changed.mjs create mode 100644 scripts/store-dist-hash-state.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b6ce5e3f..f710750c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,6 +57,77 @@ jobs: - name: Build assets run: npm run build + - name: Generate dist hash state + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + run: | + mkdir -p artifacts + node scripts/hash-dist.mjs --dist dist --output artifacts/dist-hashes-current.json + + - name: Fetch prior dist hash state + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + env: + DIST_HASH_STATE_URL: ${{ secrets.DIST_HASH_STATE_URL }} + run: | + if [ -n "$DIST_HASH_STATE_URL" ]; then + curl --fail --silent --show-error "$DIST_HASH_STATE_URL" -o artifacts/dist-hashes-previous.json + echo "Loaded prior dist hash state from authoritative store." + else + echo '{"combined":"","files":[]}' > artifacts/dist-hashes-previous.json + echo "No DIST_HASH_STATE_URL configured; using empty baseline." + fi + + - name: Diff dist hash states + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + run: | + node scripts/diff-dist-hashes.mjs \ + --current artifacts/dist-hashes-current.json \ + --previous artifacts/dist-hashes-previous.json > artifacts/dist-hashes-changed.txt + echo "Changed dist paths:" + cat artifacts/dist-hashes-changed.txt || true + + - name: Plan cache purge scope + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + run: | + node scripts/plan-cache-purge.mjs \ + --changed-file artifacts/dist-hashes-changed.txt \ + --output-file artifacts/dist-purge-targets.txt > artifacts/dist-purge-plan.json + cat artifacts/dist-purge-plan.json + + - name: Purge changed Cloudflare URLs + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + env: + CLOUDFLARE_ZONE_ID: ${{ secrets.CLOUDFLARE_ZONE_ID }} + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + CLOUDFLARE_PURGE_BASE_URL: ${{ secrets.CLOUDFLARE_PURGE_BASE_URL }} + run: | + if [ ! -s artifacts/dist-purge-targets.txt ]; then + echo "No purge targets detected. Skipping purge." + exit 0 + fi + + if [ -z "$CLOUDFLARE_ZONE_ID" ] || [ -z "$CLOUDFLARE_API_TOKEN" ] || [ -z "$CLOUDFLARE_PURGE_BASE_URL" ]; then + echo "Cloudflare secrets not configured; skipping targeted purge." + exit 0 + fi + + node scripts/purge-cloudflare-changed.mjs \ + --changed-file artifacts/dist-purge-targets.txt \ + --base-url "$CLOUDFLARE_PURGE_BASE_URL" + + - name: Record authoritative dist hash state (post-deploy) + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + env: + DIST_HASH_STATE_UPLOAD_URL: ${{ secrets.DIST_HASH_STATE_UPLOAD_URL }} + run: | + if [ -z "$DIST_HASH_STATE_UPLOAD_URL" ]; then + echo "No DIST_HASH_STATE_UPLOAD_URL configured; skipping authoritative hash-state update." + exit 0 + fi + + node scripts/store-dist-hash-state.mjs \ + --input artifacts/dist-hashes-current.json \ + --upload-url "$DIST_HASH_STATE_UPLOAD_URL" + - name: Install Playwright browsers run: npx playwright install --with-deps chromium diff --git a/package.json b/package.json index 575efe57..0d18dcc3 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,10 @@ "test:unit:shard3": "cross-env UNIT_TEST_SHARD=3/3 node scripts/run-unit-tests.mjs", "telemetry:aggregate": "node scripts/agent/telemetry-aggregator.mjs", "lint:assets": "node scripts/check-asset-references.mjs", - "lint:sw-compat": "node scripts/check-sw-compat.mjs" + "lint:sw-compat": "node scripts/check-sw-compat.mjs", + "hash:dist": "node scripts/hash-dist.mjs --dist dist --output artifacts/dist-hashes-current.json", + "diff:dist-hashes": "node scripts/diff-dist-hashes.mjs", + "purge:cloudflare:changed": "node scripts/purge-cloudflare-changed.mjs" }, "devDependencies": { "@playwright/test": "^1.58.2", diff --git a/scripts/diff-dist-hashes.mjs b/scripts/diff-dist-hashes.mjs new file mode 100644 index 00000000..3b2a786a --- /dev/null +++ b/scripts/diff-dist-hashes.mjs @@ -0,0 +1,91 @@ +#!/usr/bin/env node +import fs from 'node:fs'; +import path from 'node:path'; + +function parseArgs(argv) { + const args = { + current: null, + previous: null, + outputJson: false + }; + + for (let i = 0; i < argv.length; i += 1) { + const arg = argv[i]; + + if (arg === '--current') { + args.current = argv[i + 1] ?? args.current; + i += 1; + continue; + } + + if (arg === '--previous') { + args.previous = argv[i + 1] ?? args.previous; + i += 1; + continue; + } + + if (arg === '--json') { + args.outputJson = true; + continue; + } + + if (arg === '--help' || arg === '-h') { + console.log('Usage: node scripts/diff-dist-hashes.mjs --current --previous [--json]'); + process.exit(0); + } + + throw new Error(`Unknown argument: ${arg}`); + } + + if (!args.current || !args.previous) { + throw new Error('Both --current and --previous are required.'); + } + + return args; +} + +function readHashState(filePath) { + const resolved = path.resolve(filePath); + if (!fs.existsSync(resolved)) { + throw new Error(`Hash state file not found: ${resolved}`); + } + + const payload = JSON.parse(fs.readFileSync(resolved, 'utf8')); + if (!Array.isArray(payload.files)) { + throw new Error(`Invalid hash state format in ${resolved}: missing files[] array.`); + } + + return payload; +} + +function toMap(state) { + return new Map(state.files.map((entry) => [entry.path, entry.sha256])); +} + +function getChangedPaths(currentState, previousState) { + const current = toMap(currentState); + const previous = toMap(previousState); + const allPaths = new Set([...current.keys(), ...previous.keys()]); + + return [...allPaths] + .filter((filePath) => current.get(filePath) !== previous.get(filePath)) + .sort((a, b) => a.localeCompare(b)); +} + +function main() { + const args = parseArgs(process.argv.slice(2)); + const currentState = readHashState(args.current); + const previousState = readHashState(args.previous); + const changedPaths = getChangedPaths(currentState, previousState); + + if (args.outputJson) { + console.log(JSON.stringify({ changed: changedPaths }, null, 2)); + return; + } + + for (const changedPath of changedPaths) { + console.log(changedPath); + } +} + +main(); diff --git a/scripts/hash-dist.mjs b/scripts/hash-dist.mjs new file mode 100644 index 00000000..8637c776 --- /dev/null +++ b/scripts/hash-dist.mjs @@ -0,0 +1,115 @@ +#!/usr/bin/env node +import fs from 'node:fs'; +import path from 'node:path'; +import crypto from 'node:crypto'; + +function parseArgs(argv) { + const args = { + distDir: 'dist', + output: null, + pretty: true + }; + + for (let i = 0; i < argv.length; i += 1) { + const arg = argv[i]; + if (arg === '--dist') { + args.distDir = argv[i + 1] ?? args.distDir; + i += 1; + continue; + } + + if (arg === '--output') { + args.output = argv[i + 1] ?? args.output; + i += 1; + continue; + } + + if (arg === '--compact') { + args.pretty = false; + continue; + } + + if (arg === '--help' || arg === '-h') { + console.log('Usage: node scripts/hash-dist.mjs [--dist ] [--output ] [--compact]'); + process.exit(0); + } + + throw new Error(`Unknown argument: ${arg}`); + } + + return args; +} + +function toPosixRelative(baseDir, filePath) { + return path.relative(baseDir, filePath).split(path.sep).join('/'); +} + +function walkFiles(rootDir) { + const files = []; + const stack = [rootDir]; + + while (stack.length > 0) { + const currentDir = stack.pop(); + const entries = fs.readdirSync(currentDir, { withFileTypes: true }) + .sort((a, b) => a.name.localeCompare(b.name)); + + for (const entry of entries) { + const fullPath = path.join(currentDir, entry.name); + if (entry.isDirectory()) { + stack.push(fullPath); + continue; + } + if (entry.isFile()) { + files.push(fullPath); + } + } + } + + return files.sort((a, b) => toPosixRelative(rootDir, a).localeCompare(toPosixRelative(rootDir, b))); +} + +function hashBuffer(buffer) { + return crypto.createHash('sha256').update(buffer).digest('hex'); +} + +function buildHashState(distDir) { + const resolvedDistDir = path.resolve(distDir); + if (!fs.existsSync(resolvedDistDir)) { + throw new Error(`Dist directory does not exist: ${resolvedDistDir}`); + } + + const files = walkFiles(resolvedDistDir).map((filePath) => { + const contents = fs.readFileSync(filePath); + return { + path: toPosixRelative(resolvedDistDir, filePath), + sha256: hashBuffer(contents) + }; + }); + + const combinedInput = files + .map((entry) => `${entry.path}:${entry.sha256}`) + .join('\n'); + + return { + combined: hashBuffer(Buffer.from(combinedInput, 'utf8')), + files + }; +} + +function main() { + const args = parseArgs(process.argv.slice(2)); + const hashState = buildHashState(args.distDir); + const serialized = JSON.stringify(hashState, null, args.pretty ? 2 : 0); + + if (args.output) { + const outputPath = path.resolve(args.output); + fs.mkdirSync(path.dirname(outputPath), { recursive: true }); + fs.writeFileSync(outputPath, `${serialized}\n`); + console.log(`Wrote hash state to ${outputPath}`); + return; + } + + console.log(serialized); +} + +main(); diff --git a/scripts/plan-cache-purge.mjs b/scripts/plan-cache-purge.mjs new file mode 100644 index 00000000..341e34f7 --- /dev/null +++ b/scripts/plan-cache-purge.mjs @@ -0,0 +1,104 @@ +#!/usr/bin/env node +import fs from 'node:fs'; + +const ALWAYS_PUBLISH_PATHS = new Set(['index.html', 'embed.html', 'sw.min.js', 'site.webmanifest']); +const NON_ASSET_CHANGED_PATHS = new Set(['asset-manifest.json', '_headers', '_redirects']); + +function parseArgs(argv) { + const args = { + changedFile: null, + outputFile: null + }; + + for (let i = 0; i < argv.length; i += 1) { + const arg = argv[i]; + + if (arg === '--changed-file') { + args.changedFile = argv[i + 1] ?? null; + i += 1; + continue; + } + + if (arg === '--output-file') { + args.outputFile = argv[i + 1] ?? null; + i += 1; + continue; + } + + if (arg === '--help' || arg === '-h') { + console.log('Usage: node scripts/plan-cache-purge.mjs --changed-file [--output-file ]'); + process.exit(0); + } + + throw new Error(`Unknown argument: ${arg}`); + } + + if (!args.changedFile) { + throw new Error('--changed-file is required.'); + } + + return args; +} + +function readLines(filePath) { + if (!fs.existsSync(filePath)) { + return []; + } + return fs.readFileSync(filePath, 'utf8').split(/\r?\n/).map((line) => line.trim()).filter(Boolean); +} + +function isHashedFilename(filePath) { + return /\.[a-f0-9]{16}\.[^./]+$/i.test(filePath); +} + +function isHtmlOrServiceWorkerPath(filePath) { + return ALWAYS_PUBLISH_PATHS.has(filePath); +} + +function isNonAssetMetadataPath(filePath) { + return NON_ASSET_CHANGED_PATHS.has(filePath); +} + +function isUnhashedAssetPath(filePath) { + if (isHtmlOrServiceWorkerPath(filePath) || isNonAssetMetadataPath(filePath)) { + return false; + } + + if (isHashedFilename(filePath)) { + return false; + } + + return /^(css|js|assets|vendor|components|views|content)\//.test(filePath); +} + +function uniqueSorted(items) { + return [...new Set(items)].sort((a, b) => a.localeCompare(b)); +} + +function main() { + const args = parseArgs(process.argv.slice(2)); + const changedPaths = uniqueSorted(readLines(args.changedFile)); + const unhashedAssetPaths = changedPaths.filter(isUnhashedAssetPath); + const hashedModeFullyActive = unhashedAssetPaths.length === 0; + + const purgePaths = hashedModeFullyActive + ? changedPaths.filter(isHtmlOrServiceWorkerPath) + : changedPaths; + + const uniquePurgePaths = uniqueSorted(purgePaths); + + if (args.outputFile) { + fs.writeFileSync(args.outputFile, uniquePurgePaths.join('\n') + (uniquePurgePaths.length > 0 ? '\n' : '')); + } + + const summary = { + mode: hashedModeFullyActive ? 'hashed_full' : 'unhashed_detected', + changedCount: changedPaths.length, + purgeCount: uniquePurgePaths.length, + unhashedAssetPaths + }; + + console.log(JSON.stringify(summary, null, 2)); +} + +main(); diff --git a/scripts/purge-cloudflare-changed.mjs b/scripts/purge-cloudflare-changed.mjs new file mode 100644 index 00000000..0884bc3a --- /dev/null +++ b/scripts/purge-cloudflare-changed.mjs @@ -0,0 +1,144 @@ +#!/usr/bin/env node +import fs from 'node:fs'; + +const MAX_URLS_PER_REQUEST = 30; + +function parseArgs(argv) { + const args = { + changedPathsFile: null, + changedPaths: [], + baseUrl: process.env.CLOUDFLARE_PURGE_BASE_URL ?? '', + allowPurgeAll: false, + dryRun: false + }; + + for (let i = 0; i < argv.length; i += 1) { + const arg = argv[i]; + + if (arg === '--changed-file') { + args.changedPathsFile = argv[i + 1] ?? args.changedPathsFile; + i += 1; + continue; + } + + if (arg === '--changed') { + args.changedPaths.push(argv[i + 1] ?? ''); + i += 1; + continue; + } + + if (arg === '--base-url') { + args.baseUrl = argv[i + 1] ?? args.baseUrl; + i += 1; + continue; + } + + if (arg === '--allow-purge-all') { + args.allowPurgeAll = true; + continue; + } + + if (arg === '--dry-run') { + args.dryRun = true; + continue; + } + + if (arg === '--help' || arg === '-h') { + console.log( + 'Usage: node scripts/purge-cloudflare-changed.mjs --changed-file [--base-url https://example.com] [--dry-run]' + ); + process.exit(0); + } + + throw new Error(`Unknown argument: ${arg}`); + } + + return args; +} + +function readChangedPaths(args) { + const output = []; + + if (args.changedPathsFile) { + const fileContents = fs.readFileSync(args.changedPathsFile, 'utf8'); + output.push(...fileContents.split(/\r?\n/)); + } + + output.push(...args.changedPaths); + + return [...new Set(output.map((value) => value.trim()).filter(Boolean))].sort((a, b) => a.localeCompare(b)); +} + +function toAbsoluteUrl(baseUrl, relativePath) { + if (!baseUrl) { + return relativePath.startsWith('/') ? relativePath : `/${relativePath}`; + } + + const normalizedBase = baseUrl.endsWith('/') ? baseUrl.slice(0, -1) : baseUrl; + const normalizedPath = relativePath.startsWith('/') ? relativePath : `/${relativePath}`; + return `${normalizedBase}${normalizedPath}`; +} + +function chunk(array, size) { + const parts = []; + for (let i = 0; i < array.length; i += size) { + parts.push(array.slice(i, i + size)); + } + return parts; +} + +async function purgeBatch({ zoneId, apiToken, urls, dryRun }) { + if (dryRun) { + console.log(`[dry-run] Would purge ${urls.length} URLs`); + return; + } + + const response = await fetch(`https://api.cloudflare.com/client/v4/zones/${zoneId}/purge_cache`, { + method: 'POST', + headers: { + Authorization: `Bearer ${apiToken}`, + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ files: urls }) + }); + + const payload = await response.json(); + if (!response.ok || payload.success !== true) { + throw new Error(`Cloudflare purge failed: ${JSON.stringify(payload)}`); + } +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + const changedPaths = readChangedPaths(args); + + if (changedPaths.length === 0) { + console.log('No changed paths supplied. Skipping Cloudflare purge.'); + return; + } + + if (changedPaths.includes('*') && !args.allowPurgeAll) { + throw new Error('Refusing to purge all cache by default. Remove wildcard path or pass --allow-purge-all explicitly.'); + } + + const urls = changedPaths.map((relativePath) => toAbsoluteUrl(args.baseUrl, relativePath)); + + const zoneId = process.env.CLOUDFLARE_ZONE_ID; + const apiToken = process.env.CLOUDFLARE_API_TOKEN; + if (!args.dryRun && (!zoneId || !apiToken)) { + throw new Error('Missing CLOUDFLARE_ZONE_ID or CLOUDFLARE_API_TOKEN environment variables.'); + } + + const batches = chunk(urls, MAX_URLS_PER_REQUEST); + for (const [index, batch] of batches.entries()) { + console.log(`Purging batch ${index + 1}/${batches.length} (${batch.length} URLs)`); + await purgeBatch({ zoneId, apiToken, urls: batch, dryRun: args.dryRun }); + } + + console.log(`Cloudflare targeted purge complete. URLs purged: ${urls.length}`); +} + +main().catch((error) => { + console.error(error.message); + process.exit(1); +}); diff --git a/scripts/store-dist-hash-state.mjs b/scripts/store-dist-hash-state.mjs new file mode 100644 index 00000000..eff29d82 --- /dev/null +++ b/scripts/store-dist-hash-state.mjs @@ -0,0 +1,69 @@ +#!/usr/bin/env node +import fs from 'node:fs'; + +function parseArgs(argv) { + const args = { + input: null, + uploadUrl: process.env.DIST_HASH_STATE_UPLOAD_URL ?? '', + method: process.env.DIST_HASH_STATE_UPLOAD_METHOD ?? 'PUT' + }; + + for (let i = 0; i < argv.length; i += 1) { + const arg = argv[i]; + if (arg === '--input') { + args.input = argv[i + 1] ?? null; + i += 1; + continue; + } + if (arg === '--upload-url') { + args.uploadUrl = argv[i + 1] ?? ''; + i += 1; + continue; + } + if (arg === '--method') { + args.method = argv[i + 1] ?? args.method; + i += 1; + continue; + } + if (arg === '--help' || arg === '-h') { + console.log('Usage: node scripts/store-dist-hash-state.mjs --input [--upload-url ] [--method PUT]'); + process.exit(0); + } + throw new Error(`Unknown argument: ${arg}`); + } + + if (!args.input) { + throw new Error('--input is required.'); + } + if (!args.uploadUrl) { + throw new Error('Missing upload URL. Provide --upload-url or DIST_HASH_STATE_UPLOAD_URL.'); + } + + return args; +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + const body = fs.readFileSync(args.input); + + const response = await fetch(args.uploadUrl, { + method: args.method, + headers: { + 'Content-Type': 'application/json', + 'Cache-Control': 'no-store' + }, + body + }); + + if (!response.ok) { + const errorText = await response.text(); + throw new Error(`Failed to upload hash state (${response.status}): ${errorText}`); + } + + console.log(`Uploaded dist hash state to ${args.uploadUrl}`); +} + +main().catch((error) => { + console.error(error.message); + process.exit(1); +});