From 9da4e40ec9381423f00f639719e17389c1a47e6a Mon Sep 17 00:00:00 2001 From: thePR0M3TH3AN <53631862+PR0M3TH3AN@users.noreply.github.com> Date: Fri, 26 Sep 2025 21:51:22 -0400 Subject: [PATCH] Improve R2 auto-setup and multipart upload reliability --- js/app.js | 68 +++++++++++++++---- js/storage/r2-mgmt.js | 147 ++++++++++++++++++++++++++++++++++++------ js/storage/r2-s3.js | 118 +++++++++++++++++---------------- 3 files changed, 246 insertions(+), 87 deletions(-) diff --git a/js/app.js b/js/app.js index dc644229..cb222031 100644 --- a/js/app.js +++ b/js/app.js @@ -29,8 +29,8 @@ import { sanitizeBucketName, ensureBucket, putCors, - attachCustomDomain, - enableManagedDomain, + attachCustomDomainAndWait, + setManagedDomain, } from "./storage/r2-mgmt.js"; import { makeR2Client, multipartUpload } from "./storage/r2-s3.js"; @@ -1592,7 +1592,11 @@ class bitvidApp { console.warn("Failed to refresh bucket configuration:", err); } } - return { entry, usedManagedFallback: entry.domainType !== "custom" }; + return { + entry, + usedManagedFallback: entry.domainType !== "custom", + customDomainStatus: entry.domainType === "custom" ? "active" : "skipped", + }; } if (!apiToken) { @@ -1619,37 +1623,71 @@ class bitvidApp { let publicBaseUrl = entry?.publicBaseUrl || ""; let domainType = entry?.domainType || "managed"; let usedManagedFallback = false; + let customDomainStatus = "skipped"; if (baseDomain && zoneId) { const domain = `${this.deriveSubdomainForNpub(npub)}.${baseDomain}`; try { - publicBaseUrl = await attachCustomDomain({ + const custom = await attachCustomDomainAndWait({ accountId, bucket: bucketName, token: apiToken, zoneId, domain, + pollInterval: 2500, + timeoutMs: 120000, }); - domainType = "custom"; + customDomainStatus = custom?.status || "unknown"; + if (custom?.active && custom?.url) { + publicBaseUrl = custom.url; + domainType = "custom"; + try { + await setManagedDomain({ + accountId, + bucket: bucketName, + token: apiToken, + enabled: false, + }); + } catch (disableErr) { + console.warn("Failed to disable managed domain:", disableErr); + } + } else { + usedManagedFallback = true; + } } catch (err) { if (/already exists/i.test(err.message || "")) { publicBaseUrl = `https://${domain}`; domainType = "custom"; + customDomainStatus = "active"; + try { + await setManagedDomain({ + accountId, + bucket: bucketName, + token: apiToken, + enabled: false, + }); + } catch (disableErr) { + console.warn("Failed to disable managed domain:", disableErr); + } } else { console.warn("Failed to attach custom domain, falling back:", err); usedManagedFallback = true; + customDomainStatus = "error"; } } } if (!publicBaseUrl) { - publicBaseUrl = await enableManagedDomain({ + const managed = await setManagedDomain({ accountId, bucket: bucketName, token: apiToken, + enabled: true, }); + publicBaseUrl = managed?.url || `https://${bucketName}.${accountId}.r2.dev`; domainType = "managed"; usedManagedFallback = true; + customDomainStatus = customDomainStatus === "skipped" ? "managed" : customDomainStatus; } const mergedEntry = { @@ -1662,7 +1700,7 @@ class bitvidApp { mergeBucketEntry(this.cloudflareSettings, npub, mergedEntry) ); this.cloudflareSettings = updatedSettings; - return { entry: mergedEntry, usedManagedFallback }; + return { entry: mergedEntry, usedManagedFallback, customDomainStatus }; } async updateCloudflareBucketPreview() { @@ -1785,10 +1823,18 @@ class bitvidApp { return; } - const statusMessage = bucketResult?.usedManagedFallback && - (this.cloudflareSettings?.baseDomain || "") - ? `Using managed r2.dev domain for ${bucketEntry.bucket}. Verify your Cloudflare zone. Uploading…` - : `Uploading to ${bucketEntry.bucket}…`; + let statusMessage = `Uploading to ${bucketEntry.bucket}…`; + if (bucketResult?.usedManagedFallback) { + const baseDomain = this.cloudflareSettings?.baseDomain || ""; + if (baseDomain) { + const customStatus = bucketResult?.customDomainStatus + ? ` (custom domain status: ${bucketResult.customDomainStatus})` + : ""; + statusMessage = `Using managed r2.dev domain for ${bucketEntry.bucket}. Verify your Cloudflare zone${customStatus}. Uploading…`; + } else { + statusMessage = `Using managed r2.dev domain for ${bucketEntry.bucket}. Uploading…`; + } + } this.setCloudflareUploadStatus( statusMessage, diff --git a/js/storage/r2-mgmt.js b/js/storage/r2-mgmt.js index 58c8e58a..20434737 100644 --- a/js/storage/r2-mgmt.js +++ b/js/storage/r2-mgmt.js @@ -1,12 +1,32 @@ // js/storage/r2-mgmt.js export function sanitizeBucketName(npub) { - const base = (npub || "user") + let value = String(npub || "") .toLowerCase() - .replace(/[^a-z0-9-]/g, "") + .replace(/[^a-z0-9-]/g, "-") + .replace(/-+/g, "-") .replace(/^-+|[-]+$/g, ""); - const suffix = Date.now().toString(36); - const name = `bv-${base || "u"}-${suffix}`.slice(0, 63); - return name.length < 3 ? `bv-u-${suffix}` : name; + + if (!value) { + value = "bitvid-user"; + } + + if (!/^[a-z0-9]/.test(value)) { + value = `bv-${value}`; + } + + if (!/[a-z0-9]$/.test(value)) { + value = `${value}0`; + } + + if (value.length > 63) { + value = value.slice(0, 63).replace(/-+$/g, ""); + } + + while (value.length < 3) { + value += "0"; + } + + return value; } function buildError(error, fallbackMessage) { @@ -88,11 +108,14 @@ export async function putCors({ accountId, bucket, token, origins }) { } const rules = [ { - AllowedOrigins: filteredOrigins, - AllowedMethods: ["GET", "HEAD", "PUT", "POST"], - AllowedHeaders: ["*"], - ExposeHeaders: ["ETag", "Content-Length", "Content-Range"], - MaxAgeSeconds: 3600, + id: "bitvid-default", + allowed: { + origins: filteredOrigins, + methods: ["GET", "HEAD", "PUT", "POST", "DELETE", "OPTIONS"], + headers: ["*"], + }, + expose_headers: ["ETag", "Content-Length", "Content-Range", "Accept-Ranges"], + max_age_seconds: 3600, }, ]; await cfFetch(`/accounts/${accountId}/r2/buckets/${bucket}/cors`, { @@ -102,30 +125,112 @@ export async function putCors({ accountId, bucket, token, origins }) { }); } +function extractStatus(payload) { + return ( + payload?.result?.status || + payload?.result?.result?.status || + payload?.status || + payload?.result?.statusText || + null + ); +} + +async function pollCustomDomain({ + accountId, + bucket, + token, + domain, + pollInterval = 2500, + timeoutMs = 120000, +}) { + const path = `/accounts/${accountId}/r2/buckets/${bucket}/domains/custom/${encodeURIComponent( + domain + )}`; + const deadline = Date.now() + timeoutMs; + let lastPayload = null; + + while (Date.now() < deadline) { + lastPayload = await cfFetch(path, { token, method: "GET" }).catch((err) => { + if (err?.status === 404) { + return null; + } + throw err; + }); + + if (!lastPayload) { + break; + } + + const status = extractStatus(lastPayload); + if (status === "active") { + return { status, payload: lastPayload }; + } + + await new Promise((resolve) => setTimeout(resolve, pollInterval)); + } + + return { + status: extractStatus(lastPayload) || "unknown", + payload: lastPayload, + }; +} + // 3) Attach custom domain to bucket (auto-provisions on Cloudflare) -export async function attachCustomDomain({ +export async function attachCustomDomainAndWait({ accountId, bucket, token, zoneId, domain, + pollInterval, + timeoutMs, }) { - const { result } = await cfFetch( - `/accounts/${accountId}/r2/buckets/${bucket}/domains/custom`, - { + let shouldPoll = true; + try { + await cfFetch(`/accounts/${accountId}/r2/buckets/${bucket}/domains/custom`, { token, method: "POST", - body: { domain, zoneId, enabled: true, minTLS: "1.2" }, + body: { domain, zoneId, enabled: true }, + }); + } catch (error) { + const err = buildError(error); + if (err.status === 409 || /already exists/i.test(err.message || "")) { + shouldPoll = true; + } else { + throw err; } - ); - return `https://${result?.domain || domain}`; + } + + let status = "unknown"; + if (shouldPoll) { + const pollResult = await pollCustomDomain({ + accountId, + bucket, + token, + domain, + pollInterval, + timeoutMs, + }); + status = pollResult.status; + } + + const url = `https://${domain}`; + return { + url, + status, + active: status === "active", + }; } -// 4) Or enable r2.dev managed domain as fallback -export async function enableManagedDomain({ accountId, bucket, token }) { +// 4) Toggle managed r2.dev domain for the bucket +export async function setManagedDomain({ accountId, bucket, token, enabled }) { const { result } = await cfFetch( `/accounts/${accountId}/r2/buckets/${bucket}/domains/managed`, - { token, method: "PUT", body: { enabled: true } } + { token, method: "PUT", body: { enabled: Boolean(enabled) } } ); - return `https://${result?.domain}`; + const domain = result?.domain || result?.result?.domain || ""; + return { + enabled: Boolean(result?.enabled ?? enabled), + url: domain ? `https://${domain}` : "", + }; } diff --git a/js/storage/r2-s3.js b/js/storage/r2-s3.js index 7912541b..e71f948b 100644 --- a/js/storage/r2-s3.js +++ b/js/storage/r2-s3.js @@ -5,7 +5,28 @@ import { UploadPartCommand, CompleteMultipartUploadCommand, AbortMultipartUploadCommand, -} from "https://esm.sh/@aws-sdk/client-s3@3.614.0?target=es2022&bundle"; +} from "https://esm.sh/@aws-sdk/client-s3@3.637.0?target=es2022&bundle"; + +function computeCacheControl(key) { + if (!key) { + return "public, max-age=3600"; + } + + const lower = String(key).toLowerCase(); + if (/(?:\.)(m3u8|mpd)$/.test(lower)) { + return "public, max-age=60, must-revalidate"; + } + + if ( + /\.(m4s|ts|mp4|webm|mov|mkv|png|jpg|jpeg|gif|svg|vtt|srt|mpg|mpeg)$/.test( + lower + ) + ) { + return "public, max-age=31536000, immutable"; + } + + return "public, max-age=3600"; +} export function makeR2Client({ accountId, accessKeyId, secretAccessKey }) { if (!accountId || !accessKeyId || !secretAccessKey) { @@ -27,7 +48,6 @@ export async function multipartUpload({ file, contentType, onProgress, - concurrency = 4, }) { if (!s3) { throw new Error("S3 client is required"); @@ -42,79 +62,67 @@ export async function multipartUpload({ throw new Error("File is required"); } - const { UploadId } = await s3.send( - new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: key, - ContentType: contentType || file.type || "video/mp4", - CacheControl: "public, max-age=31536000, immutable", - }) + const createCommand = new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + ContentType: contentType || file.type || "application/octet-stream", + CacheControl: computeCacheControl(key), + }); + + createCommand.middlewareStack.add( + (next) => async (args) => { + if (args?.request?.headers) { + args.request.headers["cf-create-bucket-if-missing"] = "true"; + } + return next(args); + }, + { step: "build" } ); + const { UploadId } = await s3.send(createCommand); + if (!UploadId) { throw new Error("Failed to start multipart upload"); } - const PART = 8 * 1024 * 1024; + const MIN_PART = 5 * 1024 * 1024; + const maxParts = 10000; + const calculated = Math.ceil(file.size / maxParts); + const partSize = Math.max(MIN_PART, calculated); const total = file.size; const parts = []; - let sent = 0; + let uploadedBytes = 0; let partNumber = 1; - const totalParts = Math.ceil(total / PART); - const errors = []; - const uploadPart = async () => { - const start = sent; - if (start >= total) { - return null; - } - const end = Math.min(start + PART, total); - const body = file.slice(start, end); - const currentPart = partNumber++; - sent = end; + if (typeof onProgress === "function") { + onProgress(0); + } - return s3 - .send( + try { + while (uploadedBytes < total) { + const start = uploadedBytes; + const end = Math.min(start + partSize, total); + const chunk = file.slice(start, end); + + const { ETag } = await s3.send( new UploadPartCommand({ Bucket: bucket, Key: key, UploadId, - PartNumber: currentPart, - Body: body, + PartNumber: partNumber, + Body: chunk, }) - ) - .then(({ ETag }) => { - parts.push({ ETag, PartNumber: currentPart }); - if (typeof onProgress === "function") { - onProgress(end / total); - } - }); - }; + ); - const workers = Array.from({ length: Math.max(1, concurrency) }, () => - (async () => { - try { - while (parts.length < totalParts) { - const task = uploadPart(); - if (!task) { - break; - } - await task; - } - } catch (error) { - errors.push(error); + parts.push({ ETag, PartNumber: partNumber }); + uploadedBytes = end; + partNumber += 1; + + if (typeof onProgress === "function") { + onProgress(uploadedBytes / total); } - })() - ); - - try { - await Promise.all(workers); - if (errors.length > 0) { - throw errors[0]; } - parts.sort((a, b) => a.PartNumber - b.PartNumber); - await s3.send( new CompleteMultipartUploadCommand({ Bucket: bucket,