Merge pull request #116 from PR0M3TH3AN/codex/add-r2-multipart-upload-support

Improve R2 auto-setup and multipart upload reliability

js/app.js

@@ -29,8 +29,8 @@ import {
   sanitizeBucketName,
   ensureBucket,
   putCors,
-  attachCustomDomain,
-  enableManagedDomain,
+  attachCustomDomainAndWait,
+  setManagedDomain,
 } from "./storage/r2-mgmt.js";
 import { makeR2Client, multipartUpload } from "./storage/r2-s3.js";
 
@@ -1592,7 +1592,11 @@ class bitvidApp {
           console.warn("Failed to refresh bucket configuration:", err);
         }
       }
-      return { entry, usedManagedFallback: entry.domainType !== "custom" };
+      return {
+        entry,
+        usedManagedFallback: entry.domainType !== "custom",
+        customDomainStatus: entry.domainType === "custom" ? "active" : "skipped",
+      };
     }
 
     if (!apiToken) {
@@ -1619,37 +1623,71 @@ class bitvidApp {
     let publicBaseUrl = entry?.publicBaseUrl || "";
     let domainType = entry?.domainType || "managed";
     let usedManagedFallback = false;
+    let customDomainStatus = "skipped";
 
     if (baseDomain && zoneId) {
       const domain = `${this.deriveSubdomainForNpub(npub)}.${baseDomain}`;
       try {
-        publicBaseUrl = await attachCustomDomain({
+        const custom = await attachCustomDomainAndWait({
           accountId,
           bucket: bucketName,
           token: apiToken,
           zoneId,
           domain,
+          pollInterval: 2500,
+          timeoutMs: 120000,
         });
-        domainType = "custom";
+        customDomainStatus = custom?.status || "unknown";
+        if (custom?.active && custom?.url) {
+          publicBaseUrl = custom.url;
+          domainType = "custom";
+          try {
+            await setManagedDomain({
+              accountId,
+              bucket: bucketName,
+              token: apiToken,
+              enabled: false,
+            });
+          } catch (disableErr) {
+            console.warn("Failed to disable managed domain:", disableErr);
+          }
+        } else {
+          usedManagedFallback = true;
+        }
       } catch (err) {
         if (/already exists/i.test(err.message || "")) {
           publicBaseUrl = `https://${domain}`;
           domainType = "custom";
+          customDomainStatus = "active";
+          try {
+            await setManagedDomain({
+              accountId,
+              bucket: bucketName,
+              token: apiToken,
+              enabled: false,
+            });
+          } catch (disableErr) {
+            console.warn("Failed to disable managed domain:", disableErr);
+          }
         } else {
           console.warn("Failed to attach custom domain, falling back:", err);
           usedManagedFallback = true;
+          customDomainStatus = "error";
         }
       }
     }
 
     if (!publicBaseUrl) {
-      publicBaseUrl = await enableManagedDomain({
+      const managed = await setManagedDomain({
         accountId,
         bucket: bucketName,
         token: apiToken,
+        enabled: true,
       });
+      publicBaseUrl = managed?.url || `https://${bucketName}.${accountId}.r2.dev`;
       domainType = "managed";
       usedManagedFallback = true;
+      customDomainStatus = customDomainStatus === "skipped" ? "managed" : customDomainStatus;
     }
 
     const mergedEntry = {
@@ -1662,7 +1700,7 @@ class bitvidApp {
       mergeBucketEntry(this.cloudflareSettings, npub, mergedEntry)
     );
     this.cloudflareSettings = updatedSettings;
-    return { entry: mergedEntry, usedManagedFallback };
+    return { entry: mergedEntry, usedManagedFallback, customDomainStatus };
   }
 
   async updateCloudflareBucketPreview() {
@@ -1785,10 +1823,18 @@ class bitvidApp {
       return;
     }
 
-    const statusMessage = bucketResult?.usedManagedFallback &&
-      (this.cloudflareSettings?.baseDomain || "")
-        ? `Using managed r2.dev domain for ${bucketEntry.bucket}. Verify your Cloudflare zone. Uploading…`
-        : `Uploading to ${bucketEntry.bucket}…`;
+    let statusMessage = `Uploading to ${bucketEntry.bucket}…`;
+    if (bucketResult?.usedManagedFallback) {
+      const baseDomain = this.cloudflareSettings?.baseDomain || "";
+      if (baseDomain) {
+        const customStatus = bucketResult?.customDomainStatus
+          ? ` (custom domain status: ${bucketResult.customDomainStatus})`
+          : "";
+        statusMessage = `Using managed r2.dev domain for ${bucketEntry.bucket}. Verify your Cloudflare zone${customStatus}. Uploading…`;
+      } else {
+        statusMessage = `Using managed r2.dev domain for ${bucketEntry.bucket}. Uploading…`;
+      }
+    }
 
     this.setCloudflareUploadStatus(
       statusMessage,

js/storage/r2-mgmt.js

@@ -1,12 +1,32 @@
 // js/storage/r2-mgmt.js
 export function sanitizeBucketName(npub) {
-  const base = (npub || "user")
+  let value = String(npub || "")
     .toLowerCase()
-    .replace(/[^a-z0-9-]/g, "")
+    .replace(/[^a-z0-9-]/g, "-")
+    .replace(/-+/g, "-")
     .replace(/^-+|[-]+$/g, "");
-  const suffix = Date.now().toString(36);
-  const name = `bv-${base || "u"}-${suffix}`.slice(0, 63);
-  return name.length < 3 ? `bv-u-${suffix}` : name;
+
+  if (!value) {
+    value = "bitvid-user";
+  }
+
+  if (!/^[a-z0-9]/.test(value)) {
+    value = `bv-${value}`;
+  }
+
+  if (!/[a-z0-9]$/.test(value)) {
+    value = `${value}0`;
+  }
+
+  if (value.length > 63) {
+    value = value.slice(0, 63).replace(/-+$/g, "");
+  }
+
+  while (value.length < 3) {
+    value += "0";
+  }
+
+  return value;
 }
 
 function buildError(error, fallbackMessage) {
@@ -88,11 +108,14 @@ export async function putCors({ accountId, bucket, token, origins }) {
   }
   const rules = [
     {
-      AllowedOrigins: filteredOrigins,
-      AllowedMethods: ["GET", "HEAD", "PUT", "POST"],
-      AllowedHeaders: ["*"],
-      ExposeHeaders: ["ETag", "Content-Length", "Content-Range"],
-      MaxAgeSeconds: 3600,
+      id: "bitvid-default",
+      allowed: {
+        origins: filteredOrigins,
+        methods: ["GET", "HEAD", "PUT", "POST", "DELETE", "OPTIONS"],
+        headers: ["*"],
+      },
+      expose_headers: ["ETag", "Content-Length", "Content-Range", "Accept-Ranges"],
+      max_age_seconds: 3600,
     },
   ];
   await cfFetch(`/accounts/${accountId}/r2/buckets/${bucket}/cors`, {
@@ -102,30 +125,112 @@ export async function putCors({ accountId, bucket, token, origins }) {
   });
 }
 
+function extractStatus(payload) {
+  return (
+    payload?.result?.status ||
+    payload?.result?.result?.status ||
+    payload?.status ||
+    payload?.result?.statusText ||
+    null
+  );
+}
+
+async function pollCustomDomain({
+  accountId,
+  bucket,
+  token,
+  domain,
+  pollInterval = 2500,
+  timeoutMs = 120000,
+}) {
+  const path = `/accounts/${accountId}/r2/buckets/${bucket}/domains/custom/${encodeURIComponent(
+    domain
+  )}`;
+  const deadline = Date.now() + timeoutMs;
+  let lastPayload = null;
+
+  while (Date.now() < deadline) {
+    lastPayload = await cfFetch(path, { token, method: "GET" }).catch((err) => {
+      if (err?.status === 404) {
+        return null;
+      }
+      throw err;
+    });
+
+    if (!lastPayload) {
+      break;
+    }
+
+    const status = extractStatus(lastPayload);
+    if (status === "active") {
+      return { status, payload: lastPayload };
+    }
+
+    await new Promise((resolve) => setTimeout(resolve, pollInterval));
+  }
+
+  return {
+    status: extractStatus(lastPayload) || "unknown",
+    payload: lastPayload,
+  };
+}
+
 // 3) Attach custom domain to bucket (auto-provisions on Cloudflare)
-export async function attachCustomDomain({
+export async function attachCustomDomainAndWait({
   accountId,
   bucket,
   token,
   zoneId,
   domain,
+  pollInterval,
+  timeoutMs,
 }) {
-  const { result } = await cfFetch(
-    `/accounts/${accountId}/r2/buckets/${bucket}/domains/custom`,
-    {
+  let shouldPoll = true;
+  try {
+    await cfFetch(`/accounts/${accountId}/r2/buckets/${bucket}/domains/custom`, {
       token,
       method: "POST",
-      body: { domain, zoneId, enabled: true, minTLS: "1.2" },
+      body: { domain, zoneId, enabled: true },
+    });
+  } catch (error) {
+    const err = buildError(error);
+    if (err.status === 409 || /already exists/i.test(err.message || "")) {
+      shouldPoll = true;
+    } else {
+      throw err;
     }
-  );
-  return `https://${result?.domain || domain}`;
+  }
+
+  let status = "unknown";
+  if (shouldPoll) {
+    const pollResult = await pollCustomDomain({
+      accountId,
+      bucket,
+      token,
+      domain,
+      pollInterval,
+      timeoutMs,
+    });
+    status = pollResult.status;
+  }
+
+  const url = `https://${domain}`;
+  return {
+    url,
+    status,
+    active: status === "active",
+  };
 }
 
-// 4) Or enable r2.dev managed domain as fallback
-export async function enableManagedDomain({ accountId, bucket, token }) {
+// 4) Toggle managed r2.dev domain for the bucket
+export async function setManagedDomain({ accountId, bucket, token, enabled }) {
   const { result } = await cfFetch(
     `/accounts/${accountId}/r2/buckets/${bucket}/domains/managed`,
-    { token, method: "PUT", body: { enabled: true } }
+    { token, method: "PUT", body: { enabled: Boolean(enabled) } }
   );
-  return `https://${result?.domain}`;
+  const domain = result?.domain || result?.result?.domain || "";
+  return {
+    enabled: Boolean(result?.enabled ?? enabled),
+    url: domain ? `https://${domain}` : "",
+  };
 }

js/storage/r2-s3.js

@@ -5,7 +5,28 @@ import {
   UploadPartCommand,
   CompleteMultipartUploadCommand,
   AbortMultipartUploadCommand,
-} from "https://esm.sh/@aws-sdk/[email protected]14.0?target=es2022&bundle";
+} from "https://esm.sh/@aws-sdk/[email protected]37.0?target=es2022&bundle";
+
+function computeCacheControl(key) {
+  if (!key) {
+    return "public, max-age=3600";
+  }
+
+  const lower = String(key).toLowerCase();
+  if (/(?:\.)(m3u8|mpd)$/.test(lower)) {
+    return "public, max-age=60, must-revalidate";
+  }
+
+  if (
+    /\.(m4s|ts|mp4|webm|mov|mkv|png|jpg|jpeg|gif|svg|vtt|srt|mpg|mpeg)$/.test(
+      lower
+    )
+  ) {
+    return "public, max-age=31536000, immutable";
+  }
+
+  return "public, max-age=3600";
+}
 
 export function makeR2Client({ accountId, accessKeyId, secretAccessKey }) {
   if (!accountId || !accessKeyId || !secretAccessKey) {
@@ -27,7 +48,6 @@ export async function multipartUpload({
   file,
   contentType,
   onProgress,
-  concurrency = 4,
 }) {
   if (!s3) {
     throw new Error("S3 client is required");
@@ -42,79 +62,67 @@ export async function multipartUpload({
     throw new Error("File is required");
   }
 
-  const { UploadId } = await s3.send(
-    new CreateMultipartUploadCommand({
-      Bucket: bucket,
-      Key: key,
-      ContentType: contentType || file.type || "video/mp4",
-      CacheControl: "public, max-age=31536000, immutable",
-    })
+  const createCommand = new CreateMultipartUploadCommand({
+    Bucket: bucket,
+    Key: key,
+    ContentType: contentType || file.type || "application/octet-stream",
+    CacheControl: computeCacheControl(key),
+  });
+
+  createCommand.middlewareStack.add(
+    (next) => async (args) => {
+      if (args?.request?.headers) {
+        args.request.headers["cf-create-bucket-if-missing"] = "true";
+      }
+      return next(args);
+    },
+    { step: "build" }
   );
 
+  const { UploadId } = await s3.send(createCommand);
+
   if (!UploadId) {
     throw new Error("Failed to start multipart upload");
   }
 
-  const PART = 8 * 1024 * 1024;
+  const MIN_PART = 5 * 1024 * 1024;
+  const maxParts = 10000;
+  const calculated = Math.ceil(file.size / maxParts);
+  const partSize = Math.max(MIN_PART, calculated);
   const total = file.size;
   const parts = [];
-  let sent = 0;
+  let uploadedBytes = 0;
   let partNumber = 1;
-  const totalParts = Math.ceil(total / PART);
-  const errors = [];
 
-  const uploadPart = async () => {
-    const start = sent;
-    if (start >= total) {
-      return null;
-    }
-    const end = Math.min(start + PART, total);
-    const body = file.slice(start, end);
-    const currentPart = partNumber++;
-    sent = end;
+  if (typeof onProgress === "function") {
+    onProgress(0);
+  }
 
-    return s3
-      .send(
+  try {
+    while (uploadedBytes < total) {
+      const start = uploadedBytes;
+      const end = Math.min(start + partSize, total);
+      const chunk = file.slice(start, end);
+
+      const { ETag } = await s3.send(
         new UploadPartCommand({
           Bucket: bucket,
           Key: key,
           UploadId,
-          PartNumber: currentPart,
-          Body: body,
+          PartNumber: partNumber,
+          Body: chunk,
         })
-      )
-      .then(({ ETag }) => {
-        parts.push({ ETag, PartNumber: currentPart });
-        if (typeof onProgress === "function") {
-          onProgress(end / total);
-        }
-      });
-  };
+      );
 
-  const workers = Array.from({ length: Math.max(1, concurrency) }, () =>
-    (async () => {
-      try {
-        while (parts.length < totalParts) {
-          const task = uploadPart();
-          if (!task) {
-            break;
-          }
-          await task;
-        }
-      } catch (error) {
-        errors.push(error);
+      parts.push({ ETag, PartNumber: partNumber });
+      uploadedBytes = end;
+      partNumber += 1;
+
+      if (typeof onProgress === "function") {
+        onProgress(uploadedBytes / total);
       }
-    })()
-  );
-
-  try {
-    await Promise.all(workers);
-    if (errors.length > 0) {
-      throw errors[0];
     }
 
-    parts.sort((a, b) => a.PartNumber - b.PartNumber);
-
     await s3.send(
       new CompleteMultipartUploadCommand({
         Bucket: bucket,