Merge pull request #1620 from PR0M3TH3AN/codex/create-s3-multipart-upload-helper-module

Add provider-agnostic S3 multipart helpers

docs/storage-r2.md

@@ -57,7 +57,7 @@ The app will attempt to upload a small test file to verify the credentials and p
     - `StorageService` handles the NIP-44/04 decryption of the Master Key and subsequent decryption of the R2 connection payload.
 
 2.  **Upload Execution:**
-    - `r2Service.multipartUpload()` uses the standard AWS SDK commands (`CreateMultipartUpload`, `UploadPart`, etc.) via the generic `makeR2Client` adapter.
+    - `r2Service.multipartUpload()` uses the standard AWS SDK commands (`CreateMultipartUpload`, `UploadPart`, etc.) via `js/storage/s3-multipart.js` and the `makeR2Client` adapter.
     - The adapter automatically constructs the R2 endpoint (`https://<accountId>.r2.cloudflarestorage.com`) using the stored `accountId`.
 
 3.  **Publish:**
@@ -81,7 +81,8 @@ R2 credentials are no longer stored in plain `bitvidSettings`. They are encapsul
 ## Environment / global flags
 - `globalThis.__BITVID_DISABLE_NETWORK_IMPORTS__`: disables the AWS SDK network import used by the
   R2 S3 client loader (`js/storage/r2-s3.js`). When true, uploads will fail with a load error until
-  the SDK is bundled or otherwise provided.
+  the SDK is bundled or otherwise provided. Multipart helpers live in `js/storage/s3-multipart.js` and
+  rely on the same SDK import.
 
 ## Security considerations
 

js/r2-quick.js

@@ -6,7 +6,8 @@ import {
   attachCustomDomainAndWait,
   setManagedDomain,
 } from "./storage/r2-mgmt.js";
-import { makeR2Client, multipartUpload } from "./storage/r2-s3.js";
+import { makeR2Client } from "./storage/r2-s3.js";
+import { multipartUpload } from "./storage/s3-multipart.js";
 import { userLogger } from "./utils/logger.js";
 
 const STORAGE_KEY = "bitvid:quickR2Settings";
@@ -364,6 +365,7 @@ export function initQuickR2Upload(app) {
         key,
         file,
         contentType: file.type,
+        createBucketIfMissing: true,
         onProgress: (fraction) => {
           if (typeof fraction !== "number" || !isFinite(fraction)) {
             return;

js/services/r2Service.js

@@ -5,7 +5,7 @@
  *
  * Key Responsibilities:
  * - Managing user R2 credentials and bucket settings.
- * - performing S3-compatible multipart uploads (via `js/storage/r2-s3.js`).
+ * - performing S3-compatible multipart uploads (via `js/storage/s3-multipart.js`).
  * - Orchestrating the "Hybrid" video hosting strategy:
  *   1. Videos are uploaded to R2 (S3) for reliable direct hosting.
  *   2. A `.torrent` file is generated and uploaded alongside the video.
@@ -33,13 +33,13 @@ import {
   setManagedDomain,
   deriveShortSubdomain,
 } from "../storage/r2-mgmt.js";
+import { makeR2Client } from "../storage/r2-s3.js";
 import {
-  makeR2Client,
   multipartUpload,
   ensureBucketCors,
   ensureBucketExists,
   deleteObject,
-} from "../storage/r2-s3.js";
+} from "../storage/s3-multipart.js";
 import { truncateMiddle } from "../utils/formatters.js";
 import { userLogger } from "../utils/logger.js";
 import {
@@ -531,7 +531,11 @@ class R2Service {
 
         // Attempt to auto-create the bucket (requires Admin keys, but harmless if fails)
         try {
-          await ensureBucketExists({ s3, bucket: bucketName });
+          await ensureBucketExists({
+            s3,
+            bucket: bucketName,
+            region: settings.region,
+          });
         } catch (createErr) {
           // 403 Forbidden is expected if keys are "Object Read & Write" only.
           // We proceed assuming the user might have created it manually.
@@ -546,6 +550,7 @@ class R2Service {
             s3,
             bucket: bucketName,
             origins: corsOrigins,
+            region: settings.region,
           });
         }
       } catch (corsErr) {
@@ -645,7 +650,11 @@ class R2Service {
 
       // 2. Ensure bucket (best effort)
       try {
-        await ensureBucketExists({ s3, bucket: bucketName });
+        await ensureBucketExists({
+          s3,
+          bucket: bucketName,
+          region: settings.region,
+        });
       } catch (setupErr) {
         userLogger.warn("Bucket creation/check warning during verification:", setupErr);
         // Continue, assuming bucket might already exist and be configured
@@ -659,6 +668,7 @@ class R2Service {
             s3,
             bucket: bucketName,
             origins: corsOrigins,
+            region: settings.region,
           });
         }
       } catch (corsErr) {
@@ -673,6 +683,8 @@ class R2Service {
         key: verifyKey,
         file,
         contentType: "text/plain",
+        createBucketIfMissing: true,
+        region: settings.region,
       });
 
       // 4. Verify Public Access (Fetch)
@@ -886,19 +898,21 @@ class R2Service {
         const thumbKey = key.replace(/\.[^/.]+$/, "") + `.thumb.${thumbExt}`;
 
         try {
-            await multipartUpload({
-                s3,
-                bucket: bucketEntry.bucket,
-                key: thumbKey,
-                file: thumbnailFile,
-                contentType: thumbnailFile.type || "image/jpeg",
-            });
-            const thumbUrl = buildPublicUrl(bucketEntry.publicBaseUrl, thumbKey);
-            if (typeof metadata === "object") {
-                metadata.thumbnail = thumbUrl;
-            }
+          await multipartUpload({
+            s3,
+            bucket: bucketEntry.bucket,
+            key: thumbKey,
+            file: thumbnailFile,
+            contentType: thumbnailFile.type || "image/jpeg",
+            createBucketIfMissing: true,
+            region: effectiveSettings.region,
+          });
+          const thumbUrl = buildPublicUrl(bucketEntry.publicBaseUrl, thumbKey);
+          if (typeof metadata === "object") {
+            metadata.thumbnail = thumbUrl;
+          }
         } catch (err) {
-            userLogger.warn("Thumbnail upload failed, continuing with video...", err);
+          userLogger.warn("Thumbnail upload failed, continuing with video...", err);
         }
       }
 
@@ -910,6 +924,8 @@ class R2Service {
         key,
         file,
         contentType: file.type,
+        createBucketIfMissing: true,
+        region: effectiveSettings.region,
         onProgress: (fraction) => {
           this.updateCloudflareProgress(fraction);
         },
@@ -928,6 +944,8 @@ class R2Service {
             key: torrentKey,
             file: torrentFile,
             contentType: "application/x-bittorrent",
+            createBucketIfMissing: true,
+            region: effectiveSettings.region,
           });
           if (!torrentUrl) {
              torrentUrl = buildPublicUrl(bucketEntry.publicBaseUrl, torrentKey);
@@ -1085,6 +1103,8 @@ class R2Service {
       key,
       file,
       contentType: file.type || "application/octet-stream",
+      createBucketIfMissing: true,
+      region,
       onProgress: (fraction) => {
         if (typeof onProgress === "function") {
           onProgress(fraction);

js/storage/r2-s3.js

@@ -7,6 +7,12 @@ import {
   makeS3Client,
   requireAwsSdk,
 } from "./s3-client.js";
+import {
+  ensureBucketExists,
+  ensureBucketCors,
+  deleteObject,
+  multipartUpload,
+} from "./s3-multipart.js";
 
 export async function testS3Connection(config) {
   if (!config) {
@@ -67,27 +73,6 @@ export function isR2SdkAvailable() {
   return isS3SdkAvailable();
 }
 
-function computeCacheControl(key) {
-  if (!key) {
-    return "public, max-age=3600";
-  }
-
-  const lower = String(key).toLowerCase();
-  if (/(?:\.)(m3u8|mpd)$/.test(lower)) {
-    return "public, max-age=60, must-revalidate";
-  }
-
-  if (
-    /\.(m4s|ts|mp4|webm|mov|mkv|png|jpg|jpeg|gif|svg|vtt|srt|mpg|mpeg)$/.test(
-      lower
-    )
-  ) {
-    return "public, max-age=31536000, immutable";
-  }
-
-  return "public, max-age=3600";
-}
-
 export function makeR2Client({
   accountId,
   accessKeyId,
@@ -111,199 +96,4 @@ export function makeR2Client({
   });
 }
 
-export async function ensureBucketExists({ s3, bucket }) {
-  const { CreateBucketCommand, HeadBucketCommand } = requireAwsSdk();
-
-  try {
-    await s3.send(new HeadBucketCommand({ Bucket: bucket }));
-    return;
-  } catch (error) {
-    const status = error?.$metadata?.httpStatusCode || 0;
-    const code = error?.name || error?.Code || "";
-    if (status !== 404 && code !== "NotFound" && code !== "NoSuchBucket") {
-      throw error;
-    }
-  }
-
-  try {
-    await s3.send(new CreateBucketCommand({ Bucket: bucket }));
-  } catch (error) {
-    const status = error?.$metadata?.httpStatusCode || 0;
-    const code = error?.name || error?.Code || "";
-    if (
-      status === 409 ||
-      code === "BucketAlreadyOwnedByYou" ||
-      code === "BucketAlreadyExists"
-    ) {
-      return;
-    }
-    throw error;
-  }
-}
-
-export async function ensureBucketCors({ s3, bucket, origins }) {
-  if (!s3) {
-    throw new Error("S3 client is required to configure CORS");
-  }
-  if (!bucket) {
-    throw new Error("Bucket name is required to configure CORS");
-  }
-
-  const { PutBucketCorsCommand } = requireAwsSdk();
-
-  const allowedOrigins = (origins || []).filter(Boolean);
-  if (allowedOrigins.length === 0) {
-    return;
-  }
-
-  await ensureBucketExists({ s3, bucket });
-
-  const command = new PutBucketCorsCommand({
-    Bucket: bucket,
-    CORSConfiguration: {
-      CORSRules: [
-        {
-          AllowedHeaders: ["*"],
-          AllowedMethods: ["GET", "HEAD", "PUT", "POST", "DELETE", "OPTIONS"],
-          AllowedOrigins: allowedOrigins,
-          ExposeHeaders: [
-            "ETag",
-            "Content-Length",
-            "Content-Range",
-            "Accept-Ranges",
-          ],
-          MaxAgeSeconds: 3600,
-        },
-      ],
-    },
-  });
-
-  await s3.send(command);
-}
-
-export async function deleteObject({ s3, bucket, key }) {
-  if (!s3 || !bucket || !key) {
-    throw new Error("Missing required parameters for deleteObject");
-  }
-
-  const { DeleteObjectCommand } = requireAwsSdk();
-
-  try {
-    await s3.send(
-      new DeleteObjectCommand({
-        Bucket: bucket,
-        Key: key,
-      })
-    );
-  } catch (error) {
-    // We treat 404 (NoSuchKey) as success since the object is gone.
-    const status = error?.$metadata?.httpStatusCode || 0;
-    const code = error?.name || error?.Code || "";
-    if (status !== 404 && code !== "NoSuchKey") {
-      throw error;
-    }
-  }
-}
-
-export async function multipartUpload({
-  s3,
-  bucket,
-  key,
-  file,
-  contentType,
-  onProgress,
-}) {
-  if (!s3) {
-    throw new Error("S3 client is required");
-  }
-  if (!bucket) {
-    throw new Error("Bucket is required");
-  }
-  if (!key) {
-    throw new Error("Object key is required");
-  }
-  if (!file) {
-    throw new Error("File is required");
-  }
-
-  const {
-    CreateMultipartUploadCommand,
-    UploadPartCommand,
-    CompleteMultipartUploadCommand,
-    AbortMultipartUploadCommand,
-  } = requireAwsSdk();
-
-  console.debug("[R2] Starting multipart upload for:", key);
-
-  const createCommand = new CreateMultipartUploadCommand({
-    Bucket: bucket,
-    Key: key,
-    ContentType: contentType || file.type || "application/octet-stream",
-    CacheControl: computeCacheControl(key),
-  });
-
-  const { UploadId } = await s3.send(createCommand);
-
-  if (!UploadId) {
-    throw new Error("Failed to start multipart upload");
-  }
-
-  const MIN_PART = 5 * 1024 * 1024;
-  const maxParts = 10000;
-  const calculated = Math.ceil(file.size / maxParts);
-  const partSize = Math.max(MIN_PART, calculated);
-  const total = file.size;
-  const parts = [];
-  let uploadedBytes = 0;
-  let partNumber = 1;
-
-  if (typeof onProgress === "function") {
-    onProgress(0);
-  }
-
-  try {
-    while (uploadedBytes < total) {
-      const start = uploadedBytes;
-      const end = Math.min(start + partSize, total);
-      const chunk = file.slice(start, end);
-
-      const { ETag } = await s3.send(
-        new UploadPartCommand({
-          Bucket: bucket,
-          Key: key,
-          UploadId,
-          PartNumber: partNumber,
-          Body: chunk,
-        })
-      );
-
-      parts.push({ ETag, PartNumber: partNumber });
-      uploadedBytes = end;
-      partNumber += 1;
-
-      if (typeof onProgress === "function") {
-        onProgress(uploadedBytes / total);
-      }
-    }
-
-    await s3.send(
-      new CompleteMultipartUploadCommand({
-        Bucket: bucket,
-        Key: key,
-        UploadId,
-        MultipartUpload: { Parts: parts },
-      })
-    );
-  } catch (error) {
-    await s3
-      .send(
-        new AbortMultipartUploadCommand({
-          Bucket: bucket,
-          Key: key,
-          UploadId,
-        })
-      )
-      .catch(() => {});
-    throw error;
-  }
-}
+export { ensureBucketExists, ensureBucketCors, deleteObject, multipartUpload };

js/storage/s3-multipart.js

@@ -0,0 +1,272 @@
+// js/storage/s3-multipart.js
+
+import { requireAwsSdk } from "./s3-client.js";
+import logger from "../utils/logger.js";
+
+const DEFAULT_CACHE_CONTROL = "public, max-age=3600";
+
+function normalizeRegion(region) {
+  if (!region) {
+    return "";
+  }
+  const trimmed = String(region).trim();
+  if (!trimmed || trimmed.toLowerCase() === "auto") {
+    return "";
+  }
+  return trimmed;
+}
+
+function getCreateBucketConfig(region) {
+  const normalized = normalizeRegion(region);
+  if (!normalized || normalized === "us-east-1") {
+    return {};
+  }
+  return {
+    CreateBucketConfiguration: {
+      LocationConstraint: normalized,
+    },
+  };
+}
+
+function computeCacheControl(key) {
+  if (!key) {
+    return DEFAULT_CACHE_CONTROL;
+  }
+
+  const lower = String(key).toLowerCase();
+  if (/(?:\.)(m3u8|mpd)$/.test(lower)) {
+    return "public, max-age=60, must-revalidate";
+  }
+
+  if (
+    /\.(m4s|ts|mp4|webm|mov|mkv|png|jpg|jpeg|gif|svg|vtt|srt|mpg|mpeg)$/.test(
+      lower
+    )
+  ) {
+    return "public, max-age=31536000, immutable";
+  }
+
+  return DEFAULT_CACHE_CONTROL;
+}
+
+export async function ensureBucketExists({ s3, bucket, region } = {}) {
+  if (!s3) {
+    throw new Error("S3 client is required to ensure buckets");
+  }
+  if (!bucket) {
+    throw new Error("Bucket name is required to ensure buckets");
+  }
+
+  const { CreateBucketCommand, HeadBucketCommand } = requireAwsSdk();
+
+  try {
+    await s3.send(new HeadBucketCommand({ Bucket: bucket }));
+    return;
+  } catch (error) {
+    const status = error?.$metadata?.httpStatusCode || 0;
+    const code = error?.name || error?.Code || "";
+    if (status !== 404 && code !== "NotFound" && code !== "NoSuchBucket") {
+      throw error;
+    }
+  }
+
+  try {
+    await s3.send(
+      new CreateBucketCommand({
+        Bucket: bucket,
+        ...getCreateBucketConfig(region),
+      })
+    );
+  } catch (error) {
+    const status = error?.$metadata?.httpStatusCode || 0;
+    const code = error?.name || error?.Code || "";
+    if (
+      status === 409 ||
+      code === "BucketAlreadyOwnedByYou" ||
+      code === "BucketAlreadyExists"
+    ) {
+      return;
+    }
+    throw error;
+  }
+}
+
+export async function ensureBucketCors({ s3, bucket, origins, region } = {}) {
+  if (!s3) {
+    throw new Error("S3 client is required to configure CORS");
+  }
+  if (!bucket) {
+    throw new Error("Bucket name is required to configure CORS");
+  }
+
+  const { PutBucketCorsCommand } = requireAwsSdk();
+
+  const allowedOrigins = (origins || []).filter(Boolean);
+  if (allowedOrigins.length === 0) {
+    return;
+  }
+
+  await ensureBucketExists({ s3, bucket, region });
+
+  const command = new PutBucketCorsCommand({
+    Bucket: bucket,
+    CORSConfiguration: {
+      CORSRules: [
+        {
+          AllowedHeaders: ["*"],
+          AllowedMethods: ["GET", "HEAD", "PUT", "POST", "DELETE", "OPTIONS"],
+          AllowedOrigins: allowedOrigins,
+          ExposeHeaders: [
+            "ETag",
+            "Content-Length",
+            "Content-Range",
+            "Accept-Ranges",
+          ],
+          MaxAgeSeconds: 3600,
+        },
+      ],
+    },
+  });
+
+  await s3.send(command);
+}
+
+export async function deleteObject({ s3, bucket, key } = {}) {
+  if (!s3 || !bucket || !key) {
+    throw new Error("Missing required parameters for deleteObject");
+  }
+
+  const { DeleteObjectCommand } = requireAwsSdk();
+
+  try {
+    await s3.send(
+      new DeleteObjectCommand({
+        Bucket: bucket,
+        Key: key,
+      })
+    );
+  } catch (error) {
+    // We treat 404 (NoSuchKey) as success since the object is gone.
+    const status = error?.$metadata?.httpStatusCode || 0;
+    const code = error?.name || error?.Code || "";
+    if (status !== 404 && code !== "NoSuchKey") {
+      throw error;
+    }
+  }
+}
+
+export async function multipartUpload({
+  s3,
+  bucket,
+  key,
+  file,
+  contentType,
+  onProgress,
+  createBucketIfMissing = false,
+  region,
+} = {}) {
+  if (!s3) {
+    throw new Error("S3 client is required");
+  }
+  if (!bucket) {
+    throw new Error("Bucket is required");
+  }
+  if (!key) {
+    throw new Error("Object key is required");
+  }
+  if (!file) {
+    throw new Error("File is required");
+  }
+
+  const {
+    CreateMultipartUploadCommand,
+    UploadPartCommand,
+    CompleteMultipartUploadCommand,
+    AbortMultipartUploadCommand,
+  } = requireAwsSdk();
+
+  if (createBucketIfMissing) {
+    try {
+      await ensureBucketExists({ s3, bucket, region });
+    } catch (error) {
+      logger.dev.warn(
+        "[S3 multipart] Bucket creation check failed, continuing anyway.",
+        error
+      );
+    }
+  }
+
+  logger.dev.debug("[S3 multipart] Starting multipart upload for:", key);
+
+  const createCommand = new CreateMultipartUploadCommand({
+    Bucket: bucket,
+    Key: key,
+    ContentType: contentType || file.type || "application/octet-stream",
+    CacheControl: computeCacheControl(key),
+  });
+
+  const { UploadId } = await s3.send(createCommand);
+
+  if (!UploadId) {
+    throw new Error("Failed to start multipart upload");
+  }
+
+  const MIN_PART = 5 * 1024 * 1024;
+  const maxParts = 10000;
+  const calculated = Math.ceil(file.size / maxParts);
+  const partSize = Math.max(MIN_PART, calculated);
+  const total = file.size;
+  const parts = [];
+  let uploadedBytes = 0;
+  let partNumber = 1;
+
+  if (typeof onProgress === "function") {
+    onProgress(0);
+  }
+
+  try {
+    while (uploadedBytes < total) {
+      const start = uploadedBytes;
+      const end = Math.min(start + partSize, total);
+      const chunk = file.slice(start, end);
+
+      const { ETag } = await s3.send(
+        new UploadPartCommand({
+          Bucket: bucket,
+          Key: key,
+          UploadId,
+          PartNumber: partNumber,
+          Body: chunk,
+        })
+      );
+
+      parts.push({ ETag, PartNumber: partNumber });
+      uploadedBytes = end;
+      partNumber += 1;
+
+      if (typeof onProgress === "function") {
+        onProgress(uploadedBytes / total);
+      }
+    }
+
+    await s3.send(
+      new CompleteMultipartUploadCommand({
+        Bucket: bucket,
+        Key: key,
+        UploadId,
+        MultipartUpload: { Parts: parts },
+      })
+    );
+  } catch (error) {
+    await s3
+      .send(
+        new AbortMultipartUploadCommand({
+          Bucket: bucket,
+          Key: key,
+          UploadId,
+        })
+      )
+      .catch(() => {});
+    throw error;
+  }
+}