diff --git a/docs/storage-r2.md b/docs/storage-r2.md index 63b1cbd1..eaa1ec2f 100644 --- a/docs/storage-r2.md +++ b/docs/storage-r2.md @@ -57,7 +57,7 @@ The app will attempt to upload a small test file to verify the credentials and p - `StorageService` handles the NIP-44/04 decryption of the Master Key and subsequent decryption of the R2 connection payload. 2. **Upload Execution:** - - `r2Service.multipartUpload()` uses the standard AWS SDK commands (`CreateMultipartUpload`, `UploadPart`, etc.) via the generic `makeR2Client` adapter. + - `r2Service.multipartUpload()` uses the standard AWS SDK commands (`CreateMultipartUpload`, `UploadPart`, etc.) via `js/storage/s3-multipart.js` and the `makeR2Client` adapter. - The adapter automatically constructs the R2 endpoint (`https://.r2.cloudflarestorage.com`) using the stored `accountId`. 3. **Publish:** @@ -81,7 +81,8 @@ R2 credentials are no longer stored in plain `bitvidSettings`. They are encapsul ## Environment / global flags - `globalThis.__BITVID_DISABLE_NETWORK_IMPORTS__`: disables the AWS SDK network import used by the R2 S3 client loader (`js/storage/r2-s3.js`). When true, uploads will fail with a load error until - the SDK is bundled or otherwise provided. + the SDK is bundled or otherwise provided. Multipart helpers live in `js/storage/s3-multipart.js` and + rely on the same SDK import. ## Security considerations diff --git a/js/r2-quick.js b/js/r2-quick.js index 23c665be..fad40a5a 100644 --- a/js/r2-quick.js +++ b/js/r2-quick.js @@ -6,7 +6,8 @@ import { attachCustomDomainAndWait, setManagedDomain, } from "./storage/r2-mgmt.js"; -import { makeR2Client, multipartUpload } from "./storage/r2-s3.js"; +import { makeR2Client } from "./storage/r2-s3.js"; +import { multipartUpload } from "./storage/s3-multipart.js"; import { userLogger } from "./utils/logger.js"; const STORAGE_KEY = "bitvid:quickR2Settings"; @@ -364,6 +365,7 @@ export function initQuickR2Upload(app) { key, file, contentType: file.type, + createBucketIfMissing: true, onProgress: (fraction) => { if (typeof fraction !== "number" || !isFinite(fraction)) { return; diff --git a/js/services/r2Service.js b/js/services/r2Service.js index 9d436a4b..2b7dcb18 100644 --- a/js/services/r2Service.js +++ b/js/services/r2Service.js @@ -5,7 +5,7 @@ * * Key Responsibilities: * - Managing user R2 credentials and bucket settings. - * - performing S3-compatible multipart uploads (via `js/storage/r2-s3.js`). + * - performing S3-compatible multipart uploads (via `js/storage/s3-multipart.js`). * - Orchestrating the "Hybrid" video hosting strategy: * 1. Videos are uploaded to R2 (S3) for reliable direct hosting. * 2. A `.torrent` file is generated and uploaded alongside the video. @@ -33,13 +33,13 @@ import { setManagedDomain, deriveShortSubdomain, } from "../storage/r2-mgmt.js"; +import { makeR2Client } from "../storage/r2-s3.js"; import { - makeR2Client, multipartUpload, ensureBucketCors, ensureBucketExists, deleteObject, -} from "../storage/r2-s3.js"; +} from "../storage/s3-multipart.js"; import { truncateMiddle } from "../utils/formatters.js"; import { userLogger } from "../utils/logger.js"; import { @@ -531,7 +531,11 @@ class R2Service { // Attempt to auto-create the bucket (requires Admin keys, but harmless if fails) try { - await ensureBucketExists({ s3, bucket: bucketName }); + await ensureBucketExists({ + s3, + bucket: bucketName, + region: settings.region, + }); } catch (createErr) { // 403 Forbidden is expected if keys are "Object Read & Write" only. // We proceed assuming the user might have created it manually. @@ -546,6 +550,7 @@ class R2Service { s3, bucket: bucketName, origins: corsOrigins, + region: settings.region, }); } } catch (corsErr) { @@ -645,7 +650,11 @@ class R2Service { // 2. Ensure bucket (best effort) try { - await ensureBucketExists({ s3, bucket: bucketName }); + await ensureBucketExists({ + s3, + bucket: bucketName, + region: settings.region, + }); } catch (setupErr) { userLogger.warn("Bucket creation/check warning during verification:", setupErr); // Continue, assuming bucket might already exist and be configured @@ -659,6 +668,7 @@ class R2Service { s3, bucket: bucketName, origins: corsOrigins, + region: settings.region, }); } } catch (corsErr) { @@ -673,6 +683,8 @@ class R2Service { key: verifyKey, file, contentType: "text/plain", + createBucketIfMissing: true, + region: settings.region, }); // 4. Verify Public Access (Fetch) @@ -886,19 +898,21 @@ class R2Service { const thumbKey = key.replace(/\.[^/.]+$/, "") + `.thumb.${thumbExt}`; try { - await multipartUpload({ - s3, - bucket: bucketEntry.bucket, - key: thumbKey, - file: thumbnailFile, - contentType: thumbnailFile.type || "image/jpeg", - }); - const thumbUrl = buildPublicUrl(bucketEntry.publicBaseUrl, thumbKey); - if (typeof metadata === "object") { - metadata.thumbnail = thumbUrl; - } + await multipartUpload({ + s3, + bucket: bucketEntry.bucket, + key: thumbKey, + file: thumbnailFile, + contentType: thumbnailFile.type || "image/jpeg", + createBucketIfMissing: true, + region: effectiveSettings.region, + }); + const thumbUrl = buildPublicUrl(bucketEntry.publicBaseUrl, thumbKey); + if (typeof metadata === "object") { + metadata.thumbnail = thumbUrl; + } } catch (err) { - userLogger.warn("Thumbnail upload failed, continuing with video...", err); + userLogger.warn("Thumbnail upload failed, continuing with video...", err); } } @@ -910,6 +924,8 @@ class R2Service { key, file, contentType: file.type, + createBucketIfMissing: true, + region: effectiveSettings.region, onProgress: (fraction) => { this.updateCloudflareProgress(fraction); }, @@ -928,6 +944,8 @@ class R2Service { key: torrentKey, file: torrentFile, contentType: "application/x-bittorrent", + createBucketIfMissing: true, + region: effectiveSettings.region, }); if (!torrentUrl) { torrentUrl = buildPublicUrl(bucketEntry.publicBaseUrl, torrentKey); @@ -1085,6 +1103,8 @@ class R2Service { key, file, contentType: file.type || "application/octet-stream", + createBucketIfMissing: true, + region, onProgress: (fraction) => { if (typeof onProgress === "function") { onProgress(fraction); diff --git a/js/storage/r2-s3.js b/js/storage/r2-s3.js index 5777e6f9..9703ced9 100644 --- a/js/storage/r2-s3.js +++ b/js/storage/r2-s3.js @@ -7,6 +7,12 @@ import { makeS3Client, requireAwsSdk, } from "./s3-client.js"; +import { + ensureBucketExists, + ensureBucketCors, + deleteObject, + multipartUpload, +} from "./s3-multipart.js"; export async function testS3Connection(config) { if (!config) { @@ -67,27 +73,6 @@ export function isR2SdkAvailable() { return isS3SdkAvailable(); } -function computeCacheControl(key) { - if (!key) { - return "public, max-age=3600"; - } - - const lower = String(key).toLowerCase(); - if (/(?:\.)(m3u8|mpd)$/.test(lower)) { - return "public, max-age=60, must-revalidate"; - } - - if ( - /\.(m4s|ts|mp4|webm|mov|mkv|png|jpg|jpeg|gif|svg|vtt|srt|mpg|mpeg)$/.test( - lower - ) - ) { - return "public, max-age=31536000, immutable"; - } - - return "public, max-age=3600"; -} - export function makeR2Client({ accountId, accessKeyId, @@ -111,199 +96,4 @@ export function makeR2Client({ }); } -export async function ensureBucketExists({ s3, bucket }) { - const { CreateBucketCommand, HeadBucketCommand } = requireAwsSdk(); - - try { - await s3.send(new HeadBucketCommand({ Bucket: bucket })); - return; - } catch (error) { - const status = error?.$metadata?.httpStatusCode || 0; - const code = error?.name || error?.Code || ""; - if (status !== 404 && code !== "NotFound" && code !== "NoSuchBucket") { - throw error; - } - } - - try { - await s3.send(new CreateBucketCommand({ Bucket: bucket })); - } catch (error) { - const status = error?.$metadata?.httpStatusCode || 0; - const code = error?.name || error?.Code || ""; - if ( - status === 409 || - code === "BucketAlreadyOwnedByYou" || - code === "BucketAlreadyExists" - ) { - return; - } - throw error; - } -} - -export async function ensureBucketCors({ s3, bucket, origins }) { - if (!s3) { - throw new Error("S3 client is required to configure CORS"); - } - if (!bucket) { - throw new Error("Bucket name is required to configure CORS"); - } - - const { PutBucketCorsCommand } = requireAwsSdk(); - - const allowedOrigins = (origins || []).filter(Boolean); - if (allowedOrigins.length === 0) { - return; - } - - await ensureBucketExists({ s3, bucket }); - - const command = new PutBucketCorsCommand({ - Bucket: bucket, - CORSConfiguration: { - CORSRules: [ - { - AllowedHeaders: ["*"], - AllowedMethods: ["GET", "HEAD", "PUT", "POST", "DELETE", "OPTIONS"], - AllowedOrigins: allowedOrigins, - ExposeHeaders: [ - "ETag", - "Content-Length", - "Content-Range", - "Accept-Ranges", - ], - MaxAgeSeconds: 3600, - }, - ], - }, - }); - - await s3.send(command); -} - -export async function deleteObject({ s3, bucket, key }) { - if (!s3 || !bucket || !key) { - throw new Error("Missing required parameters for deleteObject"); - } - - const { DeleteObjectCommand } = requireAwsSdk(); - - try { - await s3.send( - new DeleteObjectCommand({ - Bucket: bucket, - Key: key, - }) - ); - } catch (error) { - // We treat 404 (NoSuchKey) as success since the object is gone. - const status = error?.$metadata?.httpStatusCode || 0; - const code = error?.name || error?.Code || ""; - if (status !== 404 && code !== "NoSuchKey") { - throw error; - } - } -} - -export async function multipartUpload({ - s3, - bucket, - key, - file, - contentType, - onProgress, -}) { - if (!s3) { - throw new Error("S3 client is required"); - } - if (!bucket) { - throw new Error("Bucket is required"); - } - if (!key) { - throw new Error("Object key is required"); - } - if (!file) { - throw new Error("File is required"); - } - - const { - CreateMultipartUploadCommand, - UploadPartCommand, - CompleteMultipartUploadCommand, - AbortMultipartUploadCommand, - } = requireAwsSdk(); - - console.debug("[R2] Starting multipart upload for:", key); - - const createCommand = new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: key, - ContentType: contentType || file.type || "application/octet-stream", - CacheControl: computeCacheControl(key), - }); - - const { UploadId } = await s3.send(createCommand); - - if (!UploadId) { - throw new Error("Failed to start multipart upload"); - } - - const MIN_PART = 5 * 1024 * 1024; - const maxParts = 10000; - const calculated = Math.ceil(file.size / maxParts); - const partSize = Math.max(MIN_PART, calculated); - const total = file.size; - const parts = []; - let uploadedBytes = 0; - let partNumber = 1; - - if (typeof onProgress === "function") { - onProgress(0); - } - - try { - while (uploadedBytes < total) { - const start = uploadedBytes; - const end = Math.min(start + partSize, total); - const chunk = file.slice(start, end); - - const { ETag } = await s3.send( - new UploadPartCommand({ - Bucket: bucket, - Key: key, - UploadId, - PartNumber: partNumber, - Body: chunk, - }) - ); - - parts.push({ ETag, PartNumber: partNumber }); - uploadedBytes = end; - partNumber += 1; - - if (typeof onProgress === "function") { - onProgress(uploadedBytes / total); - } - } - - await s3.send( - new CompleteMultipartUploadCommand({ - Bucket: bucket, - Key: key, - UploadId, - MultipartUpload: { Parts: parts }, - }) - ); - } catch (error) { - await s3 - .send( - new AbortMultipartUploadCommand({ - Bucket: bucket, - Key: key, - UploadId, - }) - ) - .catch(() => {}); - throw error; - } -} +export { ensureBucketExists, ensureBucketCors, deleteObject, multipartUpload }; diff --git a/js/storage/s3-multipart.js b/js/storage/s3-multipart.js new file mode 100644 index 00000000..b56656ce --- /dev/null +++ b/js/storage/s3-multipart.js @@ -0,0 +1,272 @@ +// js/storage/s3-multipart.js + +import { requireAwsSdk } from "./s3-client.js"; +import logger from "../utils/logger.js"; + +const DEFAULT_CACHE_CONTROL = "public, max-age=3600"; + +function normalizeRegion(region) { + if (!region) { + return ""; + } + const trimmed = String(region).trim(); + if (!trimmed || trimmed.toLowerCase() === "auto") { + return ""; + } + return trimmed; +} + +function getCreateBucketConfig(region) { + const normalized = normalizeRegion(region); + if (!normalized || normalized === "us-east-1") { + return {}; + } + return { + CreateBucketConfiguration: { + LocationConstraint: normalized, + }, + }; +} + +function computeCacheControl(key) { + if (!key) { + return DEFAULT_CACHE_CONTROL; + } + + const lower = String(key).toLowerCase(); + if (/(?:\.)(m3u8|mpd)$/.test(lower)) { + return "public, max-age=60, must-revalidate"; + } + + if ( + /\.(m4s|ts|mp4|webm|mov|mkv|png|jpg|jpeg|gif|svg|vtt|srt|mpg|mpeg)$/.test( + lower + ) + ) { + return "public, max-age=31536000, immutable"; + } + + return DEFAULT_CACHE_CONTROL; +} + +export async function ensureBucketExists({ s3, bucket, region } = {}) { + if (!s3) { + throw new Error("S3 client is required to ensure buckets"); + } + if (!bucket) { + throw new Error("Bucket name is required to ensure buckets"); + } + + const { CreateBucketCommand, HeadBucketCommand } = requireAwsSdk(); + + try { + await s3.send(new HeadBucketCommand({ Bucket: bucket })); + return; + } catch (error) { + const status = error?.$metadata?.httpStatusCode || 0; + const code = error?.name || error?.Code || ""; + if (status !== 404 && code !== "NotFound" && code !== "NoSuchBucket") { + throw error; + } + } + + try { + await s3.send( + new CreateBucketCommand({ + Bucket: bucket, + ...getCreateBucketConfig(region), + }) + ); + } catch (error) { + const status = error?.$metadata?.httpStatusCode || 0; + const code = error?.name || error?.Code || ""; + if ( + status === 409 || + code === "BucketAlreadyOwnedByYou" || + code === "BucketAlreadyExists" + ) { + return; + } + throw error; + } +} + +export async function ensureBucketCors({ s3, bucket, origins, region } = {}) { + if (!s3) { + throw new Error("S3 client is required to configure CORS"); + } + if (!bucket) { + throw new Error("Bucket name is required to configure CORS"); + } + + const { PutBucketCorsCommand } = requireAwsSdk(); + + const allowedOrigins = (origins || []).filter(Boolean); + if (allowedOrigins.length === 0) { + return; + } + + await ensureBucketExists({ s3, bucket, region }); + + const command = new PutBucketCorsCommand({ + Bucket: bucket, + CORSConfiguration: { + CORSRules: [ + { + AllowedHeaders: ["*"], + AllowedMethods: ["GET", "HEAD", "PUT", "POST", "DELETE", "OPTIONS"], + AllowedOrigins: allowedOrigins, + ExposeHeaders: [ + "ETag", + "Content-Length", + "Content-Range", + "Accept-Ranges", + ], + MaxAgeSeconds: 3600, + }, + ], + }, + }); + + await s3.send(command); +} + +export async function deleteObject({ s3, bucket, key } = {}) { + if (!s3 || !bucket || !key) { + throw new Error("Missing required parameters for deleteObject"); + } + + const { DeleteObjectCommand } = requireAwsSdk(); + + try { + await s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: key, + }) + ); + } catch (error) { + // We treat 404 (NoSuchKey) as success since the object is gone. + const status = error?.$metadata?.httpStatusCode || 0; + const code = error?.name || error?.Code || ""; + if (status !== 404 && code !== "NoSuchKey") { + throw error; + } + } +} + +export async function multipartUpload({ + s3, + bucket, + key, + file, + contentType, + onProgress, + createBucketIfMissing = false, + region, +} = {}) { + if (!s3) { + throw new Error("S3 client is required"); + } + if (!bucket) { + throw new Error("Bucket is required"); + } + if (!key) { + throw new Error("Object key is required"); + } + if (!file) { + throw new Error("File is required"); + } + + const { + CreateMultipartUploadCommand, + UploadPartCommand, + CompleteMultipartUploadCommand, + AbortMultipartUploadCommand, + } = requireAwsSdk(); + + if (createBucketIfMissing) { + try { + await ensureBucketExists({ s3, bucket, region }); + } catch (error) { + logger.dev.warn( + "[S3 multipart] Bucket creation check failed, continuing anyway.", + error + ); + } + } + + logger.dev.debug("[S3 multipart] Starting multipart upload for:", key); + + const createCommand = new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + ContentType: contentType || file.type || "application/octet-stream", + CacheControl: computeCacheControl(key), + }); + + const { UploadId } = await s3.send(createCommand); + + if (!UploadId) { + throw new Error("Failed to start multipart upload"); + } + + const MIN_PART = 5 * 1024 * 1024; + const maxParts = 10000; + const calculated = Math.ceil(file.size / maxParts); + const partSize = Math.max(MIN_PART, calculated); + const total = file.size; + const parts = []; + let uploadedBytes = 0; + let partNumber = 1; + + if (typeof onProgress === "function") { + onProgress(0); + } + + try { + while (uploadedBytes < total) { + const start = uploadedBytes; + const end = Math.min(start + partSize, total); + const chunk = file.slice(start, end); + + const { ETag } = await s3.send( + new UploadPartCommand({ + Bucket: bucket, + Key: key, + UploadId, + PartNumber: partNumber, + Body: chunk, + }) + ); + + parts.push({ ETag, PartNumber: partNumber }); + uploadedBytes = end; + partNumber += 1; + + if (typeof onProgress === "function") { + onProgress(uploadedBytes / total); + } + } + + await s3.send( + new CompleteMultipartUploadCommand({ + Bucket: bucket, + Key: key, + UploadId, + MultipartUpload: { Parts: parts }, + }) + ); + } catch (error) { + await s3 + .send( + new AbortMultipartUploadCommand({ + Bucket: bucket, + Key: key, + UploadId, + }) + ) + .catch(() => {}); + throw error; + } +}