Merge pull request #1661 from PR0M3TH3AN/feat/s3-quick-upload-4202021437560592801

Add S3 quick upload with presigned manifest support

components/upload-modal.html

@@ -265,44 +265,68 @@
                                 <p class="text-xs text-muted">
                                     Configure a generic S3-compatible bucket for direct uploads.
                                 </p>
-                                <div class="space-y-1">
-                                    <label for="input-s3-endpoint" class="text-xs font-medium text-text">Endpoint</label>
-                                    <input id="input-s3-endpoint" type="text" class="input text-xs" placeholder="https://host[:port]" />
-                                </div>
-                                <div class="grid gap-3 md:grid-cols-2">
-                                    <div class="space-y-1">
-                                        <label for="input-s3-region" class="text-xs font-medium text-text">Region (optional)</label>
-                                        <input id="input-s3-region" type="text" class="input text-xs" placeholder="auto" />
-                                    </div>
-                                    <div class="space-y-1">
-                                        <label for="input-s3-bucket" class="text-xs font-medium text-text">Bucket (explicit override if needed)</label>
-                                        <input id="input-s3-bucket" type="text" class="input text-xs" placeholder="my-bucket" />
-                                    </div>
-                                </div>
-                                <div class="grid gap-3 md:grid-cols-2">
-                                    <div class="space-y-1">
-                                        <label for="input-s3-key" class="text-xs font-medium text-text">Access Key ID</label>
-                                        <input id="input-s3-key" type="text" class="input text-xs" placeholder="AKIA..." />
-                                    </div>
-                                    <div class="space-y-1">
-                                        <label for="input-s3-secret" class="text-xs font-medium text-text">Secret Access Key</label>
-                                        <input id="input-s3-secret" type="password" class="input text-xs" />
-                                    </div>
-                                </div>
-                                <div class="space-y-1">
-                                    <label for="input-s3-public-base" class="text-xs font-medium text-text">Public Base URL Override</label>
-                                    <input id="input-s3-public-base" type="text" class="input text-xs" placeholder="https://cdn.example.com/bucket" />
-                                </div>
-                                <div class="flex flex-wrap gap-4 text-xs text-muted">
-                                    <label class="flex items-center gap-2 cursor-pointer select-none">
-                                        <input type="checkbox" id="check-s3-path-style" class="checkbox" />
-                                        <span>Force path-style URLs</span>
-                                    </label>
-                                    <label class="flex items-center gap-2 cursor-pointer select-none">
-                                        <input type="checkbox" id="check-s3-create-bucket" class="checkbox" />
-                                        <span>Create bucket if missing</span>
+                                <p class="text-xs text-status-warning bg-status-warning/10 p-2 rounded">
+                                    Warning: Browser-held keys are only safe for trusted operators.
+                                </p>
+
+                                <div class="flex items-center gap-2">
+                                    <input type="checkbox" id="quickS3ManifestToggle" class="checkbox" />
+                                    <label for="quickS3ManifestToggle" class="text-xs font-medium text-text select-none">
+                                        Use presigned manifest
                                     </label>
                                 </div>
+
+                                <div id="quickS3CredentialsContainer" class="space-y-4">
+                                    <div class="space-y-1">
+                                        <label for="input-s3-endpoint" class="text-xs font-medium text-text">Endpoint</label>
+                                        <input id="input-s3-endpoint" type="text" class="input text-xs" placeholder="https://host[:port]" />
+                                    </div>
+                                    <div class="grid gap-3 md:grid-cols-2">
+                                        <div class="space-y-1">
+                                            <label for="input-s3-region" class="text-xs font-medium text-text">Region (optional)</label>
+                                            <input id="input-s3-region" type="text" class="input text-xs" placeholder="auto" />
+                                        </div>
+                                        <div class="space-y-1">
+                                            <label for="input-s3-bucket" class="text-xs font-medium text-text">Bucket (explicit override if needed)</label>
+                                            <input id="input-s3-bucket" type="text" class="input text-xs" placeholder="my-bucket" />
+                                        </div>
+                                    </div>
+                                    <div class="grid gap-3 md:grid-cols-2">
+                                        <div class="space-y-1">
+                                            <label for="input-s3-key" class="text-xs font-medium text-text">Access Key ID</label>
+                                            <input id="input-s3-key" type="text" class="input text-xs" placeholder="AKIA..." />
+                                        </div>
+                                        <div class="space-y-1">
+                                            <label for="input-s3-secret" class="text-xs font-medium text-text">Secret Access Key</label>
+                                            <input id="input-s3-secret" type="password" class="input text-xs" />
+                                        </div>
+                                    </div>
+                                    <div class="space-y-1">
+                                        <label for="input-s3-public-base" class="text-xs font-medium text-text">Public Base URL Override</label>
+                                        <input id="input-s3-public-base" type="text" class="input text-xs" placeholder="https://cdn.example.com/bucket" />
+                                    </div>
+                                    <div class="flex flex-wrap gap-4 text-xs text-muted">
+                                        <label class="flex items-center gap-2 cursor-pointer select-none">
+                                            <input type="checkbox" id="check-s3-path-style" class="checkbox" />
+                                            <span>Force path-style URLs</span>
+                                        </label>
+                                        <label class="flex items-center gap-2 cursor-pointer select-none">
+                                            <input type="checkbox" id="check-s3-create-bucket" class="checkbox" />
+                                            <span>Create bucket if missing</span>
+                                        </label>
+                                    </div>
+                                </div>
+
+                                <div id="quickS3ManifestContainer" class="hidden space-y-2">
+                                    <label for="quickS3ManifestJson" class="text-xs font-medium text-text">Manifest JSON</label>
+                                    <textarea
+                                        id="quickS3ManifestJson"
+                                        rows="6"
+                                        class="textarea text-xs font-mono"
+                                        placeholder='{"uploadId": "...", "partUrls": ["..."]}'
+                                    ></textarea>
+                                </div>
+
                                 <div class="flex justify-end gap-2 pt-2">
                                     <button type="button" id="btn-save-s3-settings" class="btn btn-sm">Verify & Save</button>
                                 </div>

config/asset-version.js

@@ -1 +1 @@
-export const ASSET_VERSION = "2024.11.09";
+export const ASSET_VERSION = "2024.11.09-2";

js/s3-quick.js

@@ -0,0 +1,148 @@
+import { multipartUpload, ensureBucketExists, ensureBucketCors } from "./storage/s3-multipart.js";
+import { makeS3Client } from "./storage/s3-client.js";
+import { userLogger } from "./utils/logger.js";
+
+const STORAGE_KEY = "bitvid:quickS3Settings";
+
+export function loadLocalSettings() {
+  try {
+    const raw = localStorage.getItem(STORAGE_KEY);
+    if (!raw) return null;
+    return JSON.parse(raw);
+  } catch (err) {
+    userLogger.warn("Failed to load quick S3 settings:", err);
+    return null;
+  }
+}
+
+export function saveLocalSettings(settings) {
+  try {
+    localStorage.setItem(STORAGE_KEY, JSON.stringify(settings));
+  } catch (err) {
+    userLogger.warn("Failed to save quick S3 settings:", err);
+  }
+}
+
+export async function uploadWithKeys({
+  file,
+  key,
+  settings, // { endpoint, region, bucket, accessKeyId, secretAccessKey, forcePathStyle, createBucketIfMissing }
+  onProgress
+}) {
+  if (!settings.endpoint || !settings.bucket || !settings.accessKeyId || !settings.secretAccessKey) {
+    throw new Error("Missing S3 credentials.");
+  }
+
+  if (!key) {
+      throw new Error("Target object key is required.");
+  }
+
+  const s3 = makeS3Client({
+    endpoint: settings.endpoint,
+    region: settings.region,
+    credentials: {
+      accessKeyId: settings.accessKeyId,
+      secretAccessKey: settings.secretAccessKey
+    },
+    forcePathStyle: settings.forcePathStyle
+  });
+
+  if (settings.createBucketIfMissing) {
+    await ensureBucketExists({ s3, bucket: settings.bucket, region: settings.region });
+  }
+
+  // Ensure CORS only if we have keys
+  try {
+      const origins = [];
+      if (typeof window !== "undefined" && window.location) {
+          if (window.location.origin && window.location.origin !== "null") {
+              origins.push(window.location.origin);
+          }
+      }
+      // Add standard localdev origins just in case
+      origins.push('http://localhost:8000');
+      origins.push('http://127.0.0.1:8000');
+
+      const uniqueOrigins = [...new Set(origins)];
+
+      await ensureBucketCors({
+          s3,
+          bucket: settings.bucket,
+          origins: uniqueOrigins,
+          region: settings.region
+      });
+  } catch (err) {
+      userLogger.warn("Failed to ensure CORS (ignoring):", err);
+  }
+
+  return multipartUpload({
+    s3,
+    bucket: settings.bucket,
+    key,
+    file,
+    contentType: file.type,
+    onProgress,
+    createBucketIfMissing: false // Already handled
+  });
+}
+
+export async function uploadWithManifest({
+  file,
+  manifest,
+  onProgress
+}) {
+  // manifest: { uploadId, partUrls: string[], completeUrl?: string }
+  if (!manifest || !Array.isArray(manifest.partUrls) || manifest.partUrls.length === 0) {
+    throw new Error("Invalid manifest: partUrls required.");
+  }
+
+  const partUrls = manifest.partUrls;
+  const partCount = partUrls.length;
+  const fileSize = file.size;
+  // Calculate part size. We assume the manifest was generated for this file size.
+  // We divide equally, last part takes remainder.
+  const partSize = Math.ceil(fileSize / partCount);
+
+  const etags = [];
+  let uploadedBytes = 0;
+
+  for (let i = 0; i < partCount; i++) {
+    const start = i * partSize;
+    if (start >= fileSize) break; // Should not happen if calc is right
+    const end = Math.min(start + partSize, fileSize);
+    const chunk = file.slice(start, end);
+    const url = partUrls[i];
+
+    // Simple PUT to presigned URL
+    const response = await fetch(url, {
+      method: 'PUT',
+      body: chunk
+    });
+
+    if (!response.ok) {
+      throw new Error(`Failed to upload part ${i+1}: ${response.statusText}`);
+    }
+
+    const etag = response.headers.get('ETag');
+    // ETag is often needed for completion.
+    etags.push({ PartNumber: i + 1, ETag: etag });
+    uploadedBytes += chunk.size;
+
+    if (onProgress) {
+      onProgress(uploadedBytes / fileSize);
+    }
+  }
+
+  if (manifest.completeUrl) {
+    const compRes = await fetch(manifest.completeUrl, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ Parts: etags })
+    });
+    if (!compRes.ok) {
+        throw new Error(`Failed to complete upload: ${compRes.statusText}`);
+    }
+  }
+
+  return etags;
+}

js/ui/components/UploadModal.js

@@ -17,6 +17,7 @@ import {
   getActiveSigner,
   requestDefaultExtensionPermissions,
 } from "../../nostrClientFacade.js";
+import { uploadWithManifest } from "../../s3-quick.js";
 
 const INFO_HASH_PATTERN = /^[a-f0-9]{40}$/;
 
@@ -195,6 +196,13 @@ export class UploadModal {
         s3: $("#section-storage-s3"),
     };
 
+    this.toggles.s3Manifest = $("#quickS3ManifestToggle");
+    this.inputs.s3ManifestJson = $("#quickS3ManifestJson");
+    this.containers = {
+        s3Credentials: $("#quickS3CredentialsContainer"),
+        s3Manifest: $("#quickS3ManifestContainer"),
+    };
+
     // Inputs (Common)
     this.inputs = {
         title: $("#input-title"),
@@ -297,6 +305,22 @@ export class UploadModal {
         );
     }
 
+    if (this.toggles.s3Manifest) {
+        this.toggles.s3Manifest.addEventListener("change", () => {
+            const checked = this.toggles.s3Manifest.checked;
+            if (this.containers.s3Credentials) {
+                this.containers.s3Credentials.classList.toggle("hidden", checked);
+            }
+            if (this.containers.s3Manifest) {
+                this.containers.s3Manifest.classList.toggle("hidden", !checked);
+            }
+            // Hide save button if manifest
+            if (this.toggles.saveS3Settings) {
+                this.toggles.saveS3Settings.parentElement.classList.toggle("hidden", checked);
+            }
+        });
+    }
+
     // Form Submission
     this.form.addEventListener("submit", (e) => {
         e.preventDefault();
@@ -1283,85 +1307,159 @@ export class UploadModal {
       const file = this.inputs.file?.files?.[0];
       if (!file) throw new Error("Please select a video file to upload.");
 
-      const thumbnailFile = this.inputs.thumbnailFile?.files?.[0];
+      const isManifest = this.toggles.s3Manifest?.checked;
 
-      if (!this.hasValidS3Settings()) throw new Error("Please configure S3 storage credentials.");
-      if (!this.s3Service) throw new Error("S3 upload service is unavailable.");
+      // Manifest Validation
+      let manifest = null;
+      if (isManifest) {
+          const json = this.inputs.s3ManifestJson?.value?.trim();
+          if (!json) throw new Error("Please paste the presigned manifest JSON.");
+          try {
+              manifest = JSON.parse(json);
+          } catch (e) {
+              throw new Error("Invalid Manifest JSON.");
+          }
+      } else {
+          if (!this.hasValidS3Settings()) throw new Error("Please configure S3 storage credentials.");
+          if (!this.s3Service) throw new Error("S3 upload service is unavailable.");
+      }
 
       const pubkey = this.getCurrentPubkey ? this.getCurrentPubkey() : null;
       if (!pubkey) throw new Error("Please login to publish.");
 
       const npub = this.safeEncodeNpub(pubkey);
-      const settings = this.collectS3SettingsForm();
 
-      let videoKey = "";
-      let videoPublicUrl = "";
-      let torrentKey = "";
-      let torrentPublicUrl = "";
+      // If NOT using manifest, we proceed with standard flow (keys)
+      if (!isManifest) {
+          const thumbnailFile = this.inputs.thumbnailFile?.files?.[0];
+          const settings = this.collectS3SettingsForm();
 
-      try {
-          videoKey = buildR2Key(npub, file);
-          videoPublicUrl = buildS3ObjectUrl({
-              publicBaseUrl: settings.publicBaseUrl,
-              endpoint: settings.endpoint,
-              bucket: settings.bucket,
-              key: videoKey,
-              forcePathStyle: settings.forcePathStyle,
+          let videoKey = "";
+          let videoPublicUrl = "";
+          let torrentKey = "";
+          let torrentPublicUrl = "";
+
+          try {
+              videoKey = buildR2Key(npub, file);
+              videoPublicUrl = buildS3ObjectUrl({
+                  publicBaseUrl: settings.publicBaseUrl,
+                  endpoint: settings.endpoint,
+                  bucket: settings.bucket,
+                  key: videoKey,
+                  forcePathStyle: settings.forcePathStyle,
+              });
+              const baseKey = videoKey.replace(/\.[^/.]+$/, "");
+              torrentKey = (baseKey && baseKey !== videoKey) ? `${baseKey}.torrent` : `${videoKey}.torrent`;
+              torrentPublicUrl = buildS3ObjectUrl({
+                  publicBaseUrl: settings.publicBaseUrl,
+                  endpoint: settings.endpoint,
+                  bucket: settings.bucket,
+                  key: torrentKey,
+                  forcePathStyle: settings.forcePathStyle,
+              });
+          } catch (prepErr) {
+              userLogger.warn("Failed to pre-calculate S3 keys:", prepErr);
+          }
+
+          const { infoHash, torrentFile, hasValidInfoHash } = await this.generateTorrentMetadata({
+              file,
+              videoPublicUrl,
           });
-          const baseKey = videoKey.replace(/\.[^/.]+$/, "");
-          torrentKey = (baseKey && baseKey !== videoKey) ? `${baseKey}.torrent` : `${videoKey}.torrent`;
-          torrentPublicUrl = buildS3ObjectUrl({
-              publicBaseUrl: settings.publicBaseUrl,
-              endpoint: settings.endpoint,
-              bucket: settings.bucket,
-              key: torrentKey,
-              forcePathStyle: settings.forcePathStyle,
-          });
-      } catch (prepErr) {
-          userLogger.warn("Failed to pre-calculate S3 keys:", prepErr);
-      }
 
-      const { infoHash, torrentFile, hasValidInfoHash } = await this.generateTorrentMetadata({
-          file,
-          videoPublicUrl,
-      });
-
-      if (!hasValidInfoHash) {
-          const proceed = confirm(
-            "We couldn't calculate a valid info hash. Publishing will continue with URL-first playback only, and WebTorrent fallback will be unavailable. Continue?"
-          );
-          if (!proceed) {
+          if (!hasValidInfoHash) {
+              const proceed = confirm(
+                "We couldn't calculate a valid info hash. Publishing will continue with URL-first playback only, and WebTorrent fallback will be unavailable. Continue?"
+              );
+              if (!proceed) {
+                  this.updateUploadStatus(
+                    "Upload canceled. A valid info hash is required for WebTorrent fallback.",
+                    "warning"
+                  );
+                  this.isUploading = false;
+                  this.submitButton.disabled = false;
+                  this.updateProgress(null);
+                  return;
+                  }
               this.updateUploadStatus(
-                "Upload canceled. A valid info hash is required for WebTorrent fallback.",
+                "Continuing without WebTorrent fallback (info hash unavailable).",
                 "warning"
               );
+          }
+
+          await this.s3Service.uploadVideo({
+              npub,
+              file,
+              thumbnailFile,
+              torrentFile,
+              metadata,
+              infoHash: hasValidInfoHash ? infoHash : "",
+              settings,
+              createBucketIfMissing: settings.createBucketIfMissing,
+              publishVideoNote: this.publishVideoNote,
+              onReset: () => this.resetForm(),
+              forcedVideoKey: videoKey,
+              forcedVideoUrl: videoPublicUrl,
+              forcedTorrentKey: torrentKey,
+              forcedTorrentUrl: torrentPublicUrl,
+          });
+      } else {
+          // Manifest Flow
+          this.isUploading = true;
+          this.submitButton.disabled = true;
+          this.updateUploadStatus("Uploading via manifest...", "info");
+          this.updateProgress(0);
+
+          try {
+              await uploadWithManifest({
+                  file,
+                  manifest,
+                  onProgress: (fraction) => this.updateProgress(fraction),
+              });
+
+              this.updateUploadStatus("Processing metadata...", "info");
+
+              // We can't upload thumbnail/torrent without keys or manifest for them.
+              // We assume user handles that or we proceed with just the video.
+              // We need to publish the video note.
+
+              // Determine URL from manifest or use what we have?
+              // If manifest has `publicUrl`, use it.
+              let finalUrl = manifest.publicUrl || "";
+
+              // Update metadata
+              metadata.url = finalUrl || metadata.url;
+              if (!metadata.url) {
+                   // If we can't determine URL, we can't really publish a useful video note unless we use magnet only?
+                   // Or we warn the user.
+                   // For now, we'll try to publish what we have.
+                   userLogger.warn("Manifest upload complete but public URL is unknown. Video note might be incomplete.");
+              }
+
+              // Normalize & Publish
+              const { payload, errors } = normalizeVideoNotePayload(metadata);
+              if (errors.length) {
+                  throw new Error(getVideoNoteErrorMessage(errors[0]));
+              }
+
+              if (this.publishVideoNote) {
+                  this.updateUploadStatus("Publishing event...", "info");
+                  await this.publishVideoNote(payload);
+                  this.showSuccess("Video uploaded and published!");
+                  this.close();
+                  this.resetForm();
+              } else {
+                  this.showSuccess("Upload complete (publishing skipped).");
+              }
+
+          } catch (err) {
+              userLogger.error("Manifest upload failed:", err);
+              this.showError(err.message || "Upload failed.");
+          } finally {
               this.isUploading = false;
               this.submitButton.disabled = false;
               this.updateProgress(null);
-              return;
           }
-          this.updateUploadStatus(
-            "Continuing without WebTorrent fallback (info hash unavailable).",
-            "warning"
-          );
       }
-
-      await this.s3Service.uploadVideo({
-          npub,
-          file,
-          thumbnailFile,
-          torrentFile,
-          metadata,
-          infoHash: hasValidInfoHash ? infoHash : "",
-          settings,
-          createBucketIfMissing: settings.createBucketIfMissing,
-          publishVideoNote: this.publishVideoNote,
-          onReset: () => this.resetForm(),
-          forcedVideoKey: videoKey,
-          forcedVideoUrl: videoPublicUrl,
-          forcedTorrentKey: torrentKey,
-          forcedTorrentUrl: torrentPublicUrl,
-      });
   }
 
   async generateTorrentMetadata({ file, videoPublicUrl } = {}) {