diff --git a/CHANGELOG.md b/CHANGELOG.md index 678c860..49727f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - **Download limit** - Limit downloads to most recent N items per channel (default: 10) - **Interactive setup script** - Automated installation with user prompts - **Uninstall script** - Easy removal of the application +- **Offline archive site builder** - Generates a static local website from the channel/claim/version folder tree +- **Metadata/image backfill for skipped items** - Existing downloads can fill in missing JSON sidecars and thumbnails without re-downloading the file ### Technical Details - JSON-RPC client for lbrynet daemon @@ -49,6 +51,5 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Under Consideration - GUI interface - Docker containerization -- Automatic thumbnail/metadata download - Video format selection - Bandwidth limiting diff --git a/README.md b/README.md index 400cc96..f9c3441 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,8 @@ For most users: - **Flexible Input**: Accepts Odysee URLs, LBRY URIs, or claim IDs - **Audit Trail**: Maintains run history in JSONL format - **Dry-Run Mode**: Test what would happen without downloading +- **Offline Archive Site**: Builds a static browseable website from the download tree, including channel/claim metadata and local file links +- **Metadata Backfill**: Skipped existing downloads still get missing `channel.json`, `claim.json`, `metadata.json`, and image assets when available ## Installation @@ -116,6 +118,9 @@ general: base_dir: "~/Documents/lbry-downloads" max_workers: 2 dry_run: false + build_offline_site: true + offline_site_dir: "~/Documents/lbry-downloads/site" + fetch_missing_metadata_assets: true include_reposts: false direct_base_urls: - "https://odysee.com" @@ -132,6 +137,8 @@ channels: content_mode: "non_video_only" ``` +When `build_offline_site` is enabled, the downloader writes a generated static site under `offline_site_dir` after each non-dry run. The site reflects the existing archive structure and links back to the real downloaded files under each channel/claim/version folder. + ### Finding Channel URLs **Yes, you can get channel info from Odysee!** Here's how: diff --git a/config.yaml.example b/config.yaml.example index 19fcb8a..8ea4d8f 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -45,6 +45,12 @@ general: # If true, direct-mode runs will try the local lbrynet daemon as a fallback # after repeated rate-limit failures. Requires a working local daemon. direct_auto_fallback_to_p2p: false + # Build a static browseable offline site from the archive after each run. + build_offline_site: true + # Output directory for the generated static site. + offline_site_dir: "~/Documents/lbry-downloads/site" + # Fetch missing channel/claim images and sidecar metadata on reruns. + fetch_missing_metadata_assets: true # Download limit: Maximum number of downloads per channel per run # diff --git a/config_loader.py b/config_loader.py index 0a05568..0976c12 100644 --- a/config_loader.py +++ b/config_loader.py @@ -120,11 +120,27 @@ def _parse_config(raw: Dict[str, Any]) -> Config: "direct_auto_fallback_to_p2p", config.general.direct_auto_fallback_to_p2p, ), + build_offline_site=general_raw.get( + "build_offline_site", config.general.build_offline_site + ), + offline_site_dir=general_raw.get( + "offline_site_dir", config.general.offline_site_dir + ), + fetch_missing_metadata_assets=general_raw.get( + "fetch_missing_metadata_assets", + config.general.fetch_missing_metadata_assets, + ), ) # Validate and expand paths config.general.base_dir = str(expand_path(config.general.base_dir)) config.general.state_file = str(expand_path(config.general.state_file)) + if config.general.offline_site_dir: + config.general.offline_site_dir = str(expand_path(config.general.offline_site_dir)) + else: + config.general.offline_site_dir = str( + expand_path(f"{config.general.base_dir}/site") + ) # Validate max_workers if config.general.max_workers < 1: @@ -271,6 +287,12 @@ general: direct_retry_backoff_seconds: 2.0 # If true, direct-mode runs will try the local node after repeated 429s direct_auto_fallback_to_p2p: false + # Build a static offline browseable site from the archive after each run + build_offline_site: true + # Where to write the generated offline site + offline_site_dir: "~/Documents/lbry-downloads/site" + # Fetch missing channel/claim images and metadata assets on reruns + fetch_missing_metadata_assets: true # Download limit: Number of most recent matching downloads per channel (0 = all) download_limit: 10 @@ -304,3 +326,6 @@ def ensure_directories(config: Config) -> None: # Create channels directory channels_path = base_path / "channels" channels_path.mkdir(exist_ok=True) + + if config.general.build_offline_site: + Path(config.general.offline_site_dir).mkdir(parents=True, exist_ok=True) diff --git a/main.py b/main.py index d700158..f522ad0 100644 --- a/main.py +++ b/main.py @@ -17,10 +17,12 @@ from downloader import DownloadError, Downloader from direct_downloader import DirectDownloadError from lbry_client import LbryClient, LbryClientError, check_daemon_health from models import Channel, Config, DownloadAction +from offline_site import ArchiveSiteManager from planner import Planner from state_db import StateDb from utils import ( create_channel_folder_name, + extract_channel_info, expand_path, format_summary, normalize_odysee_url, @@ -124,6 +126,7 @@ def resolve_channel( folder = create_channel_folder_name(channel_name, channel_claim_id) value = claim.get("value", {}) + channel_info = extract_channel_info(claim) channel = Channel( input=channel_input, @@ -131,9 +134,18 @@ def resolve_channel( channel_claim_id=channel_claim_id, channel_name=channel_name, folder=folder, - display_name=value.get("title"), - permanent_url=claim.get("permanent_url"), - short_url=claim.get("short_url"), + display_name=channel_info.get("title"), + permanent_url=channel_info.get("permanent_url"), + short_url=channel_info.get("short_url"), + canonical_url=channel_info.get("canonical_url"), + description=channel_info.get("description"), + thumbnail_url=channel_info.get("thumbnail_url"), + cover_url=channel_info.get("cover_url"), + website_url=channel_info.get("website_url"), + email=channel_info.get("email"), + tags=channel_info.get("tags", []), + languages=channel_info.get("languages", []), + links=channel_info.get("links", []), ) logger.info(f"Resolved channel: {channel_name} ({channel_claim_id})") @@ -309,6 +321,7 @@ def run_sync( # Initialize planner and downloader base_path = Path(config.general.base_dir) planner = Planner(config, state_db.data, str(base_path)) + site_manager = ArchiveSiteManager(config, base_path) if direct: logger.info("Using DIRECT mode (Odysee proxy for metadata, Odysee CDN for downloads)") @@ -359,6 +372,9 @@ def run_sync( if existing.download_path: channel.download_path = existing.download_path + if not dry_run: + site_manager.ensure_channel_artifacts(channel) + # Enumerate claims claims = enumerate_channel_claims(client, channel, config) @@ -432,9 +448,12 @@ def run_sync( try: success = downloader.download(action, dry_run=dry_run) - if success and not dry_run and action.action not in ("skip_existing",): - # Update state - planner.update_state_from_action(action, success=True) + if success and not dry_run: + site_manager.ensure_claim_artifacts(channel, action) + + if action.action not in ("skip_existing",): + # Update state only after metadata/artifacts are written + planner.update_state_from_action(action, success=True) # Log to history history_file = base_path / "state" / "run-history.jsonl" @@ -522,11 +541,18 @@ def run_sync( channel.last_scan = datetime.now(timezone.utc).isoformat() state_db.set_channel(channel) + if not dry_run: + site_manager.ensure_channel_artifacts(channel) # Save state if not dry_run: state_db.update_last_run() state_db.save() + if config.general.build_offline_site: + try: + site_manager.build_site(channel_index=state_db.data.channels) + except Exception as exc: + logger.exception("Offline site build failed: %s", exc) logger.info("State saved") return stats diff --git a/models.py b/models.py index e3c5675..a084454 100644 --- a/models.py +++ b/models.py @@ -41,6 +41,9 @@ class GeneralConfig: direct_max_retries_per_url: int = 2 direct_retry_backoff_seconds: float = 2.0 direct_auto_fallback_to_p2p: bool = False + build_offline_site: bool = True + offline_site_dir: Optional[str] = None + fetch_missing_metadata_assets: bool = True @dataclass @@ -73,6 +76,15 @@ class Channel: display_name: Optional[str] = None permanent_url: Optional[str] = None short_url: Optional[str] = None + canonical_url: Optional[str] = None + description: Optional[str] = None + thumbnail_url: Optional[str] = None + cover_url: Optional[str] = None + website_url: Optional[str] = None + email: Optional[str] = None + tags: List[str] = field(default_factory=list) + languages: List[str] = field(default_factory=list) + links: List[Dict[str, Any]] = field(default_factory=list) last_scan: Optional[str] = None def to_dict(self) -> Dict[str, Any]: @@ -86,6 +98,15 @@ class Channel: "display_name": self.display_name, "permanent_url": self.permanent_url, "short_url": self.short_url, + "canonical_url": self.canonical_url, + "description": self.description, + "thumbnail_url": self.thumbnail_url, + "cover_url": self.cover_url, + "website_url": self.website_url, + "email": self.email, + "tags": self.tags, + "languages": self.languages, + "links": self.links, "last_scan": self.last_scan, } @@ -101,6 +122,15 @@ class Channel: display_name=data.get("display_name"), permanent_url=data.get("permanent_url"), short_url=data.get("short_url"), + canonical_url=data.get("canonical_url"), + description=data.get("description"), + thumbnail_url=data.get("thumbnail_url"), + cover_url=data.get("cover_url"), + website_url=data.get("website_url"), + email=data.get("email"), + tags=data.get("tags", []), + languages=data.get("languages", []), + links=data.get("links", []), last_scan=data.get("last_scan"), ) diff --git a/offline_site.py b/offline_site.py new file mode 100644 index 0000000..2e3d8e2 --- /dev/null +++ b/offline_site.py @@ -0,0 +1,894 @@ +"""Metadata backfill and offline site generation for the archive.""" + +import html +import json +import logging +import mimetypes +import os +import shutil +import subprocess +import tempfile +from pathlib import Path +from typing import Any, Dict, List, Optional +from urllib.parse import urlparse + +try: + import requests +except ImportError: + requests = None + +from models import Channel, Config, DownloadAction + + +logger = logging.getLogger(__name__) + + +class ArchiveSiteManager: + """Maintains archive metadata sidecars and builds a static offline site.""" + + def __init__(self, config: Config, base_dir: Path): + self.config = config + self.base_dir = base_dir + self.channels_root = self.base_dir / "channels" + self.site_root = Path(config.general.offline_site_dir) + self.session = requests.Session() if requests else None + if self.session: + self.session.headers.update( + { + "User-Agent": ( + "Mozilla/5.0 (X11; Linux x86_64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/123.0.0.0 Safari/537.36" + ) + } + ) + + def ensure_channel_artifacts(self, channel: Channel) -> None: + """Write channel metadata and fetch any missing channel assets.""" + channel_dir = self._channel_dir(channel) + channel_dir.mkdir(parents=True, exist_ok=True) + + channel_payload = { + "channel_claim_id": channel.channel_claim_id, + "channel_name": channel.channel_name, + "display_name": channel.display_name, + "input": channel.input, + "normalized_uri": channel.normalized_uri, + "permanent_url": channel.permanent_url, + "short_url": channel.short_url, + "canonical_url": channel.canonical_url, + "description": channel.description, + "thumbnail_url": channel.thumbnail_url, + "cover_url": channel.cover_url, + "website_url": channel.website_url, + "email": channel.email, + "tags": channel.tags, + "languages": channel.languages, + "links": channel.links, + "folder": channel.folder, + "download_path": channel.download_path, + "last_scan": channel.last_scan, + } + + self._write_json(channel_dir / "channel.json", channel_payload) + + if self.config.general.fetch_missing_metadata_assets: + if channel.thumbnail_url: + self._ensure_remote_asset( + channel.thumbnail_url, + channel_dir, + basename="channel-profile", + ) + if channel.cover_url: + self._ensure_remote_asset( + channel.cover_url, + channel_dir, + basename="channel-cover", + ) + + def ensure_claim_artifacts(self, channel: Channel, action: DownloadAction) -> None: + """Write/update claim- and version-level metadata for a claim.""" + version_dir = Path(action.target_dir) + claim_dir = version_dir.parent.parent + claim_dir.mkdir(parents=True, exist_ok=True) + version_dir.mkdir(parents=True, exist_ok=True) + + file_path = self._resolve_action_file_path(action, version_dir) + file_size = file_path.stat().st_size if file_path and file_path.exists() else None + + claim_payload = { + "claim_id": action.claim_id, + "channel_claim_id": action.channel_claim_id, + "name": action.claim_name, + "title": action.metadata.get("title"), + "permanent_url": action.uri, + "short_url": action.metadata.get("short_url"), + "canonical_url": action.metadata.get("canonical_url"), + "description": action.metadata.get("description"), + "thumbnail_url": action.metadata.get("thumbnail_url"), + "tags": action.metadata.get("tags", []), + "languages": action.metadata.get("languages", []), + "links": action.metadata.get("links", []), + "release_time": action.metadata.get("release_time"), + "timestamp": action.metadata.get("timestamp"), + "stream_type": action.metadata.get("stream_type"), + "source_media_type": action.metadata.get("source_media_type"), + "source_name": action.metadata.get("source_name"), + "claim_folder": self._path_label(claim_dir), + "latest_version_token": action.version_token, + "channel_folder": channel.folder, + } + self._write_json(claim_dir / "claim.json", claim_payload) + + version_payload = { + "claim_id": action.claim_id, + "channel_claim_id": action.channel_claim_id, + "name": action.claim_name, + "title": action.metadata.get("title"), + "description": action.metadata.get("description"), + "version_token": action.version_token, + "permanent_url": action.uri, + "short_url": action.metadata.get("short_url"), + "canonical_url": action.metadata.get("canonical_url"), + "download_source": self._download_source_label(action), + "file_name": file_path.name if file_path else action.metadata.get("file_name"), + "file_size": file_size, + "stream_type": action.metadata.get("stream_type"), + "source_media_type": action.metadata.get("source_media_type"), + "thumbnail_url": action.metadata.get("thumbnail_url"), + "tags": action.metadata.get("tags", []), + "languages": action.metadata.get("languages", []), + "links": action.metadata.get("links", []), + "release_time": action.metadata.get("release_time"), + "timestamp": action.metadata.get("timestamp"), + } + self._write_json(version_dir / "metadata.json", version_payload) + + if action.metadata.get("thumbnail_url") and self.config.general.fetch_missing_metadata_assets: + self._ensure_remote_asset( + action.metadata["thumbnail_url"], + claim_dir, + basename="claim-thumbnail", + ) + + def build_site(self, channel_index: Optional[Dict[str, Channel]] = None) -> None: + """Build a static offline site from the archive metadata.""" + self.site_root.mkdir(parents=True, exist_ok=True) + self._write_text(self.site_root / "assets" / "style.css", self._site_css()) + + channels = self._collect_channel_records(channel_index=channel_index) + self._write_text( + self.site_root / "index.html", + self._render_index_page(channels), + ) + + channels_site_root = self.site_root / "channels" + for channel in channels: + channel_page_dir = channels_site_root / channel["folder"] + channel_page_dir.mkdir(parents=True, exist_ok=True) + self._write_text( + channel_page_dir / "index.html", + self._render_channel_page(channel, channel_page_dir), + ) + + for claim in channel["claims"]: + claim_page_dir = channel_page_dir / "claims" / claim["folder"] + claim_page_dir.mkdir(parents=True, exist_ok=True) + self._write_text( + claim_page_dir / "index.html", + self._render_claim_page(channel, claim, claim_page_dir), + ) + + def _channel_dir(self, channel: Channel) -> Path: + if channel.download_path: + return Path(channel.download_path) / channel.folder + return self.channels_root / channel.folder + + def _resolve_action_file_path( + self, action: DownloadAction, version_dir: Path + ) -> Optional[Path]: + local_file_path = action.metadata.get("local_file_path") + if local_file_path: + path = Path(local_file_path) + if not path.is_absolute(): + path = self.base_dir / path + if path.exists(): + return path + + existing_relpath = action.metadata.get("existing_file_relpath") + if existing_relpath: + path = self.base_dir / existing_relpath + if path.exists(): + return path + + source_name = action.metadata.get("source_name") or action.metadata.get("file_name") + if source_name: + path = version_dir / source_name + if path.exists(): + return path + + candidates = [ + item + for item in version_dir.iterdir() + if item.is_file() and item.name not in {"metadata.json", "download.json", "checksums.txt"} + ] if version_dir.exists() else [] + return candidates[0] if candidates else None + + def _download_source_label(self, action: DownloadAction) -> str: + media_type = (action.metadata.get("source_media_type") or "").lower() + if media_type.startswith("video/") or media_type.startswith("audio/"): + return "odysee_stream" + return "odysee_download" + + def _ensure_remote_asset(self, url: str, target_dir: Path, basename: str) -> Optional[Path]: + if not self.session: + return None + + existing = self._existing_asset(target_dir, basename) + if existing: + return existing[0] + + try: + response = self.session.get(url, timeout=60, stream=True) + response.raise_for_status() + with tempfile.NamedTemporaryFile(dir=target_dir, delete=False) as handle: + shutil.copyfileobj(response.raw, handle) + temp_path = Path(handle.name) + + suffix = self._suffix_for_response(url, response.headers.get("Content-Type"), temp_path) + target_path = target_dir / f"{basename}{suffix}" + temp_path.replace(target_path) + return target_path + except Exception as exc: + logger.warning("Failed to fetch metadata asset %s: %s", url, exc) + temp_path = locals().get("temp_path") + if temp_path and Path(temp_path).exists(): + Path(temp_path).unlink() + return None + + def _suffix_for_url(self, url: str) -> str: + parsed = urlparse(url) + suffix = Path(parsed.path).suffix + if suffix: + return suffix + mime_guess, _ = mimetypes.guess_type(url) + return mimetypes.guess_extension(mime_guess or "") or ".bin" + + def _suffix_for_response( + self, url: str, content_type: Optional[str], downloaded_path: Path + ) -> str: + suffix = self._suffix_for_content_type(content_type) + if suffix: + return suffix + + suffix = self._suffix_for_image_bytes(downloaded_path) + if suffix: + return suffix + + return self._suffix_for_url(url) + + def _suffix_for_content_type(self, content_type: Optional[str]) -> Optional[str]: + if not content_type: + return None + mime = content_type.split(";", 1)[0].strip().lower() + if mime == "image/jpeg": + return ".jpg" + if mime == "image/png": + return ".png" + if mime == "image/webp": + return ".webp" + guessed = mimetypes.guess_extension(mime) + return guessed + + def _suffix_for_image_bytes(self, path: Path) -> Optional[str]: + try: + header = path.read_bytes()[:16] + except OSError: + return None + + if header.startswith(b"\x89PNG\r\n\x1a\n"): + return ".png" + if header.startswith(b"\xff\xd8\xff"): + return ".jpg" + if header.startswith(b"RIFF") and header[8:12] == b"WEBP": + return ".webp" + return None + + def _existing_asset(self, directory: Path, basename: str) -> List[Path]: + normalized = [] + for asset_path in sorted(directory.glob(f"{basename}.*")): + normalized.append(self._normalize_asset_extension(asset_path)) + return normalized + + def _normalize_asset_extension(self, path: Path) -> Path: + actual_suffix = self._suffix_for_image_bytes(path) + if not actual_suffix or path.suffix.lower() == actual_suffix: + return path + + corrected_path = path.with_suffix(actual_suffix) + if corrected_path.exists(): + path.unlink() + return corrected_path + + path.rename(corrected_path) + return corrected_path + + def _collect_channel_records( + self, channel_index: Optional[Dict[str, Channel]] = None + ) -> List[Dict[str, Any]]: + channel_dirs: Dict[Path, Path] = {} + if self.channels_root.exists(): + for channel_dir in self.channels_root.iterdir(): + if channel_dir.is_dir(): + channel_dirs[channel_dir.resolve()] = channel_dir + + for channel in (channel_index or {}).values(): + channel_dir = self._channel_dir(channel) + if channel_dir.exists(): + channel_dirs[channel_dir.resolve()] = channel_dir + + channels = [] + for channel_dir in sorted(channel_dirs.values()): + + channel_data = self._read_json(channel_dir / "channel.json") or { + "folder": channel_dir.name, + "channel_name": channel_dir.name, + "display_name": channel_dir.name, + "description": None, + "links": [], + } + channel_data["folder"] = channel_dir.name + channel_data["path"] = channel_dir + channel_data["profile_asset"] = self._find_asset(channel_dir, "channel-profile") + channel_data["cover_asset"] = self._find_asset(channel_dir, "channel-cover") + claims = self._collect_claim_records(channel_dir) + channel_data["claims"] = claims + channel_data["claim_count"] = len(claims) + channel_data["version_count"] = sum(len(claim["versions"]) for claim in claims) + channels.append(channel_data) + + channels.sort( + key=lambda item: (item.get("display_name") or item.get("channel_name") or "").lower() + ) + channels.sort(key=lambda item: item.get("last_scan") or "", reverse=True) + return channels + + def _collect_claim_records(self, channel_dir: Path) -> List[Dict[str, Any]]: + claims_root = channel_dir / "claims" + claims = [] + for claim_dir in sorted(claims_root.iterdir()) if claims_root.exists() else []: + if not claim_dir.is_dir(): + continue + + claim_data = self._read_json(claim_dir / "claim.json") or { + "name": claim_dir.name, + "title": claim_dir.name, + "description": None, + "links": [], + } + versions = self._collect_versions(claim_dir) + latest_version = versions[0] if versions else {} + claim_data["folder"] = claim_dir.name + claim_data["path"] = claim_dir + claim_data["thumbnail_asset"] = self._find_asset(claim_dir, "claim-thumbnail") + if not claim_data["thumbnail_asset"]: + claim_data["thumbnail_asset"] = self._extract_video_thumbnail(claim_dir, versions) + claim_data["thumbnail_url"] = claim_data.get("thumbnail_url") or latest_version.get( + "thumbnail_url" + ) + claim_data["title"] = claim_data.get("title") or latest_version.get("title") + claim_data["description"] = claim_data.get("description") or latest_version.get( + "description" + ) + claim_data["release_time"] = claim_data.get("release_time") or latest_version.get( + "release_time" + ) + claim_data["versions"] = versions + claims.append(claim_data) + + claims.sort( + key=lambda item: ( + -self._numeric_sort_value(item.get("release_time")), + (item.get("title") or item.get("name") or "").lower(), + ) + ) + return claims + + def _collect_versions(self, claim_dir: Path) -> List[Dict[str, Any]]: + versions_root = claim_dir / "versions" + versions = [] + for version_dir in sorted(versions_root.iterdir()) if versions_root.exists() else []: + if not version_dir.is_dir(): + continue + + version_data = self._read_json(version_dir / "metadata.json") or { + "version_token": version_dir.name + } + version_data["path"] = version_dir + version_data["version_token"] = version_data.get("version_token") or version_dir.name + file_path = self._find_primary_file(version_dir) + version_data["file_path"] = file_path + versions.append(version_data) + + versions.sort( + key=lambda item: ( + -self._numeric_sort_value(item.get("release_time")), + item["version_token"], + ) + ) + return versions + + def _extract_video_thumbnail( + self, claim_dir: Path, versions: List[Dict[str, Any]] + ) -> Optional[Path]: + for version in versions: + file_path = version.get("file_path") + if not file_path or not self._is_video_version(version, file_path): + continue + + target_path = claim_dir / "claim-thumbnail.jpg" + if self._run_ffmpeg_thumbnail(file_path, target_path): + return target_path + + return None + + def _is_video_version(self, version: Dict[str, Any], file_path: Path) -> bool: + media_type = self._version_media_type(version, file_path).lower() + return media_type.startswith("video/") + + def _run_ffmpeg_thumbnail(self, input_path: Path, output_path: Path) -> bool: + cmd = [ + "ffmpeg", + "-y", + "-ss", + "1", + "-i", + str(input_path), + "-frames:v", + "1", + "-vf", + "scale='min(960,iw)':-2", + str(output_path), + ] + try: + subprocess.run( + cmd, + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + return output_path.exists() and output_path.stat().st_size > 0 + except (OSError, subprocess.CalledProcessError): + if output_path.exists(): + output_path.unlink() + return False + + def _find_primary_file(self, version_dir: Path) -> Optional[Path]: + metadata_name = {"metadata.json", "download.json", "checksums.txt"} + files = [item for item in version_dir.iterdir() if item.is_file() and item.name not in metadata_name] + return files[0] if files else None + + def _find_asset(self, directory: Path, basename: str) -> Optional[Path]: + matches = self._existing_asset(directory, basename) + return matches[0] if matches else None + + def _numeric_sort_value(self, value: Any) -> int: + """Convert mixed metadata values into a safe descending sort key.""" + if value is None or value == "": + return 0 + if isinstance(value, bool): + return int(value) + if isinstance(value, (int, float)): + return int(value) + if isinstance(value, str): + stripped = value.strip() + if not stripped: + return 0 + try: + return int(stripped) + except ValueError: + try: + return int(float(stripped)) + except ValueError: + logger.debug("Non-numeric release_time value in archive metadata: %r", value) + return 0 + return 0 + + def _render_index_page(self, channels: List[Dict[str, Any]]) -> str: + cards = [] + for channel in channels: + title = channel.get("display_name") or channel.get("channel_name") or channel["folder"] + description = channel.get("description") or "" + profile = self._image_tag( + channel.get("profile_asset"), + self.site_root, + css_class="avatar", + mirror_key=f"channels/{channel['folder']}/channel-profile", + ) + link = f"channels/{channel['folder']}/index.html" + cards.append( + f""" +
+ + {profile} +
+

{html.escape(title)}

+

{html.escape(channel.get('channel_name') or '')}

+

{html.escape(self._truncate(description, 220))}

+

{channel['claim_count']} claims | {channel['version_count']} versions

+
+ +
+ """ + ) + + body = "\n".join(cards) or "

No channels found.

" + return self._page( + title="LBRY Archive", + body=f""" +
+
+

Offline Archive

+

LBRY / Odysee Archive

+

Browse downloaded channels, claims, metadata, and local files without needing the live site.

+
+
+
{body}
+ """, + root=self.site_root, + ) + + def _render_channel_page(self, channel: Dict[str, Any], page_dir: Path) -> str: + title = channel.get("display_name") or channel.get("channel_name") or channel["folder"] + cover = self._image_tag( + channel.get("cover_asset"), + page_dir, + css_class="cover", + mirror_key=f"channels/{channel['folder']}/channel-cover", + ) + profile = self._image_tag( + channel.get("profile_asset"), + page_dir, + css_class="avatar large", + mirror_key=f"channels/{channel['folder']}/channel-profile", + ) + claims_html = [] + for claim in channel["claims"]: + claim_title = claim.get("title") or claim.get("name") or claim["folder"] + thumb = self._image_tag( + claim.get("thumbnail_asset"), + page_dir, + css_class="thumb", + fallback_url=claim.get("thumbnail_url"), + mirror_key=f"channels/{channel['folder']}/claims/{claim['folder']}/claim-thumbnail", + ) + link = f"claims/{claim['folder']}/index.html" + claims_html.append( + f""" +
+ + {thumb} +
+

{html.escape(claim_title)}

+

{html.escape(self._truncate(claim.get('description') or '', 180))}

+

{len(claim['versions'])} version(s)

+
+ +
+ """ + ) + links = self._render_links(channel.get("links", [])) + return self._page( + title=title, + body=f""" + + {cover} +
+ {profile} +
+

Channel

+

{html.escape(title)}

+

{html.escape(channel.get('channel_name') or '')}

+

{html.escape(channel.get('description') or '')}

+ {links} +
+
+
{''.join(claims_html) or '

No claims found.

'}
+ """, + root=page_dir, + ) + + def _render_claim_page(self, channel: Dict[str, Any], claim: Dict[str, Any], page_dir: Path) -> str: + title = claim.get("title") or claim.get("name") or claim["folder"] + thumb = self._image_tag( + claim.get("thumbnail_asset"), + page_dir, + css_class="hero-thumb", + fallback_url=claim.get("thumbnail_url"), + mirror_key=f"channels/{channel['folder']}/claims/{claim['folder']}/claim-thumbnail", + ) + versions = [] + for version in claim["versions"]: + file_link = "" + media_preview = "" + if version.get("file_path"): + href = self._rel_href(page_dir, version["file_path"]) + file_name = version["file_path"].name + file_link = f'

Open local file: {html.escape(file_name)}

' + media_preview = self._render_version_preview(version, page_dir, href) + versions.append( + f""" +
+

{html.escape(version.get('version_token') or 'version')}

+

{html.escape(version.get('source_media_type') or version.get('stream_type') or '')}

+

{html.escape(version.get('description') or '')}

+ {media_preview} + {file_link} +
+ """ + ) + links = self._render_links(claim.get("links", [])) + return self._page( + title=title, + body=f""" + +
+ {thumb} +
+

Claim

+

{html.escape(title)}

+

{html.escape(claim.get('description') or '')}

+ {links} +
+
+
{''.join(versions) or '

No versions found.

'}
+ """, + root=page_dir, + ) + + def _image_tag( + self, + image_path: Optional[Path], + page_dir: Path, + css_class: str, + fallback_url: Optional[str] = None, + mirror_key: Optional[str] = None, + ) -> str: + if image_path: + site_path = self._mirror_site_asset(image_path, mirror_key) + href = self._rel_href(page_dir, site_path) + elif fallback_url: + href = fallback_url + else: + return "" + return f'' + + def _render_version_preview(self, version: Dict[str, Any], page_dir: Path, href: str) -> str: + file_path = version.get("file_path") + if not file_path: + return "" + + media_type = self._version_media_type(version, file_path) + if media_type.startswith("video/"): + poster = self._poster_attribute(version, page_dir) + return ( + f'" + ) + + if media_type.startswith("audio/"): + return ( + f'" + ) + + return "" + + def _version_media_type(self, version: Dict[str, Any], file_path: Path) -> str: + media_type = version.get("source_media_type") + if media_type: + return str(media_type) + guessed, _ = mimetypes.guess_type(file_path.name) + return guessed or "" + + def _poster_attribute(self, version: Dict[str, Any], page_dir: Path) -> str: + claim_dir = Path(version["path"]).parent.parent + thumbnail_asset = self._find_asset(claim_dir, "claim-thumbnail") + if thumbnail_asset: + claim_folder = claim_dir.name + channel_folder = claim_dir.parent.parent.name + site_path = self._mirror_site_asset( + thumbnail_asset, + f"channels/{channel_folder}/claims/{claim_folder}/claim-thumbnail", + ) + return f' poster="{html.escape(self._rel_href(page_dir, site_path))}"' + thumbnail_url = version.get("thumbnail_url") + if thumbnail_url: + return f' poster="{html.escape(thumbnail_url)}"' + return "" + + def _mirror_site_asset(self, source_path: Path, mirror_key: Optional[str] = None) -> Path: + source = Path(source_path) + if mirror_key: + target = self.site_root / "_assets" / f"{mirror_key}{source.suffix.lower()}" + else: + target = self.site_root / "_assets" / source.name + target.parent.mkdir(parents=True, exist_ok=True) + if not target.exists(): + shutil.copy2(source, target) + return target + + def _render_links(self, links: List[Dict[str, str]]) -> str: + if not links: + return "" + rendered = [] + for link in links: + label = link.get("label") or link.get("url") or "Link" + url = link.get("url") + if not url: + continue + rendered.append( + f'{html.escape(label)}' + ) + if not rendered: + return "" + return f'
{"".join(rendered)}
' + + def _page(self, *, title: str, body: str, root: Path) -> str: + stylesheet = self._rel_href(root, self.site_root / "assets" / "style.css") + return f""" + + + + + {html.escape(title)} + + + +
+ {body} +
+ + +""" + + def _rel_href(self, page_dir: Path, target: Path) -> str: + return os.path.relpath(Path(target).resolve(), start=Path(page_dir).resolve()) + + def _path_label(self, path: Path) -> str: + try: + return str(path.relative_to(self.base_dir)) + except ValueError: + return str(path) + + def _write_json(self, path: Path, payload: Dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "w", encoding="utf-8") as handle: + json.dump(payload, handle, indent=2, sort_keys=True) + + def _read_json(self, path: Path) -> Optional[Dict[str, Any]]: + if not path.exists(): + return None + with open(path, "r", encoding="utf-8") as handle: + return json.load(handle) + + def _write_text(self, path: Path, content: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "w", encoding="utf-8") as handle: + handle.write(content) + + def _truncate(self, text: str, limit: int) -> str: + if len(text) <= limit: + return text + return text[: limit - 3].rstrip() + "..." + + def _site_css(self) -> str: + return """ +:root { + --bg: #f5f1e8; + --ink: #1d1b19; + --muted: #6b6257; + --card: #fffaf1; + --line: #d9cfbf; + --accent: #9c4a1a; +} +* { box-sizing: border-box; } +body { + margin: 0; + background: + radial-gradient(circle at top, rgba(156,74,26,.14), transparent 35%), + linear-gradient(180deg, #f8f4ec 0%, var(--bg) 100%); + color: var(--ink); + font: 16px/1.5 Georgia, "Times New Roman", serif; +} +a { color: var(--accent); text-decoration: none; } +a:hover { text-decoration: underline; } +.shell { max-width: 1180px; margin: 0 auto; padding: 2rem 1.25rem 4rem; } +.hero, .channel-header, .claim-header { + display: grid; + gap: 1rem; + align-items: start; + margin-bottom: 2rem; +} +.hero { + min-height: 12rem; + border: 1px solid var(--line); + background: rgba(255,250,241,.8); + padding: 1.5rem; +} +.eyebrow, .meta, .muted, .breadcrumbs { color: var(--muted); } +.breadcrumbs { margin-bottom: 1rem; display: flex; gap: .75rem; flex-wrap: wrap; } +.grid { + display: grid; + gap: 1rem; + grid-template-columns: repeat(auto-fit, minmax(260px, 1fr)); +} +.card, .version { + border: 1px solid var(--line); + background: var(--card); +} +.card-link { display: block; color: inherit; height: 100%; } +.card-body { padding: 1rem; } +.avatar { + width: 72px; + height: 72px; + object-fit: cover; + border-radius: 999px; + border: 1px solid var(--line); + margin: 1rem 1rem 0; +} +.avatar.large { width: 112px; height: 112px; margin: 0; } +.cover { + width: 100%; + max-height: 240px; + object-fit: cover; + border: 1px solid var(--line); + margin-bottom: 1rem; +} +.thumb, .hero-thumb { + width: 100%; + aspect-ratio: 16 / 9; + object-fit: cover; + border-bottom: 1px solid var(--line); +} +.hero-thumb { max-width: 420px; border: 1px solid var(--line); } +.pill-row { + display: flex; + gap: .5rem; + flex-wrap: wrap; + margin-top: .75rem; +} +.pill { + display: inline-block; + border: 1px solid var(--line); + padding: .35rem .65rem; + border-radius: 999px; + background: #fff; +} +.versions { + display: grid; + gap: 1rem; +} +.version { padding: 1rem; } +.media-player, .audio-player { + width: 100%; + margin: .75rem 0; +} +.media-player { + max-width: 100%; + aspect-ratio: 16 / 9; + background: #000; +} +@media (min-width: 860px) { + .channel-header, .claim-header { + grid-template-columns: auto 1fr; + } +} +""" diff --git a/pyproject.toml b/pyproject.toml index 45a24a7..0b43fb7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,4 +55,5 @@ py-modules = [ "state_db", "planner", "downloader", + "offline_site", ] diff --git a/tests/test_config.py b/tests/test_config.py index d40324a..f96fbd3 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -149,6 +149,23 @@ channels: assert config.general.direct_retry_backoff_seconds == 1.5 assert config.general.direct_auto_fallback_to_p2p is True + def test_offline_site_defaults_to_base_dir_site(self, tmp_path): + config_file = tmp_path / "config.yaml" + config_file.write_text(""" +general: + base_dir: "~/test-archive" + +channels: + - input: "https://odysee.com/@Test:1" + enabled: true +""") + + config = load_config(str(config_file)) + + assert config.general.build_offline_site is True + assert config.general.fetch_missing_metadata_assets is True + assert config.general.offline_site_dir.endswith("/test-archive/site") + def test_invalid_direct_retry_backoff(self, tmp_path): config_file = tmp_path / "config.yaml" config_file.write_text(""" diff --git a/tests/test_offline_site.py b/tests/test_offline_site.py new file mode 100644 index 0000000..e0d3ea6 --- /dev/null +++ b/tests/test_offline_site.py @@ -0,0 +1,327 @@ +"""Tests for offline site generation and metadata backfill.""" + +import json + +from models import Channel, Config, DownloadAction +from offline_site import ArchiveSiteManager + + +WEBP_STUB = b"RIFF\x1a\x00\x00\x00WEBPVP8 " + b"\x00" * 10 + + +def test_offline_site_builds_from_archive_metadata(tmp_path): + config = Config() + config.general.base_dir = str(tmp_path) + config.general.offline_site_dir = str(tmp_path / "site") + config.general.fetch_missing_metadata_assets = False + + manager = ArchiveSiteManager(config, tmp_path) + + channel = Channel( + input="https://odysee.com/@example:1", + normalized_uri="lbry://@example#1", + channel_claim_id="chan123", + channel_name="@example", + folder="example__chan123", + display_name="Example Channel", + description="Channel description", + links=[{"label": "Website", "url": "https://example.com"}], + ) + manager.ensure_channel_artifacts(channel) + + version_dir = ( + tmp_path + / "channels" + / channel.folder + / "claims" + / "my-claim__claim123" + / "versions" + / "sd_abc123" + ) + version_dir.mkdir(parents=True) + file_path = version_dir / "video.mp4" + file_path.write_bytes(b"video-data") + thumbnail_path = version_dir.parent.parent / "claim-thumbnail.jpg" + thumbnail_path.write_bytes(b"thumb") + + action = DownloadAction( + claim_id="claim123", + claim_name="my-claim", + channel_claim_id=channel.channel_claim_id, + version_token="sd_abc123", + action="skip_existing", + uri="lbry://@example#1/my-claim#claim123", + target_dir=str(version_dir), + metadata={ + "title": "My Claim", + "description": "Claim description", + "source_media_type": "video/mp4", + "stream_type": "video", + "source_name": "video.mp4", + "local_file_path": str(file_path.relative_to(tmp_path)), + "thumbnail_url": "https://example.com/thumb.jpg", + "release_time": 1234567890, + "links": [ + { + "label": "External", + "url": "https://odysee.com/@example:1/my-claim", + } + ], + }, + ) + manager.ensure_claim_artifacts(channel, action) + manager.build_site(channel_index={channel.channel_claim_id: channel}) + + channel_json = tmp_path / "channels" / channel.folder / "channel.json" + claim_json = version_dir.parent.parent / "claim.json" + metadata_json = version_dir / "metadata.json" + site_index = tmp_path / "site" / "index.html" + channel_page = tmp_path / "site" / "channels" / channel.folder / "index.html" + claim_page = ( + tmp_path + / "site" + / "channels" + / channel.folder + / "claims" + / "my-claim__claim123" + / "index.html" + ) + + assert channel_json.exists() + assert claim_json.exists() + assert metadata_json.exists() + assert site_index.exists() + assert channel_page.exists() + assert claim_page.exists() + + channel_payload = json.loads(channel_json.read_text()) + claim_payload = json.loads(claim_json.read_text()) + version_payload = json.loads(metadata_json.read_text()) + + assert channel_payload["display_name"] == "Example Channel" + assert claim_payload["title"] == "My Claim" + assert version_payload["file_name"] == "video.mp4" + + assert "Example Channel" in site_index.read_text() + channel_page_html = channel_page.read_text() + claim_page_html = claim_page.read_text() + + assert "My Claim" in channel_page_html + assert 'class="thumb"' in channel_page_html + assert "Open local file: video.mp4" in claim_page_html + assert '