{html.escape(title)}
{html.escape(channel.get('channel_name') or '')}
{html.escape(self._truncate(description, 220))}
"""Metadata backfill and offline site generation for the archive."""
import html
import json
import logging
import mimetypes
import os
import shutil
import subprocess
import tempfile
from pathlib import Path
from typing import Any, Dict, List, Optional
from urllib.parse import urlparse
try:
import requests
except ImportError:
requests = None
from models import Channel, Config, DownloadAction
logger = logging.getLogger(__name__)
class ArchiveSiteManager:
"""Maintains archive metadata sidecars and builds a static offline site."""
def __init__(self, config: Config, base_dir: Path):
self.config = config
self.base_dir = base_dir
self.channels_root = self.base_dir / "channels"
self.site_root = Path(config.general.offline_site_dir)
self.session = requests.Session() if requests else None
if self.session:
self.session.headers.update(
{
"User-Agent": (
"Mozilla/5.0 (X11; Linux x86_64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/123.0.0.0 Safari/537.36"
)
}
)
def ensure_channel_artifacts(self, channel: Channel) -> None:
"""Write channel metadata and fetch any missing channel assets."""
channel_dir = self._channel_dir(channel)
channel_dir.mkdir(parents=True, exist_ok=True)
channel_payload = {
"channel_claim_id": channel.channel_claim_id,
"channel_name": channel.channel_name,
"display_name": channel.display_name,
"input": channel.input,
"normalized_uri": channel.normalized_uri,
"permanent_url": channel.permanent_url,
"short_url": channel.short_url,
"canonical_url": channel.canonical_url,
"description": channel.description,
"thumbnail_url": channel.thumbnail_url,
"cover_url": channel.cover_url,
"website_url": channel.website_url,
"email": channel.email,
"tags": channel.tags,
"languages": channel.languages,
"links": channel.links,
"folder": channel.folder,
"download_path": channel.download_path,
"last_scan": channel.last_scan,
}
self._write_json(channel_dir / "channel.json", channel_payload)
if self.config.general.fetch_missing_metadata_assets:
if channel.thumbnail_url:
self._ensure_remote_asset(
channel.thumbnail_url,
channel_dir,
basename="channel-profile",
)
if channel.cover_url:
self._ensure_remote_asset(
channel.cover_url,
channel_dir,
basename="channel-cover",
)
def ensure_claim_artifacts(self, channel: Channel, action: DownloadAction) -> None:
"""Write/update claim- and version-level metadata for a claim."""
version_dir = Path(action.target_dir)
claim_dir = version_dir.parent.parent
claim_dir.mkdir(parents=True, exist_ok=True)
version_dir.mkdir(parents=True, exist_ok=True)
file_path = self._resolve_action_file_path(action, version_dir)
file_size = file_path.stat().st_size if file_path and file_path.exists() else None
claim_payload = {
"claim_id": action.claim_id,
"channel_claim_id": action.channel_claim_id,
"name": action.claim_name,
"title": action.metadata.get("title"),
"permanent_url": action.uri,
"short_url": action.metadata.get("short_url"),
"canonical_url": action.metadata.get("canonical_url"),
"description": action.metadata.get("description"),
"thumbnail_url": action.metadata.get("thumbnail_url"),
"tags": action.metadata.get("tags", []),
"languages": action.metadata.get("languages", []),
"links": action.metadata.get("links", []),
"release_time": action.metadata.get("release_time"),
"timestamp": action.metadata.get("timestamp"),
"stream_type": action.metadata.get("stream_type"),
"source_media_type": action.metadata.get("source_media_type"),
"source_name": action.metadata.get("source_name"),
"claim_folder": self._path_label(claim_dir),
"latest_version_token": action.version_token,
"channel_folder": channel.folder,
}
self._write_json(claim_dir / "claim.json", claim_payload)
version_payload = {
"claim_id": action.claim_id,
"channel_claim_id": action.channel_claim_id,
"name": action.claim_name,
"title": action.metadata.get("title"),
"description": action.metadata.get("description"),
"version_token": action.version_token,
"permanent_url": action.uri,
"short_url": action.metadata.get("short_url"),
"canonical_url": action.metadata.get("canonical_url"),
"download_source": self._download_source_label(action),
"file_name": file_path.name if file_path else action.metadata.get("file_name"),
"file_size": file_size,
"stream_type": action.metadata.get("stream_type"),
"source_media_type": action.metadata.get("source_media_type"),
"thumbnail_url": action.metadata.get("thumbnail_url"),
"tags": action.metadata.get("tags", []),
"languages": action.metadata.get("languages", []),
"links": action.metadata.get("links", []),
"release_time": action.metadata.get("release_time"),
"timestamp": action.metadata.get("timestamp"),
}
self._write_json(version_dir / "metadata.json", version_payload)
if action.metadata.get("thumbnail_url") and self.config.general.fetch_missing_metadata_assets:
self._ensure_remote_asset(
action.metadata["thumbnail_url"],
claim_dir,
basename="claim-thumbnail",
)
def build_site(self, channel_index: Optional[Dict[str, Channel]] = None) -> None:
"""Build a static offline site from the archive metadata."""
self.site_root.mkdir(parents=True, exist_ok=True)
self._write_text(self.site_root / "assets" / "style.css", self._site_css())
channels = self._collect_channel_records(channel_index=channel_index)
self._write_text(
self.site_root / "index.html",
self._render_index_page(channels),
)
channels_site_root = self.site_root / "channels"
for channel in channels:
channel_page_dir = channels_site_root / channel["folder"]
channel_page_dir.mkdir(parents=True, exist_ok=True)
self._write_text(
channel_page_dir / "index.html",
self._render_channel_page(channel, channel_page_dir),
)
for claim in channel["claims"]:
claim_page_dir = channel_page_dir / "claims" / claim["folder"]
claim_page_dir.mkdir(parents=True, exist_ok=True)
self._write_text(
claim_page_dir / "index.html",
self._render_claim_page(channel, claim, claim_page_dir),
)
def _channel_dir(self, channel: Channel) -> Path:
if channel.download_path:
return Path(channel.download_path) / channel.folder
return self.channels_root / channel.folder
def _resolve_action_file_path(
self, action: DownloadAction, version_dir: Path
) -> Optional[Path]:
local_file_path = action.metadata.get("local_file_path")
if local_file_path:
path = Path(local_file_path)
if not path.is_absolute():
path = self.base_dir / path
if path.exists():
return path
existing_relpath = action.metadata.get("existing_file_relpath")
if existing_relpath:
path = self.base_dir / existing_relpath
if path.exists():
return path
source_name = action.metadata.get("source_name") or action.metadata.get("file_name")
if source_name:
path = version_dir / source_name
if path.exists():
return path
candidates = [
item
for item in version_dir.iterdir()
if item.is_file() and item.name not in {"metadata.json", "download.json", "checksums.txt"}
] if version_dir.exists() else []
return candidates[0] if candidates else None
def _download_source_label(self, action: DownloadAction) -> str:
media_type = (action.metadata.get("source_media_type") or "").lower()
if media_type.startswith("video/") or media_type.startswith("audio/"):
return "odysee_stream"
return "odysee_download"
def _ensure_remote_asset(self, url: str, target_dir: Path, basename: str) -> Optional[Path]:
if not self.session:
return None
existing = self._existing_asset(target_dir, basename)
if existing:
return existing[0]
try:
response = self.session.get(url, timeout=60, stream=True)
response.raise_for_status()
with tempfile.NamedTemporaryFile(dir=target_dir, delete=False) as handle:
shutil.copyfileobj(response.raw, handle)
temp_path = Path(handle.name)
suffix = self._suffix_for_response(url, response.headers.get("Content-Type"), temp_path)
target_path = target_dir / f"{basename}{suffix}"
temp_path.replace(target_path)
return target_path
except Exception as exc:
logger.warning("Failed to fetch metadata asset %s: %s", url, exc)
temp_path = locals().get("temp_path")
if temp_path and Path(temp_path).exists():
Path(temp_path).unlink()
return None
def _suffix_for_url(self, url: str) -> str:
parsed = urlparse(url)
suffix = Path(parsed.path).suffix
if suffix:
return suffix
mime_guess, _ = mimetypes.guess_type(url)
return mimetypes.guess_extension(mime_guess or "") or ".bin"
def _suffix_for_response(
self, url: str, content_type: Optional[str], downloaded_path: Path
) -> str:
suffix = self._suffix_for_content_type(content_type)
if suffix:
return suffix
suffix = self._suffix_for_image_bytes(downloaded_path)
if suffix:
return suffix
return self._suffix_for_url(url)
def _suffix_for_content_type(self, content_type: Optional[str]) -> Optional[str]:
if not content_type:
return None
mime = content_type.split(";", 1)[0].strip().lower()
if mime == "image/jpeg":
return ".jpg"
if mime == "image/png":
return ".png"
if mime == "image/webp":
return ".webp"
guessed = mimetypes.guess_extension(mime)
return guessed
def _suffix_for_image_bytes(self, path: Path) -> Optional[str]:
try:
header = path.read_bytes()[:16]
except OSError:
return None
if header.startswith(b"\x89PNG\r\n\x1a\n"):
return ".png"
if header.startswith(b"\xff\xd8\xff"):
return ".jpg"
if header.startswith(b"RIFF") and header[8:12] == b"WEBP":
return ".webp"
return None
def _existing_asset(self, directory: Path, basename: str) -> List[Path]:
normalized = []
for asset_path in sorted(directory.glob(f"{basename}.*")):
normalized.append(self._normalize_asset_extension(asset_path))
return normalized
def _normalize_asset_extension(self, path: Path) -> Path:
actual_suffix = self._suffix_for_image_bytes(path)
if not actual_suffix or path.suffix.lower() == actual_suffix:
return path
corrected_path = path.with_suffix(actual_suffix)
if corrected_path.exists():
path.unlink()
return corrected_path
path.rename(corrected_path)
return corrected_path
def _collect_channel_records(
self, channel_index: Optional[Dict[str, Channel]] = None
) -> List[Dict[str, Any]]:
channel_dirs: Dict[Path, Path] = {}
if self.channels_root.exists():
for channel_dir in self.channels_root.iterdir():
if channel_dir.is_dir():
channel_dirs[channel_dir.resolve()] = channel_dir
for channel in (channel_index or {}).values():
channel_dir = self._channel_dir(channel)
if channel_dir.exists():
channel_dirs[channel_dir.resolve()] = channel_dir
channels = []
for channel_dir in sorted(channel_dirs.values()):
channel_data = self._read_json(channel_dir / "channel.json") or {
"folder": channel_dir.name,
"channel_name": channel_dir.name,
"display_name": channel_dir.name,
"description": None,
"links": [],
}
channel_data["folder"] = channel_dir.name
channel_data["path"] = channel_dir
channel_data["profile_asset"] = self._find_asset(channel_dir, "channel-profile")
channel_data["cover_asset"] = self._find_asset(channel_dir, "channel-cover")
claims = self._collect_claim_records(channel_dir)
channel_data["claims"] = claims
channel_data["claim_count"] = len(claims)
channel_data["version_count"] = sum(len(claim["versions"]) for claim in claims)
channels.append(channel_data)
channels.sort(
key=lambda item: (item.get("display_name") or item.get("channel_name") or "").lower()
)
channels.sort(key=lambda item: item.get("last_scan") or "", reverse=True)
return channels
def _collect_claim_records(self, channel_dir: Path) -> List[Dict[str, Any]]:
claims_root = channel_dir / "claims"
claims = []
for claim_dir in sorted(claims_root.iterdir()) if claims_root.exists() else []:
if not claim_dir.is_dir():
continue
claim_data = self._read_json(claim_dir / "claim.json") or {
"name": claim_dir.name,
"title": claim_dir.name,
"description": None,
"links": [],
}
versions = self._collect_versions(claim_dir)
latest_version = versions[0] if versions else {}
claim_data["folder"] = claim_dir.name
claim_data["path"] = claim_dir
claim_data["thumbnail_asset"] = self._find_asset(claim_dir, "claim-thumbnail")
if not claim_data["thumbnail_asset"]:
claim_data["thumbnail_asset"] = self._extract_video_thumbnail(claim_dir, versions)
claim_data["thumbnail_url"] = claim_data.get("thumbnail_url") or latest_version.get(
"thumbnail_url"
)
claim_data["title"] = claim_data.get("title") or latest_version.get("title")
claim_data["description"] = claim_data.get("description") or latest_version.get(
"description"
)
claim_data["release_time"] = claim_data.get("release_time") or latest_version.get(
"release_time"
)
claim_data["versions"] = versions
claims.append(claim_data)
claims.sort(
key=lambda item: (
-self._numeric_sort_value(item.get("release_time")),
(item.get("title") or item.get("name") or "").lower(),
)
)
return claims
def _collect_versions(self, claim_dir: Path) -> List[Dict[str, Any]]:
versions_root = claim_dir / "versions"
versions = []
for version_dir in sorted(versions_root.iterdir()) if versions_root.exists() else []:
if not version_dir.is_dir():
continue
version_data = self._read_json(version_dir / "metadata.json") or {
"version_token": version_dir.name
}
version_data["path"] = version_dir
version_data["version_token"] = version_data.get("version_token") or version_dir.name
file_path = self._find_primary_file(version_dir)
version_data["file_path"] = file_path
versions.append(version_data)
versions.sort(
key=lambda item: (
-self._numeric_sort_value(item.get("release_time")),
item["version_token"],
)
)
return versions
def _extract_video_thumbnail(
self, claim_dir: Path, versions: List[Dict[str, Any]]
) -> Optional[Path]:
for version in versions:
file_path = version.get("file_path")
if not file_path or not self._is_video_version(version, file_path):
continue
target_path = claim_dir / "claim-thumbnail.jpg"
if self._run_ffmpeg_thumbnail(file_path, target_path):
return target_path
return None
def _is_video_version(self, version: Dict[str, Any], file_path: Path) -> bool:
media_type = self._version_media_type(version, file_path).lower()
return media_type.startswith("video/")
def _run_ffmpeg_thumbnail(self, input_path: Path, output_path: Path) -> bool:
cmd = [
"ffmpeg",
"-y",
"-ss",
"1",
"-i",
str(input_path),
"-frames:v",
"1",
"-vf",
"scale='min(960,iw)':-2",
str(output_path),
]
try:
subprocess.run(
cmd,
check=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
return output_path.exists() and output_path.stat().st_size > 0
except (OSError, subprocess.CalledProcessError):
if output_path.exists():
output_path.unlink()
return False
def _find_primary_file(self, version_dir: Path) -> Optional[Path]:
metadata_name = {"metadata.json", "download.json", "checksums.txt"}
files = [item for item in version_dir.iterdir() if item.is_file() and item.name not in metadata_name]
return files[0] if files else None
def _find_asset(self, directory: Path, basename: str) -> Optional[Path]:
matches = self._existing_asset(directory, basename)
return matches[0] if matches else None
def _numeric_sort_value(self, value: Any) -> int:
"""Convert mixed metadata values into a safe descending sort key."""
if value is None or value == "":
return 0
if isinstance(value, bool):
return int(value)
if isinstance(value, (int, float)):
return int(value)
if isinstance(value, str):
stripped = value.strip()
if not stripped:
return 0
try:
return int(stripped)
except ValueError:
try:
return int(float(stripped))
except ValueError:
logger.debug("Non-numeric release_time value in archive metadata: %r", value)
return 0
return 0
def _render_index_page(self, channels: List[Dict[str, Any]]) -> str:
cards = []
for channel in channels:
title = channel.get("display_name") or channel.get("channel_name") or channel["folder"]
description = channel.get("description") or ""
profile = self._image_tag(
channel.get("profile_asset"),
self.site_root,
css_class="avatar",
mirror_key=f"channels/{channel['folder']}/channel-profile",
)
link = f"channels/{channel['folder']}/index.html"
cards.append(
f"""
{html.escape(channel.get('channel_name') or '')} {html.escape(self._truncate(description, 220))}{html.escape(title)}
No channels found.
" return self._page( title="LBRY Archive", body=f"""Offline Archive
Browse downloaded channels, claims, metadata, and local files without needing the live site.
{html.escape(self._truncate(claim.get('description') or '', 180))}
Channel
{html.escape(channel.get('channel_name') or '')}
{html.escape(channel.get('description') or '')}
{links}No claims found.
'}Open local file: {html.escape(file_name)}
' media_preview = self._render_version_preview(version, page_dir, href) versions.append( f"""{html.escape(version.get('description') or '')}
{media_preview} {file_link}Claim
{html.escape(claim.get('description') or '')}
{links}No versions found.
'}