From 7b50110f2a1963f46295d69bc6c4a0ef41f45c73 Mon Sep 17 00:00:00 2001 From: thePR0M3TH3AN <53631862+PR0M3TH3AN@users.noreply.github.com> Date: Wed, 1 Apr 2026 20:36:35 -0400 Subject: [PATCH] Add direct-first download modes and resilience controls --- LINUX_SETUP.md | 87 ++++-- QUICKSTART.md | 41 ++- README.md | 184 +++++++++++-- config.yaml.example | 34 ++- config_loader.py | 70 +++++ direct_downloader.py | 422 +++++++++++++++-------------- downloader.py | 13 +- init.py | 15 +- install.sh | 63 +---- main.py | 271 ++++++++++++++++-- models.py | 7 + planner.py | 2 + remote-install.sh | 79 +----- setup-noninteractive.py | 7 + setup.py | 87 +----- tests/test_channel_content_mode.py | 76 ++++++ tests/test_config.py | 44 +++ tests/test_content_filters.py | 58 ++++ tests/test_direct_downloader.py | 183 +++++++++++++ tests/test_planner.py | 4 + tests/test_transport_mode.py | 15 + utils.py | 1 + 22 files changed, 1275 insertions(+), 488 deletions(-) create mode 100644 tests/test_channel_content_mode.py create mode 100644 tests/test_content_filters.py create mode 100644 tests/test_direct_downloader.py create mode 100644 tests/test_transport_mode.py diff --git a/LINUX_SETUP.md b/LINUX_SETUP.md index e58b5bc..475f1ea 100644 --- a/LINUX_SETUP.md +++ b/LINUX_SETUP.md @@ -32,7 +32,7 @@ cd LBRY-Downloader This script will: 1. Check and install system dependencies -2. Download and install LBRY SDK (lbrynet) - **No blockchain sync required!** +2. Set up the downloader for direct mode by default 3. Set up a Python virtual environment 4. Install Python packages 5. Ask for your download location @@ -42,11 +42,13 @@ This script will: **Want to uninstall later?** Just run `./uninstall.py` -### About LBRY SDK (lbrynet) - IMPORTANT +### About LBRY SDK (lbrynet) - Optional P2P Mode Only **Q: Does this download the LBRY blockchain?** -**A:** The LBRY SDK downloads **blockchain headers** (not full blocks), which is required to function: +**A:** Only if you choose to use `--p2p`. The default direct mode does not require this. + +When you do use `--p2p`, the LBRY SDK downloads **blockchain headers** (not full blocks): **First-time setup (one-time):** - Downloads ~1.7 million block headers @@ -61,7 +63,7 @@ This script will: **What this means:** - The LBRY SDK needs this data to verify which content claims are valid - This is lighter than Bitcoin's full blockchain sync -- But it IS required and will take time on first launch +- But it is only required for the optional P2P path - You can stop it with `Ctrl+C` or `lbrynet stop` **Privacy note:** This connects to LBRY's network and syncs blockchain data to verify content. This is how the LBRY protocol works. @@ -94,9 +96,9 @@ python3 --version # Should be 3.8 or higher pip3 --version ``` -### 2. Install LBRY SDK (lbrynet daemon) +### 2. Optional: Install LBRY SDK (`lbrynet`) for P2P Mode -The LBRY SDK is required to communicate with the LBRY network. +You only need the LBRY SDK if you want to run the downloader with `--p2p`. The default direct mode does not require it. #### Option A: Download Pre-built Binary (Recommended) @@ -250,18 +252,22 @@ channels: # Technology channels - input: "https://odysee.com/@LinusTechTips:1" enabled: true + content_mode: "video_only" # Educational content - input: "https://odysee.com/@veritasium:1" enabled: true + content_mode: "video_only" # News/Commentary - input: "https://odysee.com/@Timcast:1" enabled: true + content_mode: "video_only" - # Documentaries + # File / document archives - input: "https://odysee.com/@DocumentaryLab:1" enabled: true + content_mode: "non_video_only" ``` #### URL Formats Supported @@ -300,15 +306,25 @@ Add your channels under the `channels:` section: channels: - input: "https://odysee.com/@YourFavoriteChannel:1" enabled: true + content_mode: "non_video_only" tags_include: [] tags_exclude: [] - input: "https://odysee.com/@AnotherChannel:5" enabled: true + content_mode: "video_only" tags_include: [] tags_exclude: [] ``` +`content_mode` controls what each channel downloads: + +- `all`: video and non-video files +- `video_only`: only videos such as mp4/webm/mkv +- `non_video_only`: only file bundles/docs such as zip/pdf + +If you omit it, the default is `all`. + ### 4. Configure Per-Channel Download Locations (Optional) By default, all downloads go to `~/Documents/lbry-downloads/channels/`. You can specify custom locations for each channel: @@ -318,16 +334,19 @@ channels: # Downloads to default location - input: "https://odysee.com/@RegularChannel:1" enabled: true + content_mode: "all" - # Downloads to custom location (external drive) + # Downloads only videos to a custom location (external drive) - input: "https://odysee.com/@LargeVideos:1" enabled: true download_path: "/mnt/media/videos" + content_mode: "video_only" - # Downloads to custom location (home directory) + # Downloads only non-video files to a custom location - input: "https://odysee.com/@Music:1" enabled: true download_path: "~/Music/LBRY" + content_mode: "non_video_only" ``` **Why use custom locations?** @@ -363,7 +382,7 @@ general: ``` The tool downloads **most recent items first** (sorted by publish date), so: -- `download_limit: 10` = Download the 10 newest videos +- `download_limit: 10` = Download the 10 newest matching items for that channel - Older items remain in the channel but won't be downloaded yet - On next run, it will download the next 10 (if new items were published) @@ -382,7 +401,7 @@ Once you're happy with how it works, increase the limit or set to 0. ## Running the Daemon -The LBRY daemon (`lbrynet`) must be running before using the downloader. +The LBRY daemon (`lbrynet`) is only required if you want to use `--p2p`. The default direct mode does not require it. ### Start the Daemon @@ -456,16 +475,40 @@ python3 main.py --dry-run This shows what would be downloaded without actually downloading anything. +If you mostly want zip/doc/file claims and want to skip videos, the most useful first test is: + +```bash +python3 main.py --dry-run --non-video-only +``` + ### Normal Run ```bash -# Make sure lbrynet is running -lbrynet status - -# Run the downloader +# Run the downloader using direct mode (default) and each channel's configured content_mode python3 main.py ``` +### Useful Run Variants + +```bash +# Recommended default if you mainly want non-video files such as zip bundles +python3 main.py --non-video-only + +# Force video-only mode for every channel for this run +python3 main.py --video-only + +# Force non-video-only mode for every channel for this run +python3 main.py --non-video-only + +# Use the local LBRY daemon and peer-to-peer mode instead of direct mode +python3 main.py --p2p + +# Use P2P mode with a run-wide content override +python3 main.py --p2p --non-video-only +``` + +CLI flags `--video-only` and `--non-video-only` override each channel's `content_mode` for that run only. `--p2p` overrides the default direct transport mode for that run only. + ### Custom Config Location ```bash @@ -590,7 +633,7 @@ lbc_mainnet: added BlockHeightEvent(height=1665210, change=2001) **What this means:** - The LBRY SDK is downloading blockchain headers (not full blocks) -- This is required to verify content claims +- This is only required when using `--p2p` to verify content claims - Takes 5-10 minutes on first run - Subsequent runs are instant @@ -859,9 +902,9 @@ cd ~/Documents/LBRY-Downloader ./bin/lbry-test # Automatically runs with --dry-run ``` -### Prerequisites: Start the Daemon +### Optional Prerequisite: Start the Daemon for P2P Mode -**Important:** Always start the LBRY daemon before running the downloader: +Only do this if you plan to run with `--p2p`: ```bash # Check if daemon is running @@ -886,14 +929,14 @@ pip3 install --user -r requirements.txt python3 init.py # Daily use -lbrynet start # Start daemon ~/Documents/LBRY-Downloader/bin/lbry-downloader --dry-run # Test first -~/Documents/LBRY-Downloader/bin/lbry-downloader # Run sync -lbrynet stop # Stop daemon (optional) +~/Documents/LBRY-Downloader/bin/lbry-downloader --non-video-only # Recommended direct-mode run +~/Documents/LBRY-Downloader/bin/lbry-downloader --p2p # Optional local-node mode # Or if in PATH lbry-downloader --dry-run -lbry-downloader +lbry-downloader --non-video-only +lbry-downloader --p2p # Locations Install: ~/Documents/LBRY-Downloader/ diff --git a/QUICKSTART.md b/QUICKSTART.md index 517d67b..700d144 100644 --- a/QUICKSTART.md +++ b/QUICKSTART.md @@ -39,17 +39,36 @@ Edit `~/Documents/lbry-downloads/config.yaml`: channels: - input: "https://odysee.com/@YourFavoriteChannel:1" enabled: true + content_mode: "non_video_only" - input: "https://odysee.com/@AnotherChannel:5" enabled: true + content_mode: "video_only" ``` **Tip:** Just paste the Odysee URL exactly as it appears in your browser! +### Per-Channel Content Mode + +You can choose what each channel downloads: + +- `content_mode: "all"` downloads video and non-video files +- `content_mode: "video_only"` downloads only video files +- `content_mode: "non_video_only"` downloads only zip/PDF/other non-video files + +Most users who want file bundles and archives should start with: + +```yaml +channels: + - input: "https://odysee.com/@YourFavoriteChannel:1" + enabled: true + content_mode: "non_video_only" +``` + ## Launching -### Start the LBRY Daemon First +### Optional: Start the LBRY Daemon First -**Important:** The LBRY daemon must be running before using the downloader: +You only need the LBRY daemon if you plan to use `--p2p`. The default direct mode does not require it. ```bash # Start the daemon @@ -90,9 +109,21 @@ python main.py --dry-run # Test what would be downloaded (dry run) - ALWAYS DO THIS FIRST lbry-downloader --dry-run -# Actually download content +# Recommended default: download only non-video files such as zip bundles +lbry-downloader --non-video-only + +# Actually download using each channel's configured content_mode lbry-downloader +# Download only videos for every channel for this run +lbry-downloader --video-only + +# Download only non-video files for every channel for this run +lbry-downloader --non-video-only + +# Use the local LBRY node instead of the default direct mode +lbry-downloader --p2p + # Use custom config file lbry-downloader --config ./my-config.yaml @@ -100,6 +131,8 @@ lbry-downloader --config ./my-config.yaml lbry-downloader --help ``` +CLI flags `--video-only` and `--non-video-only` override per-channel `content_mode` for that run only. `--p2p` overrides the default direct transport mode for that run only. + ## Understanding Output ``` @@ -160,6 +193,8 @@ Downloads are saved to: lbrynet start ``` +This only applies when you are using `--p2p`. Default direct mode does not require `lbrynet`. + **"Config file not found"** ```bash python init.py diff --git a/README.md b/README.md index 0a5863a..400cc96 100644 --- a/README.md +++ b/README.md @@ -2,13 +2,23 @@ A Python tool that incrementally syncs downloadable file claims from configured LBRY/Odysee channels into a structured local archive. +## Recommended Defaults + +For most users: + +- Files only: `lbry-downloader --non-video-only` +- Videos only: `lbry-downloader --video-only` +- Mixed per-channel behavior: set `content_mode` in `config.yaml` and run `lbry-downloader` +- Optional local-node mode: `lbry-downloader --p2p` + ## Features - **Incremental Sync**: Only downloads new claims and updated versions - **Version Preservation**: Keeps old versions instead of overwriting them - **Robust Identity**: Uses stable LBRY claim IDs, not display names or titles - **Atomic State**: Database writes are atomic to prevent corruption -- **Daemon-Based**: Talks directly to local `lbrynet` daemon via JSON-RPC +- **Direct Mode by Default**: Uses Odysee's public proxy + CDN without requiring a local node +- **Optional P2P Mode**: Can use a local `lbrynet` daemon when explicitly requested - **Streaming Fallback**: Automatically uses daemon's streaming endpoint when P2P peers are unavailable - **Flexible Input**: Accepts Odysee URLs, LBRY URIs, or claim IDs - **Audit Trail**: Maintains run history in JSONL format @@ -41,21 +51,22 @@ cd LBRY-Downloader The setup script will: - ✅ Check and install system dependencies -- ✅ Download and install LBRY SDK (lbrynet) - **No blockchain sync required!** - ✅ Set up a Python virtual environment - ✅ Install Python packages - ✅ Ask for your download location - ✅ Configure channels to download from - ✅ Set download limits - ✅ Create launcher scripts +- ✅ Prepare a direct-first downloader setup that works without a local node +- ✅ Leave `lbrynet` as an optional install only for users who want `--p2p` -**Important:** The LBRY SDK (lbrynet) requires syncing blockchain **headers** (not full blocks) to function. On first run: +**Important:** Direct mode is now the default and does not require `lbrynet`. + +If you choose to use `--p2p`, then `lbrynet` requires syncing blockchain **headers** (not full blocks) on first use: - Downloads ~1.7 million block headers (takes 5-10 minutes) - Uses minimal storage (headers only, not full blockchain) -- Subsequent starts are instant -- This is required to verify content claims on the network - -This is lighter than Bitcoin's full blockchain sync, but still necessary for the downloader to work. +- Subsequent starts are much faster +- This is only needed for the optional node-backed mode ### Manual Install @@ -71,7 +82,9 @@ To remove LBRY Downloader: ## Prerequisites -You need to have the LBRY SDK daemon (`lbrynet`) installed and running: +Direct mode is the default, so a local LBRY daemon is not required for normal use. + +If you want to use P2P mode explicitly with `--p2p`, then you need to have the LBRY SDK daemon (`lbrynet`) installed and running: ```bash # The daemon should be accessible at http://127.0.0.1:5279 @@ -104,12 +117,19 @@ general: max_workers: 2 dry_run: false include_reposts: false + direct_base_urls: + - "https://odysee.com" + direct_max_retries_per_url: 2 + direct_retry_backoff_seconds: 2.0 + direct_auto_fallback_to_p2p: false channels: - input: "https://odysee.com/@SomeChannel:1" enabled: true + content_mode: "all" - input: "lbry://@AnotherChannel#5" enabled: true + content_mode: "non_video_only" ``` ### Finding Channel URLs @@ -183,6 +203,36 @@ channels: # No download_path - uses general.base_dir/channels/ ``` +### Per-Channel Content Mode + +You can control the download type for each channel independently. + +```yaml +channels: + - input: "https://odysee.com/@VideoChannel:1" + enabled: true + content_mode: "video_only" + + - input: "https://odysee.com/@FileArchive:1" + enabled: true + content_mode: "non_video_only" + + - input: "https://odysee.com/@MixedChannel:1" + enabled: true + content_mode: "all" +``` + +Available values: + +- `all`: Download both video and non-video files for that channel. +- `video_only`: Download only video files such as `.mp4`, `.webm`, and `.mkv`. +- `non_video_only`: Download only non-video files such as `.zip`, PDFs, and similar file claims. + +Precedence: + +- Per-channel `content_mode` is the normal default for that channel. +- CLI flags `--video-only` and `--non-video-only` override all channels for a single run. + ### Download Limit Limit how many new downloads per channel (downloads most recent first): @@ -201,6 +251,33 @@ This is useful for: - **Large channels**: Gradually sync channels with thousands of videos - **Bandwidth**: Control how much data is downloaded per run +### Direct-Mode Rate Limit Fallbacks + +If Odysee starts returning `429 Too Many Requests`, you can tune the direct downloader instead of hardcoding random public mirrors. + +```yaml +general: + direct_base_urls: + - "https://odysee.com" + # - "https://your-mirror.example" + direct_max_retries_per_url: 2 + direct_retry_backoff_seconds: 2.0 + direct_auto_fallback_to_p2p: false +``` + +What these do: + +- `direct_base_urls`: direct download bases to try in order. Odysee should normally stay first. +- `direct_max_retries_per_url`: how many retries to allow after HTTP `429` for the same direct URL. +- `direct_retry_backoff_seconds`: base delay for exponential backoff after HTTP `429`. +- `direct_auto_fallback_to_p2p`: if `true`, a direct-mode run will try the local `lbrynet` daemon after repeated rate-limit failures. + +Recommended pattern: + +- Keep `https://odysee.com` first. +- Add your own mirror only if you actually control or trust it. +- Turn on `direct_auto_fallback_to_p2p: true` only if you also maintain a working local node. + ## Launching the Application After installation, you can launch LBRY Downloader in several ways: @@ -248,7 +325,7 @@ Now you can run `lbry-downloader` from any directory! ### Prerequisites Before Running -**Important:** Make sure the LBRY daemon is running first: +**Important:** If you are using `--p2p`, make sure the LBRY daemon is running first: ```bash # Start the daemon @@ -260,15 +337,70 @@ lbrynet status ## Usage +### Recommended Default: Direct + Non-Video Downloads Only + +If your main goal is to archive file attachments such as `.zip` bundles and skip video files such as `.mp4`, this should be the primary command you use: + ```bash -# Run with default config -python main.py +python3 main.py --non-video-only +``` -# Use custom config -python main.py --config ./my-config.yaml +This mode keeps non-video claim downloads and skips video claims before the per-channel download limit is applied. -# Dry run - see what would happen -python main.py --dry-run +### Common Commands + +```bash +# Recommended default: download only non-video files such as zip archives +python3 main.py --non-video-only + +# Download only video files such as mp4/webm/mkv +python3 main.py --video-only + +# Download everything supported by the tool +python3 main.py + +# Preview what would be downloaded without downloading anything +python3 main.py --dry-run + +# Preview only non-video downloads +python3 main.py --dry-run --non-video-only + +# Use a custom config file +python3 main.py --config ./my-config.yaml + +# Run explicitly in direct mode (normally unnecessary because this is the default) +python3 main.py --direct + +# Use the local LBRY daemon and P2P network instead of direct mode +python3 main.py --p2p + +# Use P2P mode but keep only non-video files +python3 main.py --p2p --non-video-only + +# Use P2P mode but keep only video files +python3 main.py --p2p --video-only +``` + +### Flag Reference + +- `--non-video-only`: Download only non-video claims such as `.zip`, PDFs, and other non-video files. +- `--video-only`: Download only video claims such as `.mp4`, `.webm`, and `.mkv`. +- `--dry-run`: Show what would be downloaded without changing anything. +- `--direct`: Use direct mode explicitly. This is already the default. +- `--p2p`: Use the local LBRY daemon and peer-to-peer network instead of the default direct mode. +- `--config PATH`: Use a config file other than `~/Documents/lbry-downloads/config.yaml`. + +If you configure `content_mode` per channel, you usually only need the CLI content flags when you want a temporary run-wide override. + +### Running Through the Launcher + +If you installed the launcher script, the same commands work there too: + +```bash +lbry-downloader --non-video-only +lbry-downloader --video-only +lbry-downloader --dry-run --non-video-only +lbry-downloader --p2p --non-video-only ``` ## Directory Structure @@ -319,15 +451,27 @@ Make sure `lbrynet` is running: lbrynet start ``` +This only applies when you are using `--p2p`. Direct mode does not require a local daemon. + ### "Config file not found" Create a config at the default location or specify a path: ```bash -python main.py --config ./config.yaml +python3 main.py --config ./config.yaml ``` ### Dry run shows no downloads Check that your channels are enabled and have downloadable content (streams, not just reposts). +### HTTP 429 / Too Many Requests + +If direct mode is being rate-limited: + +1. Reduce how aggressively you rerun the downloader. +2. Increase `direct_retry_backoff_seconds`. +3. Keep `direct_max_retries_per_url` modest, such as `2` or `3`. +4. Add a trusted mirror to `direct_base_urls` if you actually have one. +5. Enable `direct_auto_fallback_to_p2p: true` if you want automatic fallback to the local node when direct mode keeps hitting `429`. + ## Architecture - `main.py` - Entry point and orchestration @@ -341,12 +485,12 @@ Check that your channels are enabled and have downloadable content (streams, not ## How Downloads Work -The downloader uses a two-stage approach for maximum reliability: +The downloader supports two transport modes: -1. **P2P Attempt**: First tries to download from LBRY's peer-to-peer network -2. **Streaming Fallback**: If P2P peers are unavailable (common for older content), automatically falls back to the daemon's local streaming endpoint +1. **Direct mode (default)**: Uses Odysee's public proxy for metadata and Odysee CDN for the actual download +2. **P2P mode (`--p2p`)**: Uses the local LBRY daemon and peer-to-peer network, with daemon streaming fallback when peers are unavailable -This ensures content downloads successfully even when the P2P network has no seeders. +Direct mode is the intended default for most users. P2P mode remains available when you specifically want node-backed downloading. ## License diff --git a/config.yaml.example b/config.yaml.example index f8c03dd..19fcb8a 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -4,8 +4,10 @@ lbrynet: # URL of the local LBRY daemon (lbrynet) + # Only used when running with --p2p api_url: "http://127.0.0.1:5279" # Timeout for daemon requests in seconds + # Only used when running with --p2p timeout_seconds: 60 general: @@ -31,6 +33,18 @@ general: channel_page_size: 50 # Keep records of claims even if they disappear from the channel keep_missing_claim_records: true + # Direct download base URLs to try in order. Keep Odysee first unless you + # explicitly want a mirror-first strategy. + direct_base_urls: + - "https://odysee.com" + # - "https://your-mirror.example" + # How many times to retry the same direct URL after HTTP 429 + direct_max_retries_per_url: 2 + # Base delay in seconds for exponential backoff after HTTP 429 + direct_retry_backoff_seconds: 2.0 + # If true, direct-mode runs will try the local lbrynet daemon as a fallback + # after repeated rate-limit failures. Requires a working local daemon. + direct_auto_fallback_to_p2p: false # Download limit: Maximum number of downloads per channel per run # @@ -67,20 +81,36 @@ channels: # - enabled: (optional) true/false - default: true # - download_path: (optional) Custom download location for this channel # If not set, uses general.base_dir/channels/ + # - content_mode: (optional) What file types to download for this channel + # Values: + # - all = video and non-video files + # - video_only = only video files such as mp4/webm/mkv + # - non_video_only = only non-video files such as zip/pdf + # Default: all + # Note: CLI flags --video-only and --non-video-only override + # all channel content_mode settings for that run. # - tags_include: (optional) List of tags to include (empty = all) # - tags_exclude: (optional) List of tags to exclude # # Example 1: Basic channel # - input: "https://odysee.com/@SomeChannel:1" # enabled: true + # content_mode: "all" # - # Example 2: Channel with custom download location + # Example 2: Channel with custom download location and only videos # - input: "https://odysee.com/@TechChannel:1" # enabled: true # download_path: "/mnt/media/videos/tech" + # content_mode: "video_only" # - # Example 3: Channel with tag filtering + # Example 3: Channel that downloads only file bundles, docs, and other non-video claims + # - input: "https://odysee.com/@FileArchive:1" + # enabled: true + # content_mode: "non_video_only" + # + # Example 4: Channel with tag filtering # - input: "lbry://@AnotherChannel#5" # enabled: true + # content_mode: "all" # tags_include: ["documentary", "science"] # tags_exclude: ["shorts"] diff --git a/config_loader.py b/config_loader.py index 37251dc..0a05568 100644 --- a/config_loader.py +++ b/config_loader.py @@ -20,6 +20,9 @@ class ConfigError(Exception): pass +VALID_CONTENT_MODES = {"all", "video_only", "non_video_only"} + + def load_config(config_path: Optional[str] = None) -> Config: """ Load and validate the configuration file. @@ -102,6 +105,21 @@ def _parse_config(raw: Dict[str, Any]) -> Config: download_limit=general_raw.get( "download_limit", config.general.download_limit ), + direct_base_urls=general_raw.get( + "direct_base_urls", config.general.direct_base_urls + ), + direct_max_retries_per_url=general_raw.get( + "direct_max_retries_per_url", + config.general.direct_max_retries_per_url, + ), + direct_retry_backoff_seconds=general_raw.get( + "direct_retry_backoff_seconds", + config.general.direct_retry_backoff_seconds, + ), + direct_auto_fallback_to_p2p=general_raw.get( + "direct_auto_fallback_to_p2p", + config.general.direct_auto_fallback_to_p2p, + ), ) # Validate and expand paths @@ -121,6 +139,22 @@ def _parse_config(raw: Dict[str, Any]) -> Config: # 0 means "all" (no limit), negative values are not allowed if config.general.download_limit < 0: raise ConfigError("download_limit must be 0 (for all) or a positive number") + if config.general.direct_max_retries_per_url < 0: + raise ConfigError("direct_max_retries_per_url must be 0 or a positive number") + if config.general.direct_retry_backoff_seconds <= 0: + raise ConfigError("direct_retry_backoff_seconds must be greater than 0") + if not isinstance(config.general.direct_base_urls, list): + raise ConfigError("direct_base_urls must be a list") + if not config.general.direct_base_urls: + raise ConfigError("direct_base_urls must contain at least one base URL") + normalized_direct_base_urls = [] + for idx, base_url in enumerate(config.general.direct_base_urls, start=1): + if not isinstance(base_url, str) or not base_url.strip(): + raise ConfigError( + f"direct_base_urls entry {idx} must be a non-empty string" + ) + normalized_direct_base_urls.append(base_url.rstrip("/")) + config.general.direct_base_urls = normalized_direct_base_urls # Validate log_level valid_levels = ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] @@ -175,10 +209,18 @@ def _parse_channels(channels_raw: List[Dict[str, Any]]) -> List[ChannelConfig]: input=input_val, enabled=channel_raw.get("enabled", True), download_path=download_path, + content_mode=channel_raw.get("content_mode", "all"), tags_include=channel_raw.get("tags_include", []), tags_exclude=channel_raw.get("tags_exclude", []), ) + if channel.content_mode not in VALID_CONTENT_MODES: + raise ConfigError( + f"Channel {i + 1} has invalid content_mode " + f"'{channel.content_mode}'. Must be one of: " + f"{', '.join(sorted(VALID_CONTENT_MODES))}" + ) + channels.append(channel) return channels @@ -190,30 +232,58 @@ def create_default_config() -> str: # Place this file at ~/Documents/lbry-downloads/config.yaml lbrynet: + # URL of the local LBRY daemon (lbrynet) + # Only used when running with --p2p api_url: "http://127.0.0.1:5279" + # Timeout for daemon requests in seconds + # Only used when running with --p2p timeout_seconds: 60 general: + # Base directory for all downloads and state base_dir: "~/Documents/lbry-downloads" + # Path to the state database state_file: "~/Documents/lbry-downloads/state/database.json" + # Number of concurrent downloads max_workers: 2 + # Logging level: DEBUG, INFO, WARNING, ERROR, CRITICAL log_level: "INFO" + # If true, show what would be downloaded without actually downloading dry_run: false + # Verify that previously downloaded files still exist verify_existing_files: true + # Write SHA256 checksums for downloaded files write_checksums: true + # Filename handling: 'original' or 'safe' filename_mode: "original" + # Include reposts in downloads include_reposts: false + # Number of claims to fetch per page channel_page_size: 50 + # Keep records of claims even if they disappear from the channel keep_missing_claim_records: true + # Direct download bases to try in order + direct_base_urls: + - "https://odysee.com" + # Retries per direct URL after HTTP 429 + direct_max_retries_per_url: 2 + # Base backoff in seconds after HTTP 429 (uses exponential backoff) + direct_retry_backoff_seconds: 2.0 + # If true, direct-mode runs will try the local node after repeated 429s + direct_auto_fallback_to_p2p: false + # Download limit: Number of most recent matching downloads per channel (0 = all) + download_limit: 10 channels: # Add your channels here. Examples: # - input: "https://odysee.com/@SomeChannel:1" # enabled: true + # content_mode: "non_video_only" # tags_include: [] # tags_exclude: [] # - input: "lbry://@AnotherChannel#5" # enabled: true + # content_mode: "all" """ diff --git a/direct_downloader.py b/direct_downloader.py index b9bf352..5790379 100644 --- a/direct_downloader.py +++ b/direct_downloader.py @@ -1,12 +1,11 @@ -"""Direct downloader module - downloads from Odysee CDN instead of P2P.""" +"""Direct downloader module - downloads from Odysee's direct web endpoints.""" import json import logging -import re -import shutil import time from pathlib import Path -from typing import Dict, Optional +from typing import Dict +from urllib.parse import quote try: import requests @@ -26,59 +25,127 @@ class DirectDownloadError(Exception): class DirectDownloader: - """Downloads content directly from Odysee CDN.""" + """Downloads content directly from Odysee's web-exposed endpoints.""" + + stream_path = "/$/stream" + download_path = "/$/download" def __init__(self, config: Config, base_dir: Path): self.config = config self.base_dir = base_dir + self.direct_base_urls = [ + base_url.rstrip("/") for base_url in config.general.direct_base_urls + ] self.session = requests.Session() self.session.headers.update( - {"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36"} + { + "User-Agent": ( + "Mozilla/5.0 (X11; Linux x86_64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/123.0.0.0 Safari/537.36" + ) + } ) + def _claim_slug(self, action: DownloadAction) -> str: + """Choose the best available slug for Odysee URLs.""" + slug = ( + action.metadata.get("normalized_name") + or action.metadata.get("name") + or action.claim_name + ) + return slug + + def build_stream_url(self, action: DownloadAction, base_url: str | None = None) -> str: + """Build Odysee's public direct-stream URL for a claim.""" + slug = self._claim_slug(action) + chosen_base = (base_url or self.direct_base_urls[0]).rstrip("/") + return f"{chosen_base}{self.stream_path}/{quote(slug, safe='')}/{action.claim_id}" + + def build_download_url(self, action: DownloadAction, base_url: str | None = None) -> str: + """Build Odysee's public download-entry URL for a claim.""" + slug = self._claim_slug(action) + chosen_base = (base_url or self.direct_base_urls[0]).rstrip("/") + return ( + f"{chosen_base}{self.download_path}/{quote(slug, safe='')}/{action.claim_id}" + ) + + def _is_streaming_media(self, action: DownloadAction) -> bool: + """Return True when the claim looks like audio/video media.""" + stream_type = (action.metadata.get("stream_type") or "").lower() + media_type = (action.metadata.get("source_media_type") or "").lower() + + if stream_type in {"video", "audio"}: + return True + if media_type.startswith("video/") or media_type.startswith("audio/"): + return True + return False + + def build_candidate_urls(self, action: DownloadAction) -> list[str]: + """Build Odysee direct-download URL candidates in priority order.""" + candidate_urls = [] + for base_url in self.direct_base_urls: + stream_url = self.build_stream_url(action, base_url=base_url) + download_url = self.build_download_url(action, base_url=base_url) + if self._is_streaming_media(action): + candidate_urls.extend([stream_url, download_url]) + else: + candidate_urls.extend([download_url, stream_url]) + return candidate_urls + def download(self, action: DownloadAction, dry_run: bool = False) -> bool: """ - Download a claim directly from Odysee CDN. + Download a claim directly from Odysee. Args: - action: The download action - dry_run: If True, don't actually download + action: The download action. + dry_run: If True, don't actually download. Returns: - True if download succeeded + True if download succeeded. """ if action.action == "skip_existing": - logger.debug(f"Skipping already downloaded: {action.claim_name}") + existing_path = action.metadata.get("existing_file_relpath") + if existing_path: + logger.info( + "Skipping existing file: %s (%s)", + action.claim_name, + self.base_dir / existing_path, + ) + else: + logger.info(f"Skipping existing file: {action.claim_name}") return True if dry_run: - logger.info(f"[DRY-RUN] Would download from CDN: {action.claim_name}") + logger.info(f"[DRY-RUN] Would download from Odysee: {action.claim_name}") return True - # Create directories version_dir = Path(action.target_dir) version_dir.mkdir(parents=True, exist_ok=True) - logger.info(f"Downloading from CDN: {action.claim_name}") + file_name = action.metadata.get("source_name") or f"{action.claim_name}.bin" + file_path = version_dir / file_name + + logger.info(f"Downloading from Odysee: {action.claim_name}") + logger.info(f"Saving to: {file_path}") try: - # Get download URL from Odysee - download_url = self._get_download_url(action.uri) - if not download_url: - raise DirectDownloadError("Could not get download URL from Odysee") - - logger.info(f"Download URL: {download_url}") - - # Download the file - file_path = self._download_file( - download_url, version_dir, action.claim_name - ) - if not file_path: - raise DirectDownloadError("File download failed") + last_error = None + for index, download_url in enumerate( + self.build_candidate_urls(action), start=1 + ): + logger.info(f"Direct URL candidate {index}: {download_url}") + success, failure_reason = self._download_file( + download_url, file_path, action.claim_name + ) + if success: + break + last_error = failure_reason + else: + raise DirectDownloadError(last_error or "Odysee direct download failed") logger.info(f"Downloaded: {action.claim_name} -> {file_path.name}") - # Write metadata file_info = { "download_path": str(file_path), "file_name": file_path.name, @@ -86,11 +153,9 @@ class DirectDownloader: } self._write_metadata(version_dir, action, file_info) - # Write checksum if enabled if self.config.general.write_checksums: self._write_checksum(version_dir, file_path) - # Update action metadata try: rel_path = file_path.relative_to(self.base_dir) action.metadata["local_file_path"] = str(rel_path) @@ -99,204 +164,149 @@ class DirectDownloader: return True + except DirectDownloadError: + raise except Exception as e: logger.error(f"Direct download failed for {action.claim_name}: {e}") raise DirectDownloadError(f"Download failed: {e}") - def _get_download_url(self, lbry_uri: str) -> Optional[str]: - """ - Get direct download URL from Odysee API. - - Args: - lbry_uri: LBRY URI like lbry://@channel/claim#claim_id - - Returns: - Direct download URL or None - """ - try: - # First, resolve the claim to get the permanent URL - # Odysee API endpoint for streaming - api_url = "https://api.lbry.tv/api/v1/proxy" - - # Get the claim info - claim_name = lbry_uri.split("/")[-1].split("#")[0] - - # Try to get streaming URL from Odysee - # This uses their API to get CDN links - headers = { - "Content-Type": "application/json", - } - - # Method 1: Try to get via Odysee's API - payload = { - "jsonrpc": "2.0", - "method": "get", - "params": { - "uri": lbry_uri, - "save_file": False, - }, - "id": 1, - } - - logger.debug(f"Querying Odysee API for: {lbry_uri}") - response = self.session.post( - api_url, json=payload, headers=headers, timeout=30 - ) - - if response.status_code == 200: - data = response.json() - if "result" in data and data["result"]: - result = data["result"] - - # Try to get streaming_url - if "streaming_url" in result and result["streaming_url"]: - return result["streaming_url"] - - # Try to get download URL from value - if "download_path" in result: - return result["download_path"] - - # Try to construct from lbry.tv - if "permanent_url" in result: - claim_id = result.get("claim_id", "") - if claim_id: - # Construct Odysee direct URL - return f"https://player.odycdn.com/v6/streams/{claim_id}/" - - # Method 2: Try using lbry.tv player URL directly - # Extract claim info from URI - claim_parts = lbry_uri.replace("lbry://", "").split("/") - if len(claim_parts) >= 2: - claim_name_encoded = claim_parts[-1].split("#")[0] - # Try to construct direct download URL - # This is an educated guess based on Odysee's URL structure - odysee_url = f"https://odysee.com/{lbry_uri.replace('lbry://', '').replace('#', ':')}" - logger.debug(f"Trying Odysee URL: {odysee_url}") - - # Get the page to find the download link - page_response = self.session.get(odysee_url, timeout=30) - if page_response.status_code == 200: - # Look for download URL in page - # Odysee embeds this in their JavaScript - match = re.search(r'"download_url":"([^"]+)"', page_response.text) - if match: - url = match.group(1).replace("\\u0026", "&") - return url - - # Look for streaming URL - match = re.search(r'"streaming_url":"([^"]+)"', page_response.text) - if match: - url = match.group(1).replace("\\u0026", "&") - return url - - logger.warning(f"Could not extract download URL for: {lbry_uri}") - return None - - except Exception as e: - logger.error(f"Error getting download URL: {e}") - return None - def _download_file( - self, url: str, version_dir: Path, claim_name: str - ) -> Optional[Path]: + self, url: str, file_path: Path, claim_name: str + ) -> tuple[bool, str | None]: """ - Download file from URL to version directory with progress bar. + Download file from URL with progress display. Args: - url: Download URL - version_dir: Target directory - claim_name: Name of the claim + url: Download URL. + file_path: Target file path. + claim_name: Name of the claim for display. Returns: - Path to downloaded file or None + Tuple of success flag and optional failure reason. """ - try: - # Determine filename from URL or claim name - filename = self._extract_filename(url, claim_name) - file_path = version_dir / filename + attempts = self.config.general.direct_max_retries_per_url + 1 + for attempt in range(1, attempts + 1): + try: + print(f"\n Downloading: {claim_name}") + print(f" Filename: {file_path.name}") + print(f" Saving to: {file_path}") + print(f" Source: {url}") - # Print clear message with filename - print(f"\n📥 Downloading: {claim_name}") - print(f" Filename: {filename}") - print(f" Destination: {file_path.parent}") - - # Stream download with progress - response = self.session.get(url, stream=True, timeout=300) - response.raise_for_status() - - total_size = int(response.headers.get("content-length", 0)) - - if total_size > 0: - size_mb = total_size / (1024 * 1024) - print(f" Size: {size_mb:.2f} MB") - else: - print(f" Size: Unknown") - print() - - start_time = time.time() - last_update = start_time - downloaded = 0 - chunk_size = 8192 - - with open(file_path, "wb") as f: - for chunk in response.iter_content(chunk_size=chunk_size): - if chunk: - f.write(chunk) - downloaded += len(chunk) - - # Update progress bar every 0.5 seconds - current_time = time.time() - if current_time - last_update >= 0.5: - self._print_progress_bar( - downloaded, total_size, start_time, claim_name - ) - last_update = current_time - - # Final progress update - self._print_progress_bar( - downloaded, total_size, start_time, claim_name, final=True + response = self.session.get( + url, + stream=True, + timeout=300, + headers={"Referer": "https://odysee.com/"}, ) - print() # New line after progress bar + response.raise_for_status() - # Verify file was downloaded - if file_path.exists() and file_path.stat().st_size > 0: - size_mb = file_path.stat().st_size / (1024 * 1024) - elapsed = time.time() - start_time - speed_mbps = (size_mb / elapsed) if elapsed > 0 else 0 - print(f"✅ Complete: {size_mb:.2f} MB @ {speed_mbps:.2f} MB/s\n") - return file_path - else: - print(f"❌ Error: Downloaded file is empty\n") - return None + content_type = response.headers.get("content-type", "") + if "text/html" in content_type: + logger.warning("Got HTML response instead of file content") + return False, "Got HTML response instead of file content" - except Exception as e: - print(f"❌ Error downloading {claim_name}: {e}\n") - logger.error(f"Error downloading file: {e}") - return None + total_size = int(response.headers.get("content-length", 0)) + if total_size > 0: + size_mb = total_size / (1024 * 1024) + print(f" Size: {size_mb:.2f} MB") + else: + print(" Size: Unknown") + print() + + start_time = time.time() + last_update = start_time + downloaded = 0 + + with open(file_path, "wb") as f: + for chunk in response.iter_content(chunk_size=8192): + if chunk: + f.write(chunk) + downloaded += len(chunk) + + current_time = time.time() + if current_time - last_update >= 0.5: + self._print_progress_bar( + downloaded, total_size, start_time + ) + last_update = current_time + + self._print_progress_bar( + downloaded, total_size, start_time, final=True + ) + print() + + if file_path.exists() and file_path.stat().st_size > 0: + size_mb = file_path.stat().st_size / (1024 * 1024) + elapsed = time.time() - start_time + speed_mbps = (size_mb / elapsed) if elapsed > 0 else 0 + print(f" Complete: {size_mb:.2f} MB @ {speed_mbps:.2f} MB/s\n") + return True, None + + if file_path.exists(): + file_path.unlink() + print(" Error: Downloaded file is empty\n") + return False, "Downloaded file is empty" + + except requests.exceptions.HTTPError as e: + status_code = getattr(e.response, "status_code", None) + if status_code == 401: + logger.error(f"HTTP 401 downloading file: {e}") + reason = "Odysee CDN rejected this direct URL as unauthorized" + elif status_code == 429: + logger.error(f"HTTP 429 downloading file: {e}") + if file_path.exists(): + file_path.unlink() + if attempt < attempts: + backoff = ( + self.config.general.direct_retry_backoff_seconds + * (2 ** (attempt - 1)) + ) + logger.warning( + "HTTP 429 downloading %s. Retrying in %.1fs (%s/%s)", + claim_name, + backoff, + attempt, + attempts - 1, + ) + time.sleep(backoff) + continue + reason = ( + "Odysee CDN rate-limited the direct download request after retries" + ) + else: + logger.error(f"HTTP error downloading file: {e}") + reason = f"HTTP error: {e}" + if file_path.exists(): + file_path.unlink() + return False, reason + except Exception as e: + logger.error(f"Error downloading file: {e}") + if file_path.exists(): + file_path.unlink() + return False, str(e) + return False, "Direct download failed" def _print_progress_bar( self, downloaded: int, total_size: int, start_time: float, - claim_name: str, final: bool = False, - ): + ) -> None: """Print a progress bar to the console.""" if total_size > 0: percent = min(100, int((downloaded / total_size) * 100)) filled = int(50 * percent / 100) - bar = "█" * filled + "░" * (50 - filled) + bar = "\u2588" * filled + "\u2591" * (50 - filled) - # Calculate speed elapsed = time.time() - start_time + speed_mbps = 0 if elapsed > 0: speed_mbps = (downloaded / (1024 * 1024)) / elapsed speed_str = f"{speed_mbps:.2f} MB/s" else: speed_str = "calculating..." - # Calculate ETA if not final and speed_mbps > 0: remaining = (total_size - downloaded) / (1024 * 1024) eta = remaining / speed_mbps @@ -304,27 +314,15 @@ class DirectDownloader: else: eta_str = "" - # Clear line and print progress print( - f"\r [{bar}] {percent}% | {speed_str} | {eta_str}", end="", flush=True + f"\r [{bar}] {percent}% | {speed_str} | {eta_str}", + end="", + flush=True, ) else: - # Unknown size, just show downloaded amount downloaded_mb = downloaded / (1024 * 1024) print(f"\r Downloaded: {downloaded_mb:.2f} MB", end="", flush=True) - def _extract_filename(self, url: str, claim_name: str) -> str: - """Extract filename from URL or use claim name.""" - # Try to get from URL - if "." in url.split("/")[-1].split("?")[0]: - filename = url.split("/")[-1].split("?")[0] - # Sanitize - filename = re.sub(r'[<>"|?*]', "_", filename) - return filename[:255] # Max filename length - - # Use claim name with extension - return f"{claim_name}.mp4" # Default to mp4 - def _write_metadata( self, version_dir: Path, action: DownloadAction, file_info: Dict ) -> None: @@ -336,10 +334,14 @@ class DirectDownloader: "title": action.metadata.get("title"), "version_token": action.version_token, "permanent_url": action.uri, - "download_source": "odysee_cdn", + "download_source": ( + "odysee_stream" if self._is_streaming_media(action) else "odysee_download" + ), "downloaded_at": time.strftime("%Y-%m-%dT%H:%M:%S"), "file_name": file_info.get("file_name"), "file_size": file_info.get("size"), + "stream_type": action.metadata.get("stream_type"), + "source_media_type": action.metadata.get("source_media_type"), } metadata_path = version_dir / "metadata.json" diff --git a/downloader.py b/downloader.py index 88f6100..37837ab 100644 --- a/downloader.py +++ b/downloader.py @@ -49,7 +49,15 @@ class Downloader: DownloadError: If the download fails. """ if action.action == "skip_existing": - logger.debug(f"Skipping already downloaded: {action.claim_name}") + existing_path = action.metadata.get("existing_file_relpath") + if existing_path: + logger.info( + "Skipping existing file: %s (%s)", + action.claim_name, + self.base_dir / existing_path, + ) + else: + logger.info(f"Skipping existing file: {action.claim_name}") return True if dry_run: @@ -61,6 +69,7 @@ class Downloader: version_dir.mkdir(parents=True, exist_ok=True) logger.info(f"Downloading: {action.claim_name}") + logger.info(f"Saving to directory: {version_dir}") try: # Call daemon to start P2P download @@ -281,6 +290,7 @@ class Downloader: streaming_url = f"http://localhost:5280/stream/{sd_hash}" logger.info(f"Downloading via daemon streaming endpoint...") + logger.info(f"Streaming destination: {target_path}") try: response = requests.get(streaming_url, stream=True, timeout=300) @@ -291,6 +301,7 @@ class Downloader: chunk_size = 8192 print(f"\n📥 Downloading: {action.claim_name}") + print(f" Saving to: {target_path}") print(f" Size: {total_size / (1024 * 1024):.2f} MB") print() diff --git a/init.py b/init.py index 3a04a74..2506562 100644 --- a/init.py +++ b/init.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 """ -Initialize LBRY Downloader configuration. +Initialize Odysee/LBRY Downloader configuration. This script creates the initial configuration file and directory structure. """ @@ -14,7 +14,7 @@ from utils import expand_path def main(): - print("LBRY Downloader - Initialization") + print("Odysee/LBRY Downloader - Initialization") print("=" * 40) # Default paths @@ -51,12 +51,13 @@ def main(): print("Next steps:") print("1. Edit the configuration file to add your channels") print(f" nano {config_file}") - print("\n2. Ensure lbrynet daemon is running:") - print(" lbrynet start") - print("\n3. Run the downloader:") - print(" python main.py") - print("\n4. Or test first with dry-run:") + print("\n2. Test first with dry-run:") print(" python main.py --dry-run") + print("\n3. Recommended first real run:") + print(" python main.py --non-video-only") + print("\n4. Optional P2P mode if you want the local node path:") + print(" lbrynet start") + print(" python main.py --p2p") print("=" * 40) diff --git a/install.sh b/install.sh index 0971ccd..d862421 100755 --- a/install.sh +++ b/install.sh @@ -12,7 +12,7 @@ set -e echo "================================================" -echo " LBRY Downloader - Installation" +echo " Odysee/LBRY Downloader - Installation" echo "================================================" echo "" @@ -117,51 +117,14 @@ pip install --upgrade pip pip install -r requirements.txt print_status "Python dependencies installed" -# Check for lbrynet echo "" -echo "Checking for LBRY SDK (lbrynet)..." +echo "Checking for optional LBRY SDK (lbrynet)..." if command -v lbrynet &> /dev/null; then - print_status "lbrynet found at: $(which lbrynet)" + print_status "Optional P2P dependency found at: $(which lbrynet)" lbrynet --version else - print_warning "lbrynet not found in PATH" - echo "" - echo "The LBRY SDK (lbrynet) is required to download content." - echo "" - read -p "Would you like to download and install lbrynet now? (y/N): " -n 1 -r - echo - - if [[ $REPLY =~ ^[Yy]$ ]]; then - echo "Downloading lbrynet..." - mkdir -p ~/.local/bin - - # Get latest release - LBRY_VERSION="0.113.0" - DOWNLOAD_URL="https://github.com/lbryio/lbry-sdk/releases/download/v${LBRY_VERSION}/lbry-sdk-linux.zip" - - cd ~/.local/bin - if wget -q "$DOWNLOAD_URL"; then - unzip -q lbry-sdk-linux.zip - rm lbry-sdk-linux.zip - chmod +x lbrynet - print_status "lbrynet installed to ~/.local/bin" - - # Check if ~/.local/bin is in PATH - if [[ ":$PATH:" != *":$HOME/.local/bin:"* ]]; then - echo 'export PATH="$HOME/.local/bin:$PATH"' >> ~/.bashrc - print_status "Added ~/.local/bin to PATH in ~/.bashrc" - print_warning "Please run: source ~/.bashrc" - fi - - cd "$SCRIPT_DIR" - else - print_error "Failed to download lbrynet" - echo "Please install manually from: https://github.com/lbryio/lbry-sdk/releases" - fi - else - print_warning "Skipping lbrynet installation" - echo "You'll need to install it manually before using the downloader" - fi + print_info "lbrynet not found. Default direct mode does not require it." + print_info "Install it later only if you want to run with --p2p." fi # Initialize configuration @@ -201,13 +164,6 @@ fi cd "$DOWNLOADER_DIR" source venv/bin/activate 2>/dev/null || true -# Check if lbrynet is running -if ! curl -s http://127.0.0.1:5279 > /dev/null 2>&1; then - echo "Starting lbrynet daemon..." - lbrynet start & - sleep 5 -fi - python main.py "$@" EOF @@ -233,7 +189,7 @@ echo "1. Edit configuration to add your channels:" echo " nano ~/Documents/lbry-downloads/config.yaml" echo "" echo "2. Start the LBRY daemon:" -echo " lbrynet start" +echo " Optional only for P2P mode: lbrynet start" echo "" echo "3. Test with dry-run:" echo " python main.py --dry-run" @@ -241,9 +197,12 @@ echo " # OR:" echo " ~/bin/lbry-sync --dry-run" echo "" echo "4. Run the downloader:" -echo " python main.py" +echo " python main.py --non-video-only" echo " # OR:" -echo " ~/bin/lbry-sync" +echo " ~/bin/lbry-sync --non-video-only" +echo "" +echo "5. Optional local-node mode:" +echo " python main.py --p2p" echo "" echo "For detailed instructions, see:" echo " - LINUX_SETUP.md" diff --git a/main.py b/main.py index 08bef73..d700158 100644 --- a/main.py +++ b/main.py @@ -14,6 +14,7 @@ from typing import List, Optional from config_loader import ConfigError, ensure_directories, load_config from downloader import DownloadError, Downloader +from direct_downloader import DirectDownloadError from lbry_client import LbryClient, LbryClientError, check_daemon_health from models import Channel, Config, DownloadAction from planner import Planner @@ -26,6 +27,20 @@ from utils import ( ) +VIDEO_EXTENSIONS = { + ".mp4", + ".webm", + ".mkv", + ".mov", + ".avi", + ".m4v", + ".mpeg", + ".mpg", + ".wmv", + ".flv", +} + + def setup_logging(log_level: str) -> None: """Configure logging.""" logging.basicConfig( @@ -83,6 +98,24 @@ def resolve_channel( channel_claim_id = claim.get("claim_id") channel_name = claim.get("name") + # Odysee's public proxy can resolve vanity channel URLs to older + # non-controlling channel claims. Enumerating streams from those claim IDs + # returns zero items, so normalize to the controlling claim before scanning. + if not client.supports_file_ops and channel_name: + controlling_claim = client.find_controlling_channel(channel_name) + if controlling_claim and controlling_claim.get("claim_id"): + controlling_claim_id = controlling_claim.get("claim_id") + if controlling_claim_id != channel_claim_id: + logger.debug( + "Normalized proxy-resolved channel %s from %s to controlling claim %s", + channel_name, + channel_claim_id, + controlling_claim_id, + ) + claim = controlling_claim + channel_claim_id = controlling_claim_id + channel_name = claim.get("name") + if not channel_claim_id: logger.error(f"No claim_id in resolved channel: {channel_input}") return None @@ -164,14 +197,96 @@ def enumerate_channel_claims( return all_claims -def run_sync(config: Config, dry_run: bool = False, direct: bool = False) -> dict: +def _action_is_video(action: DownloadAction) -> bool: + """Return True when a planned action looks like a video download.""" + media_type = (action.metadata.get("source_media_type") or "").lower() + if media_type.startswith("video/"): + return True + + stream_type = (action.metadata.get("stream_type") or "").lower() + if stream_type == "video": + return True + + source_name = action.metadata.get("source_name") or action.metadata.get("file_name") + if source_name and Path(source_name).suffix.lower() in VIDEO_EXTENSIONS: + return True + + return False + + +def filter_actions_by_content_type( + actions: List[DownloadAction], + *, + video_only: bool = False, + non_video_only: bool = False, +) -> List[DownloadAction]: + """Filter planned actions by content type before limits and execution.""" + if not video_only and not non_video_only: + return actions + + filtered_actions = [] + for action in actions: + is_video = _action_is_video(action) + if video_only and is_video: + filtered_actions.append(action) + elif non_video_only and not is_video: + filtered_actions.append(action) + + return filtered_actions + + +def resolve_content_filter_mode( + channel_config, + *, + video_only: bool = False, + non_video_only: bool = False, +) -> tuple[bool, bool, str]: + """Resolve the effective content filter for a channel.""" + if video_only: + return True, False, "video-only (CLI override)" + if non_video_only: + return False, True, "non-video-only (CLI override)" + + content_mode = getattr(channel_config, "content_mode", "all") + if content_mode == "video_only": + return True, False, "video-only (channel config)" + if content_mode == "non_video_only": + return False, True, "non-video-only (channel config)" + + return False, False, "all content" + + +def resolve_transport_mode( + *, + direct: bool = False, + p2p: bool = False, +) -> tuple[bool, str]: + """Resolve the effective transport mode.""" + if p2p: + return False, "p2p" + return True, "direct" + + +def _is_rate_limit_error(exc: Exception) -> bool: + """Return True when the error indicates direct-mode rate limiting.""" + return "rate-limited" in str(exc).lower() + + +def run_sync( + config: Config, + dry_run: bool = False, + direct: bool = True, + *, + video_only: bool = False, + non_video_only: bool = False, +) -> dict: """ Run the main synchronization process. Args: config: Application configuration. dry_run: If True, don't actually download. - direct: If True, use Odysee CDN instead of P2P. + direct: If True, use Odysee proxy/CDN instead of P2P. Returns: Statistics dictionary. @@ -195,30 +310,28 @@ def run_sync(config: Config, dry_run: bool = False, direct: bool = False) -> dic base_path = Path(config.general.base_dir) planner = Planner(config, state_db.data, str(base_path)) - # Initialize LBRY client (needed for channel/claim resolution in both modes) - client = LbryClient( - api_url=config.lbrynet.api_url, - timeout=config.lbrynet.timeout_seconds, - ) - - # Check daemon health (needed for metadata even in direct mode) - logger.info(f"Checking daemon at {config.lbrynet.api_url}") - try: - check_daemon_health(client) - logger.info("Daemon is healthy") - except LbryClientError as e: - logger.error(str(e)) - logger.error( - "Daemon is required for channel/claim resolution even in --direct mode" - ) - return stats - if direct: - logger.info("Using DIRECT mode (Odysee CDN for downloads)") + logger.info("Using DIRECT mode (Odysee proxy for metadata, Odysee CDN for downloads)") + client = LbryClient.create_odysee_proxy(timeout=config.lbrynet.timeout_seconds) from direct_downloader import DirectDownloader downloader = DirectDownloader(config, base_path) + p2p_fallback_client = None + p2p_fallback_downloader = None + p2p_fallback_available = None else: + client = LbryClient( + api_url=config.lbrynet.api_url, + timeout=config.lbrynet.timeout_seconds, + ) + logger.info(f"Checking daemon at {config.lbrynet.api_url}") + try: + check_daemon_health(client) + logger.info("Daemon is healthy") + except LbryClientError as e: + logger.error(str(e)) + return stats + logger.info("Using P2P mode (LBRY network)") downloader = Downloader(client, config, base_path) @@ -251,6 +364,26 @@ def run_sync(config: Config, dry_run: bool = False, direct: bool = False) -> dic # Plan downloads actions = planner.process_channel_claims(channel, claims) + actions_before_filter = len(actions) + channel_video_only, channel_non_video_only, mode_label = ( + resolve_content_filter_mode( + channel_config, + video_only=video_only, + non_video_only=non_video_only, + ) + ) + actions = filter_actions_by_content_type( + actions, + video_only=channel_video_only, + non_video_only=channel_non_video_only, + ) + + if len(actions) != actions_before_filter: + logger.info( + "Content filter (%s) skipped %s planned claim(s).", + mode_label, + actions_before_filter - len(actions), + ) # Apply download limit if configured # Separate skip actions from download actions @@ -314,7 +447,61 @@ def run_sync(config: Config, dry_run: bool = False, direct: bool = False) -> dic "success" if success else "failure", ) - except DownloadError as e: + except (DownloadError, DirectDownloadError) as e: + if ( + direct + and config.general.direct_auto_fallback_to_p2p + and isinstance(e, DirectDownloadError) + and _is_rate_limit_error(e) + and not dry_run + ): + if p2p_fallback_available is None: + try: + p2p_fallback_client = LbryClient( + api_url=config.lbrynet.api_url, + timeout=config.lbrynet.timeout_seconds, + ) + check_daemon_health(p2p_fallback_client) + p2p_fallback_downloader = Downloader( + p2p_fallback_client, config, base_path + ) + p2p_fallback_available = True + logger.warning( + "Direct mode hit rate limiting. Local P2P fallback is available." + ) + except LbryClientError as fallback_init_error: + p2p_fallback_available = False + logger.warning( + "Direct mode hit rate limiting, but P2P fallback is unavailable: %s", + fallback_init_error, + ) + + if p2p_fallback_available and p2p_fallback_downloader is not None: + try: + logger.warning( + "Retrying %s via P2P fallback after direct-mode rate limiting.", + action.claim_name, + ) + success = p2p_fallback_downloader.download( + action, dry_run=False + ) + if success: + planner.update_state_from_action(action, success=True) + history_file = base_path / "state" / "run-history.jsonl" + state_db.log_run_history( + str(history_file), + channel.channel_claim_id, + action.claim_id, + action.action, + action.version_token, + "success", + ) + continue + except DownloadError as fallback_error: + e = DirectDownloadError( + f"{e}; P2P fallback also failed: {fallback_error}" + ) + logger.error(f"Download failed: {e}") stats["failures"] += 1 @@ -352,10 +539,10 @@ def main(): formatter_class=argparse.RawDescriptionHelpFormatter, epilog=""" Examples: - python main.py # Run with default config (P2P mode) + python main.py # Run with default config (DIRECT mode) python main.py --config ./config.yaml # Use custom config python main.py --dry-run # Plan actions but don't download - python main.py --direct # Download from Odysee CDN (more reliable) + python main.py --p2p # Use the local LBRY node instead """, ) @@ -372,16 +559,39 @@ Examples: help="Show what would be downloaded without actually downloading", ) - parser.add_argument( + transport_group = parser.add_mutually_exclusive_group() + transport_group.add_argument( "--direct", "-d", action="store_true", - help="Download directly from Odysee CDN instead of using P2P network", + help="Use direct mode explicitly (default: Odysee public proxy + CDN)", + ) + transport_group.add_argument( + "--p2p", + action="store_true", + help="Use the local LBRY daemon and peer-to-peer network instead of direct mode", + ) + + content_filter_group = parser.add_mutually_exclusive_group() + content_filter_group.add_argument( + "--video-only", + action="store_true", + help="Only include video downloads such as mp4/webm/mkv", + ) + content_filter_group.add_argument( + "--non-video-only", + action="store_true", + help="Only include non-video downloads such as zip files", ) args = parser.parse_args() try: + direct_mode, transport_label = resolve_transport_mode( + direct=args.direct, + p2p=args.p2p, + ) + # Load config config = load_config(args.config) @@ -390,12 +600,19 @@ Examples: logger = logging.getLogger(__name__) logger.info("LBRY Downloader starting") + logger.info("Effective transport mode: %s", transport_label) # Ensure directories exist ensure_directories(config) # Run sync - stats = run_sync(config, dry_run=args.dry_run, direct=args.direct) + stats = run_sync( + config, + dry_run=args.dry_run, + direct=direct_mode, + video_only=args.video_only, + non_video_only=args.non_video_only, + ) # Print summary summary = format_summary( diff --git a/models.py b/models.py index ddd4228..e3c5675 100644 --- a/models.py +++ b/models.py @@ -12,6 +12,7 @@ class ChannelConfig: input: str enabled: bool = True download_path: Optional[str] = None + content_mode: str = "all" tags_include: List[str] = field(default_factory=list) tags_exclude: List[str] = field(default_factory=list) @@ -34,6 +35,12 @@ class GeneralConfig: download_limit: int = ( 10 # Number of most recent downloads per channel, or "all" (0) ) + direct_base_urls: List[str] = field( + default_factory=lambda: ["https://odysee.com"] + ) + direct_max_retries_per_url: int = 2 + direct_retry_backoff_seconds: float = 2.0 + direct_auto_fallback_to_p2p: bool = False @dataclass diff --git a/planner.py b/planner.py index 2044c1a..4f68196 100644 --- a/planner.py +++ b/planner.py @@ -183,6 +183,8 @@ class Planner: target_dir=str(version_dir), metadata=metadata, ) + if version.file_relpath: + action.metadata["existing_file_relpath"] = version.file_relpath logger.debug(f"Skipping existing: {claim_name}") return action else: diff --git a/remote-install.sh b/remote-install.sh index ba2f3a3..2d22c28 100755 --- a/remote-install.sh +++ b/remote-install.sh @@ -20,7 +20,7 @@ REPO_URL="https://github.com/PR0M3TH3AN/LBRY-Downloader.git" INSTALL_DIR="${HOME}/Documents/LBRY-Downloader" echo "==========================================" -echo " LBRY Downloader - Remote Installer" +echo " Odysee/LBRY Downloader - Remote Installer" echo "==========================================" echo "" @@ -74,70 +74,11 @@ fi print_status "Python $PYTHON_VERSION found" -# Check if lbrynet is installed -if ! command -v lbrynet &> /dev/null; then - print_warning "LBRY SDK (lbrynet) not found" - echo "" - echo "The LBRY SDK is required to download content from the LBRY network." - echo "It can be downloaded from: https://github.com/lbryio/lbry-sdk/releases" - echo "" - read -p "Download and install lbrynet automatically? (Y/n): " -n 1 -r - echo - if [[ ! $REPLY =~ ^[Nn]$ ]]; then - print_status "Installing LBRY SDK..." - - # Create bin directory - mkdir -p ~/.local/bin - - # Download latest lbrynet for Linux - LBRY_VERSION="0.113.0" - DOWNLOAD_URL="https://github.com/lbryio/lbry-sdk/releases/download/v${LBRY_VERSION}/lbrynet-linux.zip" - - cd ~/.local/bin - - # Download with progress - echo "Downloading lbrynet v${LBRY_VERSION}..." - if wget --progress=bar:force "$DOWNLOAD_URL" -O lbrynet-linux.zip 2>&1; then - print_status "Downloaded lbrynet" - - # Check if file exists and has content - if [ -f "lbrynet-linux.zip" ] && [ -s "lbrynet-linux.zip" ]; then - # Extract - if unzip -q lbrynet-linux.zip; then - rm lbrynet-linux.zip - chmod +x lbrynet - print_status "Extracted lbrynet" - - # Add to PATH if not already there - if ! echo "$PATH" | grep -q "$HOME/.local/bin"; then - echo 'export PATH="$HOME/.local/bin:$PATH"' >> ~/.bashrc - print_status "Added ~/.local/bin to PATH" - export PATH="$HOME/.local/bin:$PATH" - fi - - print_status "LBRY SDK installed successfully!" - print_info "lbrynet installed to: ~/.local/bin/lbrynet" - else - print_error "Failed to extract lbrynet" - rm -f lbrynet-linux.zip - fi - else - print_error "Download failed - file not found or empty" - fi - else - print_error "Failed to download lbrynet" - echo "Please install manually from:" - echo " https://github.com/lbryio/lbry-sdk/releases" - fi - - # Return to original directory - cd - > /dev/null - else - print_warning "Skipping lbrynet installation" - echo "You'll need to install it manually before using the downloader" - fi +if command -v lbrynet &> /dev/null; then + print_status "Optional P2P dependency found: $(which lbrynet)" else - print_status "lbrynet found at: $(which lbrynet)" + print_info "lbrynet not found. That's fine for default direct mode." + print_info "Install it later only if you want to run with --p2p." fi # Check if running in interactive mode @@ -187,19 +128,17 @@ echo "" echo "1️⃣ Reload your shell (IMPORTANT):" echo " source ~/.bashrc" echo "" -echo "2️⃣ Test the downloader (daemon starts automatically):" +echo "2️⃣ Test the downloader:" echo " lbry-downloader --dry-run" echo "" echo "3️⃣ Download content:" -echo " lbry-downloader" +echo " lbry-downloader --non-video-only" echo "" echo "💡 If 'lbry-downloader' command not found after reload, use:" echo " ~/Documents/LBRY-Downloader/bin/lbry-downloader" echo "" -echo "⚠️ If lbrynet was not installed automatically, install it from:" -echo " https://github.com/lbryio/lbry-sdk/releases" -echo " Then run: lbrynet start" -echo " ~/Documents/LBRY-Downloader/bin/lbry-downloader" +echo "⚙️ Optional P2P mode:" +echo " Install lbrynet only if you want: ~/Documents/LBRY-Downloader/bin/lbry-downloader --p2p" echo "" echo "📁 Config file: ~/Documents/lbry-downloads/config.yaml" echo "" diff --git a/setup-noninteractive.py b/setup-noninteractive.py index ed6024e..165a5b9 100755 --- a/setup-noninteractive.py +++ b/setup-noninteractive.py @@ -76,7 +76,9 @@ if current_dir != Path(INSTALL_DIR): config_content = f"""# LBRY Downloader Configuration lbrynet: + # Only used when running with --p2p api_url: "http://127.0.0.1:5279" + # Only used when running with --p2p timeout_seconds: 60 general: @@ -97,6 +99,7 @@ channels: # Add your channels here: # - input: "https://odysee.com/@ChannelName:1" # enabled: true + # content_mode: "non_video_only" """ config_path = Path(DOWNLOAD_DIR) / "config.yaml" @@ -126,3 +129,7 @@ print(f"To run: {launcher}") print() print("Edit the config to add channels, then run:") print(f" {launcher} --dry-run") +print("Recommended first real run:") +print(f" {launcher} --non-video-only") +print("Optional local-node mode:") +print(f" {launcher} --p2p") diff --git a/setup.py b/setup.py index 11f1540..9f3c18e 100755 --- a/setup.py +++ b/setup.py @@ -350,8 +350,10 @@ def create_config( lbrynet: # URL of the local LBRY daemon (lbrynet) + # Only used when running with --p2p api_url: "http://127.0.0.1:5279" # Timeout for daemon requests in seconds + # Only used when running with --p2p timeout_seconds: 60 general: @@ -386,12 +388,14 @@ channels: for channel in channels: config += f''' - input: "{channel["input"]}" enabled: true + content_mode: "non_video_only" ''' if not channels: config += """ # Add your channels here: # - input: "https://odysee.com/@ChannelName:1" # enabled: true + # content_mode: "non_video_only" """ return config @@ -405,54 +409,13 @@ def create_launcher_script(install_dir: Path, venv_path: Path) -> None: bin_dir = install_dir / "bin" bin_dir.mkdir(exist_ok=True) - # Create main launcher with daemon auto-check + # Create main launcher launcher = bin_dir / "lbry-downloader" launcher_content = f"""#!/bin/bash # LBRY Downloader Launcher -# Auto-checks and starts lbrynet daemon if needed INSTALL_DIR="{install_dir}" VENV_DIR="{venv_path}" -DAEMON_URL="http://127.0.0.1:5279" - -# Check if daemon is running -if ! curl -s "$DAEMON_URL" > /dev/null 2>&1; then - echo "LBRY daemon not running. Starting..." - - # Try to start daemon - if command -v lbrynet &> /dev/null; then - lbrynet start & - echo "Waiting for daemon to start..." - - # Wait up to 30 seconds for daemon - for i in $(seq 1 30); do - if curl -s "$DAEMON_URL" > /dev/null 2>&1; then - echo "✓ Daemon started successfully" - break - fi - sleep 1 - done - - # Check if daemon started - if ! curl -s "$DAEMON_URL" > /dev/null 2>&1; then - echo "✗ Failed to start daemon automatically" - echo "" - echo "Please start it manually:" - echo " lbrynet start" - echo "" - echo "Or see LINUX_SETUP.md for installation instructions" - exit 1 - fi - else - echo "✗ lbrynet not found in PATH" - echo "" - echo "Please install LBRY SDK:" - echo " https://github.com/lbryio/lbry-sdk/releases" - echo "" - echo "Or see LINUX_SETUP.md for detailed instructions" - exit 1 - fi -fi cd "$INSTALL_DIR" source "$VENV_DIR/bin/activate" @@ -464,37 +427,13 @@ python main.py "$@" launcher.chmod(0o755) print_success(f"Created launcher: {launcher}") - # Create quick test script with daemon auto-check + # Create quick test script test_script = bin_dir / "lbry-test" test_content = f"""#!/bin/bash # Quick test script INSTALL_DIR="{install_dir}" VENV_DIR="{venv_path}" -DAEMON_URL="http://127.0.0.1:5279" - -# Check if daemon is running -if ! curl -s "$DAEMON_URL" > /dev/null 2>&1; then - echo "LBRY daemon not running. Starting..." - - if command -v lbrynet &> /dev/null; then - lbrynet start & - echo "Waiting for daemon to start..." - sleep 5 - - if curl -s "$DAEMON_URL" > /dev/null 2>&1; then - echo "✓ Daemon started" - else - echo "✗ Failed to start daemon" - echo "Run manually: lbrynet start" - exit 1 - fi - else - echo "✗ lbrynet not found. Install from:" - echo " https://github.com/lbryio/lbry-sdk/releases" - exit 1 - fi -fi cd "$INSTALL_DIR" source "$VENV_DIR/bin/activate" @@ -533,10 +472,10 @@ def run_setup() -> None: """Main setup function.""" print(f""" {Colors.BOLD}╔══════════════════════════════════════════════════════════════╗ -║ LBRY Downloader - Interactive Setup ║ +║ Odysee/LBRY Downloader - Interactive Setup ║ ║ ║ ║ This script will help you install and configure the ║ -║ LBRY Downloader tool. ║ +║ direct-first archive downloader. ║ ╚══════════════════════════════════════════════════════════════╝{Colors.END} """) @@ -662,20 +601,20 @@ def run_setup() -> None: ║ Setup Complete! ║ ╚══════════════════════════════════════════════════════════════╝{Colors.END} -🎉 LBRY Downloader is ready to use! +🎉 Odysee/LBRY Downloader is ready to use! ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 📋 QUICK START (Run these commands): ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -1️⃣ Start the LBRY daemon: - lbrynet start +1️⃣ Optional P2P prerequisite: + Optional only for P2P mode: lbrynet start 2️⃣ Test your configuration: lbry-downloader --dry-run 3️⃣ Download content: - lbry-downloader + lbry-downloader --non-video-only ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ ℹ️ IMPORTANT NOTES: @@ -688,7 +627,7 @@ def run_setup() -> None: If {Colors.BOLD}lbry-downloader{Colors.END} command not found, run: source ~/.bashrc -If lbrynet is not installed, see LINUX_SETUP.md for installation. +Direct mode is the default. Use {Colors.BOLD}--p2p{Colors.END} only if you explicitly want the local node path. To add more channels later: nano {config_path} diff --git a/tests/test_channel_content_mode.py b/tests/test_channel_content_mode.py new file mode 100644 index 0000000..21c5d5d --- /dev/null +++ b/tests/test_channel_content_mode.py @@ -0,0 +1,76 @@ +"""Tests for per-channel content mode config and resolution.""" + +import pytest + +from config_loader import ConfigError, _parse_config +from main import resolve_content_filter_mode +from models import ChannelConfig + + +def test_channel_content_mode_parses_from_config(): + config = _parse_config( + { + "channels": [ + { + "input": "https://odysee.com/@Files:1", + "enabled": True, + "content_mode": "non_video_only", + } + ] + } + ) + + assert config.channels[0].content_mode == "non_video_only" + + +def test_invalid_channel_content_mode_is_rejected(): + with pytest.raises(ConfigError) as exc_info: + _parse_config( + { + "channels": [ + { + "input": "https://odysee.com/@Files:1", + "enabled": True, + "content_mode": "zip_only", + } + ] + } + ) + + assert "content_mode" in str(exc_info.value) + + +def test_channel_content_mode_defaults_to_all(): + channel_config = ChannelConfig(input="https://odysee.com/@Mixed:1") + + assert resolve_content_filter_mode(channel_config) == ( + False, + False, + "all content", + ) + + +def test_channel_content_mode_applies_when_no_cli_override(): + channel_config = ChannelConfig( + input="https://odysee.com/@Videos:1", + content_mode="video_only", + ) + + assert resolve_content_filter_mode(channel_config) == ( + True, + False, + "video-only (channel config)", + ) + + +def test_cli_override_beats_channel_content_mode(): + channel_config = ChannelConfig( + input="https://odysee.com/@Files:1", + content_mode="non_video_only", + ) + + assert resolve_content_filter_mode(channel_config, video_only=True) == ( + True, + False, + "video-only (CLI override)", + ) diff --git a/tests/test_config.py b/tests/test_config.py index 2085f77..d40324a 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -122,6 +122,50 @@ channels: assert "filename_mode" in str(exc_info.value) + def test_valid_direct_retry_config(self, tmp_path): + config_file = tmp_path / "config.yaml" + config_file.write_text(""" +general: + base_dir: "~/test" + direct_base_urls: + - "https://odysee.com/" + - "https://mirror.example/" + direct_max_retries_per_url: 3 + direct_retry_backoff_seconds: 1.5 + direct_auto_fallback_to_p2p: true + +channels: + - input: "https://odysee.com/@Test:1" + enabled: true +""") + + config = load_config(str(config_file)) + + assert config.general.direct_base_urls == [ + "https://odysee.com", + "https://mirror.example", + ] + assert config.general.direct_max_retries_per_url == 3 + assert config.general.direct_retry_backoff_seconds == 1.5 + assert config.general.direct_auto_fallback_to_p2p is True + + def test_invalid_direct_retry_backoff(self, tmp_path): + config_file = tmp_path / "config.yaml" + config_file.write_text(""" +general: + base_dir: "~/test" + direct_retry_backoff_seconds: 0 + +channels: + - input: "https://odysee.com/@Test:1" + enabled: true +""") + + with pytest.raises(ConfigError) as exc_info: + load_config(str(config_file)) + + assert "direct_retry_backoff_seconds" in str(exc_info.value) + class TestDefaultConfig: def test_create_default_config(self): diff --git a/tests/test_content_filters.py b/tests/test_content_filters.py new file mode 100644 index 0000000..11b1420 --- /dev/null +++ b/tests/test_content_filters.py @@ -0,0 +1,58 @@ +"""Tests for CLI content-type filtering.""" + +from main import filter_actions_by_content_type +from models import DownloadAction + + +def make_action(name: str, **metadata): + base_metadata = { + "source_name": name, + "file_name": name, + } + base_metadata.update(metadata) + return DownloadAction( + claim_id=f"claim-{name}", + claim_name=name, + channel_claim_id="channel-1", + version_token=f"version-{name}", + action="download_new", + uri=f"lbry://{name}", + target_dir="/tmp", + metadata=base_metadata, + ) + + +def test_video_only_keeps_only_video_actions(): + video_action = make_action("clip.mp4", source_media_type="video/mp4") + zip_action = make_action("bundle.zip", source_media_type="application/zip") + + actions = filter_actions_by_content_type( + [video_action, zip_action], + video_only=True, + ) + + assert actions == [video_action] + + +def test_non_video_only_keeps_zip_and_skips_video(): + video_action = make_action("clip.mp4", source_media_type="video/mp4") + zip_action = make_action("bundle.zip", source_media_type="application/zip") + + actions = filter_actions_by_content_type( + [video_action, zip_action], + non_video_only=True, + ) + + assert actions == [zip_action] + + +def test_video_filter_falls_back_to_filename_extension(): + video_action = make_action("clip.mkv") + other_action = make_action("bundle.zip") + + actions = filter_actions_by_content_type( + [video_action, other_action], + video_only=True, + ) + + assert actions == [video_action] diff --git a/tests/test_direct_downloader.py b/tests/test_direct_downloader.py new file mode 100644 index 0000000..e2093a3 --- /dev/null +++ b/tests/test_direct_downloader.py @@ -0,0 +1,183 @@ +"""Tests for the direct Odysee downloader.""" + +import logging +from pathlib import Path +from types import SimpleNamespace + +import requests +from direct_downloader import DirectDownloader +from models import Config, DownloadAction + + +def make_action(**metadata_overrides): + metadata = { + "normalized_name": "veritasium-trailer", + "name": "veritasium-trailer", + "source_name": "veritasium-trailer.mp4", + } + metadata.update(metadata_overrides) + + return DownloadAction( + claim_id="19201da2f75357d70b151f7d87ddd94227884126", + claim_name="veritasium-trailer", + channel_claim_id="fb364ef587872515f545a5b4b3182b58073f230f", + version_token="sd_deadbeefdeadbeef", + action="download_new", + uri="lbry://veritasium-trailer#19201da2f75357d70b151f7d87ddd94227884126", + target_dir="/tmp/lbry-test", + metadata=metadata, + ) + + +class TestDirectDownloader: + def test_build_stream_url_uses_normalized_name(self): + downloader = DirectDownloader(Config(), Path("/tmp")) + action = make_action() + + assert ( + downloader.build_stream_url(action) + == "https://odysee.com/$/stream/veritasium-trailer/" + "19201da2f75357d70b151f7d87ddd94227884126" + ) + + def test_build_stream_url_quotes_special_slug(self): + downloader = DirectDownloader(Config(), Path("/tmp")) + action = make_action(normalized_name="Title With Spaces & Symbols") + + assert downloader.build_stream_url(action).startswith( + "https://odysee.com/$/stream/Title%20With%20Spaces%20%26%20Symbols/" + ) + + def test_build_download_url_uses_claim_id(self): + downloader = DirectDownloader(Config(), Path("/tmp")) + action = make_action() + + assert ( + downloader.build_download_url(action) + == "https://odysee.com/$/download/veritasium-trailer/" + "19201da2f75357d70b151f7d87ddd94227884126" + ) + + def test_file_claims_prefer_download_endpoint(self): + downloader = DirectDownloader(Config(), Path("/tmp")) + action = make_action( + source_name="archive.zip", + source_media_type="application/zip", + stream_type="binary", + ) + + assert downloader.build_candidate_urls(action) == [ + "https://odysee.com/$/download/veritasium-trailer/" + "19201da2f75357d70b151f7d87ddd94227884126", + "https://odysee.com/$/stream/veritasium-trailer/" + "19201da2f75357d70b151f7d87ddd94227884126", + ] + + def test_video_claims_prefer_stream_endpoint(self): + downloader = DirectDownloader(Config(), Path("/tmp")) + action = make_action( + source_media_type="video/mp4", + stream_type="video", + ) + + assert downloader.build_candidate_urls(action) == [ + "https://odysee.com/$/stream/veritasium-trailer/" + "19201da2f75357d70b151f7d87ddd94227884126", + "https://odysee.com/$/download/veritasium-trailer/" + "19201da2f75357d70b151f7d87ddd94227884126", + ] + + def test_candidate_urls_include_configured_mirrors(self): + config = Config() + config.general.direct_base_urls = [ + "https://odysee.com", + "https://mirror.example", + ] + downloader = DirectDownloader(config, Path("/tmp")) + action = make_action( + source_media_type="video/mp4", + stream_type="video", + ) + + assert downloader.build_candidate_urls(action) == [ + "https://odysee.com/$/stream/veritasium-trailer/" + "19201da2f75357d70b151f7d87ddd94227884126", + "https://odysee.com/$/download/veritasium-trailer/" + "19201da2f75357d70b151f7d87ddd94227884126", + "https://mirror.example/$/stream/veritasium-trailer/" + "19201da2f75357d70b151f7d87ddd94227884126", + "https://mirror.example/$/download/veritasium-trailer/" + "19201da2f75357d70b151f7d87ddd94227884126", + ] + + def test_429_retries_use_backoff_budget(self, monkeypatch, tmp_path): + config = Config() + config.general.direct_max_retries_per_url = 2 + config.general.direct_retry_backoff_seconds = 0.01 + downloader = DirectDownloader(config, Path("/tmp")) + file_path = tmp_path / "retry-test.mp4" + + calls = {"count": 0} + sleeps = [] + + class FakeResponse: + headers = {} + + def raise_for_status(self): + http_error = requests.exceptions.HTTPError("429") + http_error.response = SimpleNamespace(status_code=429) + raise http_error + + def fake_get(*args, **kwargs): + calls["count"] += 1 + return FakeResponse() + + monkeypatch.setattr(downloader.session, "get", fake_get) + monkeypatch.setattr("direct_downloader.time.sleep", lambda seconds: sleeps.append(seconds)) + + success, reason = downloader._download_file( + "https://odysee.com/$/download/test/abc", + file_path, + "retry-test", + ) + + assert success is False + assert "rate-limited" in reason.lower() + assert calls["count"] == 3 + assert sleeps == [0.01, 0.02] + + def test_skip_existing_logs_existing_file_path(self, caplog): + downloader = DirectDownloader(Config(), Path("/tmp")) + action = make_action( + existing_file_relpath="channels/test/claims/item/versions/sd/file.zip", + ) + action.action = "skip_existing" + + with caplog.at_level(logging.INFO): + success = downloader.download(action) + + assert success is True + assert "Skipping existing file" in caplog.text + assert "/tmp/channels/test/claims/item/versions/sd/file.zip" in caplog.text + + def test_download_logs_destination_path(self, monkeypatch, caplog, tmp_path): + downloader = DirectDownloader(Config(), Path("/tmp")) + action = make_action() + action.target_dir = str(tmp_path) + + monkeypatch.setattr( + downloader, + "_download_file", + lambda url, file_path, claim_name: (True, None), + ) + monkeypatch.setattr(downloader, "_write_metadata", lambda *args, **kwargs: None) + monkeypatch.setattr(downloader, "_write_checksum", lambda *args, **kwargs: None) + target_file = tmp_path / "veritasium-trailer.mp4" + target_file.write_bytes(b"data") + + with caplog.at_level(logging.INFO): + success = downloader.download(action) + + assert success is True + assert "Saving to:" in caplog.text + assert str(target_file) in caplog.text diff --git a/tests/test_planner.py b/tests/test_planner.py index 8013a5d..a33bf73 100644 --- a/tests/test_planner.py +++ b/tests/test_planner.py @@ -104,6 +104,10 @@ class TestPlanner: assert len(actions) == 1 assert actions[0].action == "skip_existing" + assert ( + actions[0].metadata["existing_file_relpath"] + == f"channels/Test__channel123/claims/{claim_folder}/versions/{version_token}/video.mp4" + ) def test_new_version_detected(self): channel = Channel( diff --git a/tests/test_transport_mode.py b/tests/test_transport_mode.py new file mode 100644 index 0000000..8117b9b --- /dev/null +++ b/tests/test_transport_mode.py @@ -0,0 +1,15 @@ +"""Tests for transport mode selection.""" + +from main import resolve_transport_mode + + +def test_transport_mode_defaults_to_direct(): + assert resolve_transport_mode() == (True, "direct") + + +def test_transport_mode_p2p_override_wins(): + assert resolve_transport_mode(p2p=True) == (False, "p2p") + + +def test_transport_mode_explicit_direct(): + assert resolve_transport_mode(direct=True) == (True, "direct") diff --git a/utils.py b/utils.py index d91d204..6c63bcd 100644 --- a/utils.py +++ b/utils.py @@ -256,6 +256,7 @@ def extract_claim_metadata(claim_data: Dict[str, Any]) -> Dict[str, Any]: "normalized_name": claim.get("normalized_name"), "claim_type": claim.get("claim_type"), "value_type": claim.get("value_type"), + "stream_type": value.get("stream_type"), "permanent_url": claim.get("permanent_url"), "short_url": claim.get("short_url"), "canonical_url": claim.get("canonical_url"),