mirror of
https://github.com/PR0M3TH3AN/LBRY-Downloader.git
synced 2026-10-27 08:22:24 +00:00
Add direct-first download modes and resilience controls
This commit is contained in:
thePR0M3TH3AN committed
1 parent
66d7a02458
commit
7b50110f2a
22 files changed
+1235
-448
No files matched your search
+65
-22
@@ -32,7 +32,7 @@ cd LBRY-Downloader
|
||||
|
||||
This script will:
|
||||
1. Check and install system dependencies
|
||||
2. Download and install LBRY SDK (lbrynet) - **No blockchain sync required!**
|
||||
2. Set up the downloader for direct mode by default
|
||||
3. Set up a Python virtual environment
|
||||
4. Install Python packages
|
||||
5. Ask for your download location
|
||||
@@ -42,11 +42,13 @@ This script will:
|
||||
|
||||
**Want to uninstall later?** Just run `./uninstall.py`
|
||||
|
||||
### About LBRY SDK (lbrynet) - IMPORTANT
|
||||
### About LBRY SDK (lbrynet) - Optional P2P Mode Only
|
||||
|
||||
**Q: Does this download the LBRY blockchain?**
|
||||
|
||||
**A:** The LBRY SDK downloads **blockchain headers** (not full blocks), which is required to function:
|
||||
**A:** Only if you choose to use `--p2p`. The default direct mode does not require this.
|
||||
|
||||
When you do use `--p2p`, the LBRY SDK downloads **blockchain headers** (not full blocks):
|
||||
|
||||
**First-time setup (one-time):**
|
||||
- Downloads ~1.7 million block headers
|
||||
@@ -61,7 +63,7 @@ This script will:
|
||||
**What this means:**
|
||||
- The LBRY SDK needs this data to verify which content claims are valid
|
||||
- This is lighter than Bitcoin's full blockchain sync
|
||||
- But it IS required and will take time on first launch
|
||||
- But it is only required for the optional P2P path
|
||||
- You can stop it with `Ctrl+C` or `lbrynet stop`
|
||||
|
||||
**Privacy note:** This connects to LBRY's network and syncs blockchain data to verify content. This is how the LBRY protocol works.
|
||||
@@ -94,9 +96,9 @@ python3 --version # Should be 3.8 or higher
|
||||
pip3 --version
|
||||
```
|
||||
|
||||
### 2. Install LBRY SDK (lbrynet daemon)
|
||||
### 2. Optional: Install LBRY SDK (`lbrynet`) for P2P Mode
|
||||
|
||||
The LBRY SDK is required to communicate with the LBRY network.
|
||||
You only need the LBRY SDK if you want to run the downloader with `--p2p`. The default direct mode does not require it.
|
||||
|
||||
#### Option A: Download Pre-built Binary (Recommended)
|
||||
|
||||
@@ -250,18 +252,22 @@ channels:
|
||||
# Technology channels
|
||||
- input: "https://odysee.com/@LinusTechTips:1"
|
||||
enabled: true
|
||||
content_mode: "video_only"
|
||||
|
||||
# Educational content
|
||||
- input: "https://odysee.com/@veritasium:1"
|
||||
enabled: true
|
||||
content_mode: "video_only"
|
||||
|
||||
# News/Commentary
|
||||
- input: "https://odysee.com/@Timcast:1"
|
||||
enabled: true
|
||||
content_mode: "video_only"
|
||||
|
||||
# Documentaries
|
||||
# File / document archives
|
||||
- input: "https://odysee.com/@DocumentaryLab:1"
|
||||
enabled: true
|
||||
content_mode: "non_video_only"
|
||||
```
|
||||
|
||||
#### URL Formats Supported
|
||||
@@ -300,15 +306,25 @@ Add your channels under the `channels:` section:
|
||||
channels:
|
||||
- input: "https://odysee.com/@YourFavoriteChannel:1"
|
||||
enabled: true
|
||||
content_mode: "non_video_only"
|
||||
tags_include: []
|
||||
tags_exclude: []
|
||||
|
||||
- input: "https://odysee.com/@AnotherChannel:5"
|
||||
enabled: true
|
||||
content_mode: "video_only"
|
||||
tags_include: []
|
||||
tags_exclude: []
|
||||
```
|
||||
|
||||
`content_mode` controls what each channel downloads:
|
||||
|
||||
- `all`: video and non-video files
|
||||
- `video_only`: only videos such as mp4/webm/mkv
|
||||
- `non_video_only`: only file bundles/docs such as zip/pdf
|
||||
|
||||
If you omit it, the default is `all`.
|
||||
|
||||
### 4. Configure Per-Channel Download Locations (Optional)
|
||||
|
||||
By default, all downloads go to `~/Documents/lbry-downloads/channels/`. You can specify custom locations for each channel:
|
||||
@@ -318,16 +334,19 @@ channels:
|
||||
# Downloads to default location
|
||||
- input: "https://odysee.com/@RegularChannel:1"
|
||||
enabled: true
|
||||
content_mode: "all"
|
||||
|
||||
# Downloads to custom location (external drive)
|
||||
# Downloads only videos to a custom location (external drive)
|
||||
- input: "https://odysee.com/@LargeVideos:1"
|
||||
enabled: true
|
||||
download_path: "/mnt/media/videos"
|
||||
content_mode: "video_only"
|
||||
|
||||
# Downloads to custom location (home directory)
|
||||
# Downloads only non-video files to a custom location
|
||||
- input: "https://odysee.com/@Music:1"
|
||||
enabled: true
|
||||
download_path: "~/Music/LBRY"
|
||||
content_mode: "non_video_only"
|
||||
```
|
||||
|
||||
**Why use custom locations?**
|
||||
@@ -363,7 +382,7 @@ general:
|
||||
```
|
||||
|
||||
The tool downloads **most recent items first** (sorted by publish date), so:
|
||||
- `download_limit: 10` = Download the 10 newest videos
|
||||
- `download_limit: 10` = Download the 10 newest matching items for that channel
|
||||
- Older items remain in the channel but won't be downloaded yet
|
||||
- On next run, it will download the next 10 (if new items were published)
|
||||
|
||||
@@ -382,7 +401,7 @@ Once you're happy with how it works, increase the limit or set to 0.
|
||||
|
||||
## Running the Daemon
|
||||
|
||||
The LBRY daemon (`lbrynet`) must be running before using the downloader.
|
||||
The LBRY daemon (`lbrynet`) is only required if you want to use `--p2p`. The default direct mode does not require it.
|
||||
|
||||
### Start the Daemon
|
||||
|
||||
@@ -456,16 +475,40 @@ python3 main.py --dry-run
|
||||
|
||||
This shows what would be downloaded without actually downloading anything.
|
||||
|
||||
If you mostly want zip/doc/file claims and want to skip videos, the most useful first test is:
|
||||
|
||||
```bash
|
||||
python3 main.py --dry-run --non-video-only
|
||||
```
|
||||
|
||||
### Normal Run
|
||||
|
||||
```bash
|
||||
# Make sure lbrynet is running
|
||||
lbrynet status
|
||||
|
||||
# Run the downloader
|
||||
# Run the downloader using direct mode (default) and each channel's configured content_mode
|
||||
python3 main.py
|
||||
```
|
||||
|
||||
### Useful Run Variants
|
||||
|
||||
```bash
|
||||
# Recommended default if you mainly want non-video files such as zip bundles
|
||||
python3 main.py --non-video-only
|
||||
|
||||
# Force video-only mode for every channel for this run
|
||||
python3 main.py --video-only
|
||||
|
||||
# Force non-video-only mode for every channel for this run
|
||||
python3 main.py --non-video-only
|
||||
|
||||
# Use the local LBRY daemon and peer-to-peer mode instead of direct mode
|
||||
python3 main.py --p2p
|
||||
|
||||
# Use P2P mode with a run-wide content override
|
||||
python3 main.py --p2p --non-video-only
|
||||
```
|
||||
|
||||
CLI flags `--video-only` and `--non-video-only` override each channel's `content_mode` for that run only. `--p2p` overrides the default direct transport mode for that run only.
|
||||
|
||||
### Custom Config Location
|
||||
|
||||
```bash
|
||||
@@ -590,7 +633,7 @@ lbc_mainnet: added BlockHeightEvent(height=1665210, change=2001)
|
||||
|
||||
**What this means:**
|
||||
- The LBRY SDK is downloading blockchain headers (not full blocks)
|
||||
- This is required to verify content claims
|
||||
- This is only required when using `--p2p` to verify content claims
|
||||
- Takes 5-10 minutes on first run
|
||||
- Subsequent runs are instant
|
||||
|
||||
@@ -859,9 +902,9 @@ cd ~/Documents/LBRY-Downloader
|
||||
./bin/lbry-test # Automatically runs with --dry-run
|
||||
```
|
||||
|
||||
### Prerequisites: Start the Daemon
|
||||
### Optional Prerequisite: Start the Daemon for P2P Mode
|
||||
|
||||
**Important:** Always start the LBRY daemon before running the downloader:
|
||||
Only do this if you plan to run with `--p2p`:
|
||||
|
||||
```bash
|
||||
# Check if daemon is running
|
||||
@@ -886,14 +929,14 @@ pip3 install --user -r requirements.txt
|
||||
python3 init.py
|
||||
|
||||
# Daily use
|
||||
lbrynet start # Start daemon
|
||||
~/Documents/LBRY-Downloader/bin/lbry-downloader --dry-run # Test first
|
||||
~/Documents/LBRY-Downloader/bin/lbry-downloader # Run sync
|
||||
lbrynet stop # Stop daemon (optional)
|
||||
~/Documents/LBRY-Downloader/bin/lbry-downloader --non-video-only # Recommended direct-mode run
|
||||
~/Documents/LBRY-Downloader/bin/lbry-downloader --p2p # Optional local-node mode
|
||||
|
||||
# Or if in PATH
|
||||
lbry-downloader --dry-run
|
||||
lbry-downloader
|
||||
lbry-downloader --non-video-only
|
||||
lbry-downloader --p2p
|
||||
|
||||
# Locations
|
||||
Install: ~/Documents/LBRY-Downloader/
|
||||
|
||||
+38
-3
@@ -39,17 +39,36 @@ Edit `~/Documents/lbry-downloads/config.yaml`:
|
||||
channels:
|
||||
- input: "https://odysee.com/@YourFavoriteChannel:1"
|
||||
enabled: true
|
||||
content_mode: "non_video_only"
|
||||
- input: "https://odysee.com/@AnotherChannel:5"
|
||||
enabled: true
|
||||
content_mode: "video_only"
|
||||
```
|
||||
|
||||
**Tip:** Just paste the Odysee URL exactly as it appears in your browser!
|
||||
|
||||
### Per-Channel Content Mode
|
||||
|
||||
You can choose what each channel downloads:
|
||||
|
||||
- `content_mode: "all"` downloads video and non-video files
|
||||
- `content_mode: "video_only"` downloads only video files
|
||||
- `content_mode: "non_video_only"` downloads only zip/PDF/other non-video files
|
||||
|
||||
Most users who want file bundles and archives should start with:
|
||||
|
||||
```yaml
|
||||
channels:
|
||||
- input: "https://odysee.com/@YourFavoriteChannel:1"
|
||||
enabled: true
|
||||
content_mode: "non_video_only"
|
||||
```
|
||||
|
||||
## Launching
|
||||
|
||||
### Start the LBRY Daemon First
|
||||
### Optional: Start the LBRY Daemon First
|
||||
|
||||
**Important:** The LBRY daemon must be running before using the downloader:
|
||||
You only need the LBRY daemon if you plan to use `--p2p`. The default direct mode does not require it.
|
||||
|
||||
```bash
|
||||
# Start the daemon
|
||||
@@ -90,9 +109,21 @@ python main.py --dry-run
|
||||
# Test what would be downloaded (dry run) - ALWAYS DO THIS FIRST
|
||||
lbry-downloader --dry-run
|
||||
|
||||
# Actually download content
|
||||
# Recommended default: download only non-video files such as zip bundles
|
||||
lbry-downloader --non-video-only
|
||||
|
||||
# Actually download using each channel's configured content_mode
|
||||
lbry-downloader
|
||||
|
||||
# Download only videos for every channel for this run
|
||||
lbry-downloader --video-only
|
||||
|
||||
# Download only non-video files for every channel for this run
|
||||
lbry-downloader --non-video-only
|
||||
|
||||
# Use the local LBRY node instead of the default direct mode
|
||||
lbry-downloader --p2p
|
||||
|
||||
# Use custom config file
|
||||
lbry-downloader --config ./my-config.yaml
|
||||
|
||||
@@ -100,6 +131,8 @@ lbry-downloader --config ./my-config.yaml
|
||||
lbry-downloader --help
|
||||
```
|
||||
|
||||
CLI flags `--video-only` and `--non-video-only` override per-channel `content_mode` for that run only. `--p2p` overrides the default direct transport mode for that run only.
|
||||
|
||||
## Understanding Output
|
||||
|
||||
```
|
||||
@@ -160,6 +193,8 @@ Downloads are saved to:
|
||||
lbrynet start
|
||||
```
|
||||
|
||||
This only applies when you are using `--p2p`. Default direct mode does not require `lbrynet`.
|
||||
|
||||
**"Config file not found"**
|
||||
```bash
|
||||
python init.py
|
||||
|
||||
@@ -2,13 +2,23 @@
|
||||
|
||||
A Python tool that incrementally syncs downloadable file claims from configured LBRY/Odysee channels into a structured local archive.
|
||||
|
||||
## Recommended Defaults
|
||||
|
||||
For most users:
|
||||
|
||||
- Files only: `lbry-downloader --non-video-only`
|
||||
- Videos only: `lbry-downloader --video-only`
|
||||
- Mixed per-channel behavior: set `content_mode` in `config.yaml` and run `lbry-downloader`
|
||||
- Optional local-node mode: `lbry-downloader --p2p`
|
||||
|
||||
## Features
|
||||
|
||||
- **Incremental Sync**: Only downloads new claims and updated versions
|
||||
- **Version Preservation**: Keeps old versions instead of overwriting them
|
||||
- **Robust Identity**: Uses stable LBRY claim IDs, not display names or titles
|
||||
- **Atomic State**: Database writes are atomic to prevent corruption
|
||||
- **Daemon-Based**: Talks directly to local `lbrynet` daemon via JSON-RPC
|
||||
- **Direct Mode by Default**: Uses Odysee's public proxy + CDN without requiring a local node
|
||||
- **Optional P2P Mode**: Can use a local `lbrynet` daemon when explicitly requested
|
||||
- **Streaming Fallback**: Automatically uses daemon's streaming endpoint when P2P peers are unavailable
|
||||
- **Flexible Input**: Accepts Odysee URLs, LBRY URIs, or claim IDs
|
||||
- **Audit Trail**: Maintains run history in JSONL format
|
||||
@@ -41,21 +51,22 @@ cd LBRY-Downloader
|
||||
|
||||
The setup script will:
|
||||
- ✅ Check and install system dependencies
|
||||
- ✅ Download and install LBRY SDK (lbrynet) - **No blockchain sync required!**
|
||||
- ✅ Set up a Python virtual environment
|
||||
- ✅ Install Python packages
|
||||
- ✅ Ask for your download location
|
||||
- ✅ Configure channels to download from
|
||||
- ✅ Set download limits
|
||||
- ✅ Create launcher scripts
|
||||
- ✅ Prepare a direct-first downloader setup that works without a local node
|
||||
- ✅ Leave `lbrynet` as an optional install only for users who want `--p2p`
|
||||
|
||||
**Important:** The LBRY SDK (lbrynet) requires syncing blockchain **headers** (not full blocks) to function. On first run:
|
||||
**Important:** Direct mode is now the default and does not require `lbrynet`.
|
||||
|
||||
If you choose to use `--p2p`, then `lbrynet` requires syncing blockchain **headers** (not full blocks) on first use:
|
||||
- Downloads ~1.7 million block headers (takes 5-10 minutes)
|
||||
- Uses minimal storage (headers only, not full blockchain)
|
||||
- Subsequent starts are instant
|
||||
- This is required to verify content claims on the network
|
||||
|
||||
This is lighter than Bitcoin's full blockchain sync, but still necessary for the downloader to work.
|
||||
- Subsequent starts are much faster
|
||||
- This is only needed for the optional node-backed mode
|
||||
|
||||
### Manual Install
|
||||
|
||||
@@ -71,7 +82,9 @@ To remove LBRY Downloader:
|
||||
|
||||
## Prerequisites
|
||||
|
||||
You need to have the LBRY SDK daemon (`lbrynet`) installed and running:
|
||||
Direct mode is the default, so a local LBRY daemon is not required for normal use.
|
||||
|
||||
If you want to use P2P mode explicitly with `--p2p`, then you need to have the LBRY SDK daemon (`lbrynet`) installed and running:
|
||||
|
||||
```bash
|
||||
# The daemon should be accessible at http://127.0.0.1:5279
|
||||
@@ -104,12 +117,19 @@ general:
|
||||
max_workers: 2
|
||||
dry_run: false
|
||||
include_reposts: false
|
||||
direct_base_urls:
|
||||
- "https://odysee.com"
|
||||
direct_max_retries_per_url: 2
|
||||
direct_retry_backoff_seconds: 2.0
|
||||
direct_auto_fallback_to_p2p: false
|
||||
|
||||
channels:
|
||||
- input: "https://odysee.com/@SomeChannel:1"
|
||||
enabled: true
|
||||
content_mode: "all"
|
||||
- input: "lbry://@AnotherChannel#5"
|
||||
enabled: true
|
||||
content_mode: "non_video_only"
|
||||
```
|
||||
|
||||
### Finding Channel URLs
|
||||
@@ -183,6 +203,36 @@ channels:
|
||||
# No download_path - uses general.base_dir/channels/
|
||||
```
|
||||
|
||||
### Per-Channel Content Mode
|
||||
|
||||
You can control the download type for each channel independently.
|
||||
|
||||
```yaml
|
||||
channels:
|
||||
- input: "https://odysee.com/@VideoChannel:1"
|
||||
enabled: true
|
||||
content_mode: "video_only"
|
||||
|
||||
- input: "https://odysee.com/@FileArchive:1"
|
||||
enabled: true
|
||||
content_mode: "non_video_only"
|
||||
|
||||
- input: "https://odysee.com/@MixedChannel:1"
|
||||
enabled: true
|
||||
content_mode: "all"
|
||||
```
|
||||
|
||||
Available values:
|
||||
|
||||
- `all`: Download both video and non-video files for that channel.
|
||||
- `video_only`: Download only video files such as `.mp4`, `.webm`, and `.mkv`.
|
||||
- `non_video_only`: Download only non-video files such as `.zip`, PDFs, and similar file claims.
|
||||
|
||||
Precedence:
|
||||
|
||||
- Per-channel `content_mode` is the normal default for that channel.
|
||||
- CLI flags `--video-only` and `--non-video-only` override all channels for a single run.
|
||||
|
||||
### Download Limit
|
||||
|
||||
Limit how many new downloads per channel (downloads most recent first):
|
||||
@@ -201,6 +251,33 @@ This is useful for:
|
||||
- **Large channels**: Gradually sync channels with thousands of videos
|
||||
- **Bandwidth**: Control how much data is downloaded per run
|
||||
|
||||
### Direct-Mode Rate Limit Fallbacks
|
||||
|
||||
If Odysee starts returning `429 Too Many Requests`, you can tune the direct downloader instead of hardcoding random public mirrors.
|
||||
|
||||
```yaml
|
||||
general:
|
||||
direct_base_urls:
|
||||
- "https://odysee.com"
|
||||
# - "https://your-mirror.example"
|
||||
direct_max_retries_per_url: 2
|
||||
direct_retry_backoff_seconds: 2.0
|
||||
direct_auto_fallback_to_p2p: false
|
||||
```
|
||||
|
||||
What these do:
|
||||
|
||||
- `direct_base_urls`: direct download bases to try in order. Odysee should normally stay first.
|
||||
- `direct_max_retries_per_url`: how many retries to allow after HTTP `429` for the same direct URL.
|
||||
- `direct_retry_backoff_seconds`: base delay for exponential backoff after HTTP `429`.
|
||||
- `direct_auto_fallback_to_p2p`: if `true`, a direct-mode run will try the local `lbrynet` daemon after repeated rate-limit failures.
|
||||
|
||||
Recommended pattern:
|
||||
|
||||
- Keep `https://odysee.com` first.
|
||||
- Add your own mirror only if you actually control or trust it.
|
||||
- Turn on `direct_auto_fallback_to_p2p: true` only if you also maintain a working local node.
|
||||
|
||||
## Launching the Application
|
||||
|
||||
After installation, you can launch LBRY Downloader in several ways:
|
||||
@@ -248,7 +325,7 @@ Now you can run `lbry-downloader` from any directory!
|
||||
|
||||
### Prerequisites Before Running
|
||||
|
||||
**Important:** Make sure the LBRY daemon is running first:
|
||||
**Important:** If you are using `--p2p`, make sure the LBRY daemon is running first:
|
||||
|
||||
```bash
|
||||
# Start the daemon
|
||||
@@ -260,15 +337,70 @@ lbrynet status
|
||||
|
||||
## Usage
|
||||
|
||||
### Recommended Default: Direct + Non-Video Downloads Only
|
||||
|
||||
If your main goal is to archive file attachments such as `.zip` bundles and skip video files such as `.mp4`, this should be the primary command you use:
|
||||
|
||||
```bash
|
||||
# Run with default config
|
||||
python main.py
|
||||
python3 main.py --non-video-only
|
||||
```
|
||||
|
||||
# Use custom config
|
||||
python main.py --config ./my-config.yaml
|
||||
This mode keeps non-video claim downloads and skips video claims before the per-channel download limit is applied.
|
||||
|
||||
# Dry run - see what would happen
|
||||
python main.py --dry-run
|
||||
### Common Commands
|
||||
|
||||
```bash
|
||||
# Recommended default: download only non-video files such as zip archives
|
||||
python3 main.py --non-video-only
|
||||
|
||||
# Download only video files such as mp4/webm/mkv
|
||||
python3 main.py --video-only
|
||||
|
||||
# Download everything supported by the tool
|
||||
python3 main.py
|
||||
|
||||
# Preview what would be downloaded without downloading anything
|
||||
python3 main.py --dry-run
|
||||
|
||||
# Preview only non-video downloads
|
||||
python3 main.py --dry-run --non-video-only
|
||||
|
||||
# Use a custom config file
|
||||
python3 main.py --config ./my-config.yaml
|
||||
|
||||
# Run explicitly in direct mode (normally unnecessary because this is the default)
|
||||
python3 main.py --direct
|
||||
|
||||
# Use the local LBRY daemon and P2P network instead of direct mode
|
||||
python3 main.py --p2p
|
||||
|
||||
# Use P2P mode but keep only non-video files
|
||||
python3 main.py --p2p --non-video-only
|
||||
|
||||
# Use P2P mode but keep only video files
|
||||
python3 main.py --p2p --video-only
|
||||
```
|
||||
|
||||
### Flag Reference
|
||||
|
||||
- `--non-video-only`: Download only non-video claims such as `.zip`, PDFs, and other non-video files.
|
||||
- `--video-only`: Download only video claims such as `.mp4`, `.webm`, and `.mkv`.
|
||||
- `--dry-run`: Show what would be downloaded without changing anything.
|
||||
- `--direct`: Use direct mode explicitly. This is already the default.
|
||||
- `--p2p`: Use the local LBRY daemon and peer-to-peer network instead of the default direct mode.
|
||||
- `--config PATH`: Use a config file other than `~/Documents/lbry-downloads/config.yaml`.
|
||||
|
||||
If you configure `content_mode` per channel, you usually only need the CLI content flags when you want a temporary run-wide override.
|
||||
|
||||
### Running Through the Launcher
|
||||
|
||||
If you installed the launcher script, the same commands work there too:
|
||||
|
||||
```bash
|
||||
lbry-downloader --non-video-only
|
||||
lbry-downloader --video-only
|
||||
lbry-downloader --dry-run --non-video-only
|
||||
lbry-downloader --p2p --non-video-only
|
||||
```
|
||||
|
||||
## Directory Structure
|
||||
@@ -319,15 +451,27 @@ Make sure `lbrynet` is running:
|
||||
lbrynet start
|
||||
```
|
||||
|
||||
This only applies when you are using `--p2p`. Direct mode does not require a local daemon.
|
||||
|
||||
### "Config file not found"
|
||||
Create a config at the default location or specify a path:
|
||||
```bash
|
||||
python main.py --config ./config.yaml
|
||||
python3 main.py --config ./config.yaml
|
||||
```
|
||||
|
||||
### Dry run shows no downloads
|
||||
Check that your channels are enabled and have downloadable content (streams, not just reposts).
|
||||
|
||||
### HTTP 429 / Too Many Requests
|
||||
|
||||
If direct mode is being rate-limited:
|
||||
|
||||
1. Reduce how aggressively you rerun the downloader.
|
||||
2. Increase `direct_retry_backoff_seconds`.
|
||||
3. Keep `direct_max_retries_per_url` modest, such as `2` or `3`.
|
||||
4. Add a trusted mirror to `direct_base_urls` if you actually have one.
|
||||
5. Enable `direct_auto_fallback_to_p2p: true` if you want automatic fallback to the local node when direct mode keeps hitting `429`.
|
||||
|
||||
## Architecture
|
||||
|
||||
- `main.py` - Entry point and orchestration
|
||||
@@ -341,12 +485,12 @@ Check that your channels are enabled and have downloadable content (streams, not
|
||||
|
||||
## How Downloads Work
|
||||
|
||||
The downloader uses a two-stage approach for maximum reliability:
|
||||
The downloader supports two transport modes:
|
||||
|
||||
1. **P2P Attempt**: First tries to download from LBRY's peer-to-peer network
|
||||
2. **Streaming Fallback**: If P2P peers are unavailable (common for older content), automatically falls back to the daemon's local streaming endpoint
|
||||
1. **Direct mode (default)**: Uses Odysee's public proxy for metadata and Odysee CDN for the actual download
|
||||
2. **P2P mode (`--p2p`)**: Uses the local LBRY daemon and peer-to-peer network, with daemon streaming fallback when peers are unavailable
|
||||
|
||||
This ensures content downloads successfully even when the P2P network has no seeders.
|
||||
Direct mode is the intended default for most users. P2P mode remains available when you specifically want node-backed downloading.
|
||||
|
||||
## License
|
||||
|
||||
|
||||
+32
-2
@@ -4,8 +4,10 @@
|
||||
|
||||
lbrynet:
|
||||
# URL of the local LBRY daemon (lbrynet)
|
||||
# Only used when running with --p2p
|
||||
api_url: "http://127.0.0.1:5279"
|
||||
# Timeout for daemon requests in seconds
|
||||
# Only used when running with --p2p
|
||||
timeout_seconds: 60
|
||||
|
||||
general:
|
||||
@@ -31,6 +33,18 @@ general:
|
||||
channel_page_size: 50
|
||||
# Keep records of claims even if they disappear from the channel
|
||||
keep_missing_claim_records: true
|
||||
# Direct download base URLs to try in order. Keep Odysee first unless you
|
||||
# explicitly want a mirror-first strategy.
|
||||
direct_base_urls:
|
||||
- "https://odysee.com"
|
||||
# - "https://your-mirror.example"
|
||||
# How many times to retry the same direct URL after HTTP 429
|
||||
direct_max_retries_per_url: 2
|
||||
# Base delay in seconds for exponential backoff after HTTP 429
|
||||
direct_retry_backoff_seconds: 2.0
|
||||
# If true, direct-mode runs will try the local lbrynet daemon as a fallback
|
||||
# after repeated rate-limit failures. Requires a working local daemon.
|
||||
direct_auto_fallback_to_p2p: false
|
||||
|
||||
# Download limit: Maximum number of downloads per channel per run
|
||||
#
|
||||
@@ -67,20 +81,36 @@ channels:
|
||||
# - enabled: (optional) true/false - default: true
|
||||
# - download_path: (optional) Custom download location for this channel
|
||||
# If not set, uses general.base_dir/channels/
|
||||
# - content_mode: (optional) What file types to download for this channel
|
||||
# Values:
|
||||
# - all = video and non-video files
|
||||
# - video_only = only video files such as mp4/webm/mkv
|
||||
# - non_video_only = only non-video files such as zip/pdf
|
||||
# Default: all
|
||||
# Note: CLI flags --video-only and --non-video-only override
|
||||
# all channel content_mode settings for that run.
|
||||
# - tags_include: (optional) List of tags to include (empty = all)
|
||||
# - tags_exclude: (optional) List of tags to exclude
|
||||
#
|
||||
# Example 1: Basic channel
|
||||
# - input: "https://odysee.com/@SomeChannel:1"
|
||||
# enabled: true
|
||||
# content_mode: "all"
|
||||
#
|
||||
# Example 2: Channel with custom download location
|
||||
# Example 2: Channel with custom download location and only videos
|
||||
# - input: "https://odysee.com/@TechChannel:1"
|
||||
# enabled: true
|
||||
# download_path: "/mnt/media/videos/tech"
|
||||
# content_mode: "video_only"
|
||||
#
|
||||
# Example 3: Channel with tag filtering
|
||||
# Example 3: Channel that downloads only file bundles, docs, and other non-video claims
|
||||
# - input: "https://odysee.com/@FileArchive:1"
|
||||
# enabled: true
|
||||
# content_mode: "non_video_only"
|
||||
#
|
||||
# Example 4: Channel with tag filtering
|
||||
# - input: "lbry://@AnotherChannel#5"
|
||||
# enabled: true
|
||||
# content_mode: "all"
|
||||
# tags_include: ["documentary", "science"]
|
||||
# tags_exclude: ["shorts"]
|
||||
@@ -20,6 +20,9 @@ class ConfigError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
VALID_CONTENT_MODES = {"all", "video_only", "non_video_only"}
|
||||
|
||||
|
||||
def load_config(config_path: Optional[str] = None) -> Config:
|
||||
"""
|
||||
Load and validate the configuration file.
|
||||
@@ -102,6 +105,21 @@ def _parse_config(raw: Dict[str, Any]) -> Config:
|
||||
download_limit=general_raw.get(
|
||||
"download_limit", config.general.download_limit
|
||||
),
|
||||
direct_base_urls=general_raw.get(
|
||||
"direct_base_urls", config.general.direct_base_urls
|
||||
),
|
||||
direct_max_retries_per_url=general_raw.get(
|
||||
"direct_max_retries_per_url",
|
||||
config.general.direct_max_retries_per_url,
|
||||
),
|
||||
direct_retry_backoff_seconds=general_raw.get(
|
||||
"direct_retry_backoff_seconds",
|
||||
config.general.direct_retry_backoff_seconds,
|
||||
),
|
||||
direct_auto_fallback_to_p2p=general_raw.get(
|
||||
"direct_auto_fallback_to_p2p",
|
||||
config.general.direct_auto_fallback_to_p2p,
|
||||
),
|
||||
)
|
||||
|
||||
# Validate and expand paths
|
||||
@@ -121,6 +139,22 @@ def _parse_config(raw: Dict[str, Any]) -> Config:
|
||||
# 0 means "all" (no limit), negative values are not allowed
|
||||
if config.general.download_limit < 0:
|
||||
raise ConfigError("download_limit must be 0 (for all) or a positive number")
|
||||
if config.general.direct_max_retries_per_url < 0:
|
||||
raise ConfigError("direct_max_retries_per_url must be 0 or a positive number")
|
||||
if config.general.direct_retry_backoff_seconds <= 0:
|
||||
raise ConfigError("direct_retry_backoff_seconds must be greater than 0")
|
||||
if not isinstance(config.general.direct_base_urls, list):
|
||||
raise ConfigError("direct_base_urls must be a list")
|
||||
if not config.general.direct_base_urls:
|
||||
raise ConfigError("direct_base_urls must contain at least one base URL")
|
||||
normalized_direct_base_urls = []
|
||||
for idx, base_url in enumerate(config.general.direct_base_urls, start=1):
|
||||
if not isinstance(base_url, str) or not base_url.strip():
|
||||
raise ConfigError(
|
||||
f"direct_base_urls entry {idx} must be a non-empty string"
|
||||
)
|
||||
normalized_direct_base_urls.append(base_url.rstrip("/"))
|
||||
config.general.direct_base_urls = normalized_direct_base_urls
|
||||
|
||||
# Validate log_level
|
||||
valid_levels = ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]
|
||||
@@ -175,10 +209,18 @@ def _parse_channels(channels_raw: List[Dict[str, Any]]) -> List[ChannelConfig]:
|
||||
input=input_val,
|
||||
enabled=channel_raw.get("enabled", True),
|
||||
download_path=download_path,
|
||||
content_mode=channel_raw.get("content_mode", "all"),
|
||||
tags_include=channel_raw.get("tags_include", []),
|
||||
tags_exclude=channel_raw.get("tags_exclude", []),
|
||||
)
|
||||
|
||||
if channel.content_mode not in VALID_CONTENT_MODES:
|
||||
raise ConfigError(
|
||||
f"Channel {i + 1} has invalid content_mode "
|
||||
f"'{channel.content_mode}'. Must be one of: "
|
||||
f"{', '.join(sorted(VALID_CONTENT_MODES))}"
|
||||
)
|
||||
|
||||
channels.append(channel)
|
||||
|
||||
return channels
|
||||
@@ -190,30 +232,58 @@ def create_default_config() -> str:
|
||||
# Place this file at ~/Documents/lbry-downloads/config.yaml
|
||||
|
||||
lbrynet:
|
||||
# URL of the local LBRY daemon (lbrynet)
|
||||
# Only used when running with --p2p
|
||||
api_url: "http://127.0.0.1:5279"
|
||||
# Timeout for daemon requests in seconds
|
||||
# Only used when running with --p2p
|
||||
timeout_seconds: 60
|
||||
|
||||
general:
|
||||
# Base directory for all downloads and state
|
||||
base_dir: "~/Documents/lbry-downloads"
|
||||
# Path to the state database
|
||||
state_file: "~/Documents/lbry-downloads/state/database.json"
|
||||
# Number of concurrent downloads
|
||||
max_workers: 2
|
||||
# Logging level: DEBUG, INFO, WARNING, ERROR, CRITICAL
|
||||
log_level: "INFO"
|
||||
# If true, show what would be downloaded without actually downloading
|
||||
dry_run: false
|
||||
# Verify that previously downloaded files still exist
|
||||
verify_existing_files: true
|
||||
# Write SHA256 checksums for downloaded files
|
||||
write_checksums: true
|
||||
# Filename handling: 'original' or 'safe'
|
||||
filename_mode: "original"
|
||||
# Include reposts in downloads
|
||||
include_reposts: false
|
||||
# Number of claims to fetch per page
|
||||
channel_page_size: 50
|
||||
# Keep records of claims even if they disappear from the channel
|
||||
keep_missing_claim_records: true
|
||||
# Direct download bases to try in order
|
||||
direct_base_urls:
|
||||
- "https://odysee.com"
|
||||
# Retries per direct URL after HTTP 429
|
||||
direct_max_retries_per_url: 2
|
||||
# Base backoff in seconds after HTTP 429 (uses exponential backoff)
|
||||
direct_retry_backoff_seconds: 2.0
|
||||
# If true, direct-mode runs will try the local node after repeated 429s
|
||||
direct_auto_fallback_to_p2p: false
|
||||
# Download limit: Number of most recent matching downloads per channel (0 = all)
|
||||
download_limit: 10
|
||||
|
||||
channels:
|
||||
# Add your channels here. Examples:
|
||||
# - input: "https://odysee.com/@SomeChannel:1"
|
||||
# enabled: true
|
||||
# content_mode: "non_video_only"
|
||||
# tags_include: []
|
||||
# tags_exclude: []
|
||||
# - input: "lbry://@AnotherChannel#5"
|
||||
# enabled: true
|
||||
# content_mode: "all"
|
||||
"""
|
||||
|
||||
|
||||
|
||||
+179
-177
@@ -1,12 +1,11 @@
|
||||
"""Direct downloader module - downloads from Odysee CDN instead of P2P."""
|
||||
"""Direct downloader module - downloads from Odysee's direct web endpoints."""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import shutil
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Dict, Optional
|
||||
from typing import Dict
|
||||
from urllib.parse import quote
|
||||
|
||||
try:
|
||||
import requests
|
||||
@@ -26,59 +25,127 @@ class DirectDownloadError(Exception):
|
||||
|
||||
|
||||
class DirectDownloader:
|
||||
"""Downloads content directly from Odysee CDN."""
|
||||
"""Downloads content directly from Odysee's web-exposed endpoints."""
|
||||
|
||||
stream_path = "/$/stream"
|
||||
download_path = "/$/download"
|
||||
|
||||
def __init__(self, config: Config, base_dir: Path):
|
||||
self.config = config
|
||||
self.base_dir = base_dir
|
||||
self.direct_base_urls = [
|
||||
base_url.rstrip("/") for base_url in config.general.direct_base_urls
|
||||
]
|
||||
self.session = requests.Session()
|
||||
self.session.headers.update(
|
||||
{"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36"}
|
||||
{
|
||||
"User-Agent": (
|
||||
"Mozilla/5.0 (X11; Linux x86_64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/123.0.0.0 Safari/537.36"
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
def _claim_slug(self, action: DownloadAction) -> str:
|
||||
"""Choose the best available slug for Odysee URLs."""
|
||||
slug = (
|
||||
action.metadata.get("normalized_name")
|
||||
or action.metadata.get("name")
|
||||
or action.claim_name
|
||||
)
|
||||
return slug
|
||||
|
||||
def build_stream_url(self, action: DownloadAction, base_url: str | None = None) -> str:
|
||||
"""Build Odysee's public direct-stream URL for a claim."""
|
||||
slug = self._claim_slug(action)
|
||||
chosen_base = (base_url or self.direct_base_urls[0]).rstrip("/")
|
||||
return f"{chosen_base}{self.stream_path}/{quote(slug, safe='')}/{action.claim_id}"
|
||||
|
||||
def build_download_url(self, action: DownloadAction, base_url: str | None = None) -> str:
|
||||
"""Build Odysee's public download-entry URL for a claim."""
|
||||
slug = self._claim_slug(action)
|
||||
chosen_base = (base_url or self.direct_base_urls[0]).rstrip("/")
|
||||
return (
|
||||
f"{chosen_base}{self.download_path}/{quote(slug, safe='')}/{action.claim_id}"
|
||||
)
|
||||
|
||||
def _is_streaming_media(self, action: DownloadAction) -> bool:
|
||||
"""Return True when the claim looks like audio/video media."""
|
||||
stream_type = (action.metadata.get("stream_type") or "").lower()
|
||||
media_type = (action.metadata.get("source_media_type") or "").lower()
|
||||
|
||||
if stream_type in {"video", "audio"}:
|
||||
return True
|
||||
if media_type.startswith("video/") or media_type.startswith("audio/"):
|
||||
return True
|
||||
return False
|
||||
|
||||
def build_candidate_urls(self, action: DownloadAction) -> list[str]:
|
||||
"""Build Odysee direct-download URL candidates in priority order."""
|
||||
candidate_urls = []
|
||||
for base_url in self.direct_base_urls:
|
||||
stream_url = self.build_stream_url(action, base_url=base_url)
|
||||
download_url = self.build_download_url(action, base_url=base_url)
|
||||
if self._is_streaming_media(action):
|
||||
candidate_urls.extend([stream_url, download_url])
|
||||
else:
|
||||
candidate_urls.extend([download_url, stream_url])
|
||||
return candidate_urls
|
||||
|
||||
def download(self, action: DownloadAction, dry_run: bool = False) -> bool:
|
||||
"""
|
||||
Download a claim directly from Odysee CDN.
|
||||
Download a claim directly from Odysee.
|
||||
|
||||
Args:
|
||||
action: The download action
|
||||
dry_run: If True, don't actually download
|
||||
action: The download action.
|
||||
dry_run: If True, don't actually download.
|
||||
|
||||
Returns:
|
||||
True if download succeeded
|
||||
True if download succeeded.
|
||||
"""
|
||||
if action.action == "skip_existing":
|
||||
logger.debug(f"Skipping already downloaded: {action.claim_name}")
|
||||
existing_path = action.metadata.get("existing_file_relpath")
|
||||
if existing_path:
|
||||
logger.info(
|
||||
"Skipping existing file: %s (%s)",
|
||||
action.claim_name,
|
||||
self.base_dir / existing_path,
|
||||
)
|
||||
else:
|
||||
logger.info(f"Skipping existing file: {action.claim_name}")
|
||||
return True
|
||||
|
||||
if dry_run:
|
||||
logger.info(f"[DRY-RUN] Would download from CDN: {action.claim_name}")
|
||||
logger.info(f"[DRY-RUN] Would download from Odysee: {action.claim_name}")
|
||||
return True
|
||||
|
||||
# Create directories
|
||||
version_dir = Path(action.target_dir)
|
||||
version_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
logger.info(f"Downloading from CDN: {action.claim_name}")
|
||||
file_name = action.metadata.get("source_name") or f"{action.claim_name}.bin"
|
||||
file_path = version_dir / file_name
|
||||
|
||||
logger.info(f"Downloading from Odysee: {action.claim_name}")
|
||||
logger.info(f"Saving to: {file_path}")
|
||||
|
||||
try:
|
||||
# Get download URL from Odysee
|
||||
download_url = self._get_download_url(action.uri)
|
||||
if not download_url:
|
||||
raise DirectDownloadError("Could not get download URL from Odysee")
|
||||
|
||||
logger.info(f"Download URL: {download_url}")
|
||||
|
||||
# Download the file
|
||||
file_path = self._download_file(
|
||||
download_url, version_dir, action.claim_name
|
||||
last_error = None
|
||||
for index, download_url in enumerate(
|
||||
self.build_candidate_urls(action), start=1
|
||||
):
|
||||
logger.info(f"Direct URL candidate {index}: {download_url}")
|
||||
success, failure_reason = self._download_file(
|
||||
download_url, file_path, action.claim_name
|
||||
)
|
||||
if not file_path:
|
||||
raise DirectDownloadError("File download failed")
|
||||
if success:
|
||||
break
|
||||
last_error = failure_reason
|
||||
else:
|
||||
raise DirectDownloadError(last_error or "Odysee direct download failed")
|
||||
|
||||
logger.info(f"Downloaded: {action.claim_name} -> {file_path.name}")
|
||||
|
||||
# Write metadata
|
||||
file_info = {
|
||||
"download_path": str(file_path),
|
||||
"file_name": file_path.name,
|
||||
@@ -86,11 +153,9 @@ class DirectDownloader:
|
||||
}
|
||||
self._write_metadata(version_dir, action, file_info)
|
||||
|
||||
# Write checksum if enabled
|
||||
if self.config.general.write_checksums:
|
||||
self._write_checksum(version_dir, file_path)
|
||||
|
||||
# Update action metadata
|
||||
try:
|
||||
rel_path = file_path.relative_to(self.base_dir)
|
||||
action.metadata["local_file_path"] = str(rel_path)
|
||||
@@ -99,204 +164,149 @@ class DirectDownloader:
|
||||
|
||||
return True
|
||||
|
||||
except DirectDownloadError:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Direct download failed for {action.claim_name}: {e}")
|
||||
raise DirectDownloadError(f"Download failed: {e}")
|
||||
|
||||
def _get_download_url(self, lbry_uri: str) -> Optional[str]:
|
||||
"""
|
||||
Get direct download URL from Odysee API.
|
||||
|
||||
Args:
|
||||
lbry_uri: LBRY URI like lbry://@channel/claim#claim_id
|
||||
|
||||
Returns:
|
||||
Direct download URL or None
|
||||
"""
|
||||
try:
|
||||
# First, resolve the claim to get the permanent URL
|
||||
# Odysee API endpoint for streaming
|
||||
api_url = "https://api.lbry.tv/api/v1/proxy"
|
||||
|
||||
# Get the claim info
|
||||
claim_name = lbry_uri.split("/")[-1].split("#")[0]
|
||||
|
||||
# Try to get streaming URL from Odysee
|
||||
# This uses their API to get CDN links
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
# Method 1: Try to get via Odysee's API
|
||||
payload = {
|
||||
"jsonrpc": "2.0",
|
||||
"method": "get",
|
||||
"params": {
|
||||
"uri": lbry_uri,
|
||||
"save_file": False,
|
||||
},
|
||||
"id": 1,
|
||||
}
|
||||
|
||||
logger.debug(f"Querying Odysee API for: {lbry_uri}")
|
||||
response = self.session.post(
|
||||
api_url, json=payload, headers=headers, timeout=30
|
||||
)
|
||||
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
if "result" in data and data["result"]:
|
||||
result = data["result"]
|
||||
|
||||
# Try to get streaming_url
|
||||
if "streaming_url" in result and result["streaming_url"]:
|
||||
return result["streaming_url"]
|
||||
|
||||
# Try to get download URL from value
|
||||
if "download_path" in result:
|
||||
return result["download_path"]
|
||||
|
||||
# Try to construct from lbry.tv
|
||||
if "permanent_url" in result:
|
||||
claim_id = result.get("claim_id", "")
|
||||
if claim_id:
|
||||
# Construct Odysee direct URL
|
||||
return f"https://player.odycdn.com/v6/streams/{claim_id}/"
|
||||
|
||||
# Method 2: Try using lbry.tv player URL directly
|
||||
# Extract claim info from URI
|
||||
claim_parts = lbry_uri.replace("lbry://", "").split("/")
|
||||
if len(claim_parts) >= 2:
|
||||
claim_name_encoded = claim_parts[-1].split("#")[0]
|
||||
# Try to construct direct download URL
|
||||
# This is an educated guess based on Odysee's URL structure
|
||||
odysee_url = f"https://odysee.com/{lbry_uri.replace('lbry://', '').replace('#', ':')}"
|
||||
logger.debug(f"Trying Odysee URL: {odysee_url}")
|
||||
|
||||
# Get the page to find the download link
|
||||
page_response = self.session.get(odysee_url, timeout=30)
|
||||
if page_response.status_code == 200:
|
||||
# Look for download URL in page
|
||||
# Odysee embeds this in their JavaScript
|
||||
match = re.search(r'"download_url":"([^"]+)"', page_response.text)
|
||||
if match:
|
||||
url = match.group(1).replace("\\u0026", "&")
|
||||
return url
|
||||
|
||||
# Look for streaming URL
|
||||
match = re.search(r'"streaming_url":"([^"]+)"', page_response.text)
|
||||
if match:
|
||||
url = match.group(1).replace("\\u0026", "&")
|
||||
return url
|
||||
|
||||
logger.warning(f"Could not extract download URL for: {lbry_uri}")
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting download URL: {e}")
|
||||
return None
|
||||
|
||||
def _download_file(
|
||||
self, url: str, version_dir: Path, claim_name: str
|
||||
) -> Optional[Path]:
|
||||
self, url: str, file_path: Path, claim_name: str
|
||||
) -> tuple[bool, str | None]:
|
||||
"""
|
||||
Download file from URL to version directory with progress bar.
|
||||
Download file from URL with progress display.
|
||||
|
||||
Args:
|
||||
url: Download URL
|
||||
version_dir: Target directory
|
||||
claim_name: Name of the claim
|
||||
url: Download URL.
|
||||
file_path: Target file path.
|
||||
claim_name: Name of the claim for display.
|
||||
|
||||
Returns:
|
||||
Path to downloaded file or None
|
||||
Tuple of success flag and optional failure reason.
|
||||
"""
|
||||
attempts = self.config.general.direct_max_retries_per_url + 1
|
||||
for attempt in range(1, attempts + 1):
|
||||
try:
|
||||
# Determine filename from URL or claim name
|
||||
filename = self._extract_filename(url, claim_name)
|
||||
file_path = version_dir / filename
|
||||
print(f"\n Downloading: {claim_name}")
|
||||
print(f" Filename: {file_path.name}")
|
||||
print(f" Saving to: {file_path}")
|
||||
print(f" Source: {url}")
|
||||
|
||||
# Print clear message with filename
|
||||
print(f"\n📥 Downloading: {claim_name}")
|
||||
print(f" Filename: {filename}")
|
||||
print(f" Destination: {file_path.parent}")
|
||||
|
||||
# Stream download with progress
|
||||
response = self.session.get(url, stream=True, timeout=300)
|
||||
response = self.session.get(
|
||||
url,
|
||||
stream=True,
|
||||
timeout=300,
|
||||
headers={"Referer": "https://odysee.com/"},
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
total_size = int(response.headers.get("content-length", 0))
|
||||
content_type = response.headers.get("content-type", "")
|
||||
if "text/html" in content_type:
|
||||
logger.warning("Got HTML response instead of file content")
|
||||
return False, "Got HTML response instead of file content"
|
||||
|
||||
total_size = int(response.headers.get("content-length", 0))
|
||||
if total_size > 0:
|
||||
size_mb = total_size / (1024 * 1024)
|
||||
print(f" Size: {size_mb:.2f} MB")
|
||||
else:
|
||||
print(f" Size: Unknown")
|
||||
print(" Size: Unknown")
|
||||
print()
|
||||
|
||||
start_time = time.time()
|
||||
last_update = start_time
|
||||
downloaded = 0
|
||||
chunk_size = 8192
|
||||
|
||||
with open(file_path, "wb") as f:
|
||||
for chunk in response.iter_content(chunk_size=chunk_size):
|
||||
for chunk in response.iter_content(chunk_size=8192):
|
||||
if chunk:
|
||||
f.write(chunk)
|
||||
downloaded += len(chunk)
|
||||
|
||||
# Update progress bar every 0.5 seconds
|
||||
current_time = time.time()
|
||||
if current_time - last_update >= 0.5:
|
||||
self._print_progress_bar(
|
||||
downloaded, total_size, start_time, claim_name
|
||||
downloaded, total_size, start_time
|
||||
)
|
||||
last_update = current_time
|
||||
|
||||
# Final progress update
|
||||
self._print_progress_bar(
|
||||
downloaded, total_size, start_time, claim_name, final=True
|
||||
downloaded, total_size, start_time, final=True
|
||||
)
|
||||
print() # New line after progress bar
|
||||
print()
|
||||
|
||||
# Verify file was downloaded
|
||||
if file_path.exists() and file_path.stat().st_size > 0:
|
||||
size_mb = file_path.stat().st_size / (1024 * 1024)
|
||||
elapsed = time.time() - start_time
|
||||
speed_mbps = (size_mb / elapsed) if elapsed > 0 else 0
|
||||
print(f"✅ Complete: {size_mb:.2f} MB @ {speed_mbps:.2f} MB/s\n")
|
||||
return file_path
|
||||
else:
|
||||
print(f"❌ Error: Downloaded file is empty\n")
|
||||
return None
|
||||
print(f" Complete: {size_mb:.2f} MB @ {speed_mbps:.2f} MB/s\n")
|
||||
return True, None
|
||||
|
||||
if file_path.exists():
|
||||
file_path.unlink()
|
||||
print(" Error: Downloaded file is empty\n")
|
||||
return False, "Downloaded file is empty"
|
||||
|
||||
except requests.exceptions.HTTPError as e:
|
||||
status_code = getattr(e.response, "status_code", None)
|
||||
if status_code == 401:
|
||||
logger.error(f"HTTP 401 downloading file: {e}")
|
||||
reason = "Odysee CDN rejected this direct URL as unauthorized"
|
||||
elif status_code == 429:
|
||||
logger.error(f"HTTP 429 downloading file: {e}")
|
||||
if file_path.exists():
|
||||
file_path.unlink()
|
||||
if attempt < attempts:
|
||||
backoff = (
|
||||
self.config.general.direct_retry_backoff_seconds
|
||||
* (2 ** (attempt - 1))
|
||||
)
|
||||
logger.warning(
|
||||
"HTTP 429 downloading %s. Retrying in %.1fs (%s/%s)",
|
||||
claim_name,
|
||||
backoff,
|
||||
attempt,
|
||||
attempts - 1,
|
||||
)
|
||||
time.sleep(backoff)
|
||||
continue
|
||||
reason = (
|
||||
"Odysee CDN rate-limited the direct download request after retries"
|
||||
)
|
||||
else:
|
||||
logger.error(f"HTTP error downloading file: {e}")
|
||||
reason = f"HTTP error: {e}"
|
||||
if file_path.exists():
|
||||
file_path.unlink()
|
||||
return False, reason
|
||||
except Exception as e:
|
||||
print(f"❌ Error downloading {claim_name}: {e}\n")
|
||||
logger.error(f"Error downloading file: {e}")
|
||||
return None
|
||||
if file_path.exists():
|
||||
file_path.unlink()
|
||||
return False, str(e)
|
||||
return False, "Direct download failed"
|
||||
|
||||
def _print_progress_bar(
|
||||
self,
|
||||
downloaded: int,
|
||||
total_size: int,
|
||||
start_time: float,
|
||||
claim_name: str,
|
||||
final: bool = False,
|
||||
):
|
||||
) -> None:
|
||||
"""Print a progress bar to the console."""
|
||||
if total_size > 0:
|
||||
percent = min(100, int((downloaded / total_size) * 100))
|
||||
filled = int(50 * percent / 100)
|
||||
bar = "█" * filled + "░" * (50 - filled)
|
||||
bar = "\u2588" * filled + "\u2591" * (50 - filled)
|
||||
|
||||
# Calculate speed
|
||||
elapsed = time.time() - start_time
|
||||
speed_mbps = 0
|
||||
if elapsed > 0:
|
||||
speed_mbps = (downloaded / (1024 * 1024)) / elapsed
|
||||
speed_str = f"{speed_mbps:.2f} MB/s"
|
||||
else:
|
||||
speed_str = "calculating..."
|
||||
|
||||
# Calculate ETA
|
||||
if not final and speed_mbps > 0:
|
||||
remaining = (total_size - downloaded) / (1024 * 1024)
|
||||
eta = remaining / speed_mbps
|
||||
@@ -304,27 +314,15 @@ class DirectDownloader:
|
||||
else:
|
||||
eta_str = ""
|
||||
|
||||
# Clear line and print progress
|
||||
print(
|
||||
f"\r [{bar}] {percent}% | {speed_str} | {eta_str}", end="", flush=True
|
||||
f"\r [{bar}] {percent}% | {speed_str} | {eta_str}",
|
||||
end="",
|
||||
flush=True,
|
||||
)
|
||||
else:
|
||||
# Unknown size, just show downloaded amount
|
||||
downloaded_mb = downloaded / (1024 * 1024)
|
||||
print(f"\r Downloaded: {downloaded_mb:.2f} MB", end="", flush=True)
|
||||
|
||||
def _extract_filename(self, url: str, claim_name: str) -> str:
|
||||
"""Extract filename from URL or use claim name."""
|
||||
# Try to get from URL
|
||||
if "." in url.split("/")[-1].split("?")[0]:
|
||||
filename = url.split("/")[-1].split("?")[0]
|
||||
# Sanitize
|
||||
filename = re.sub(r'[<>"|?*]', "_", filename)
|
||||
return filename[:255] # Max filename length
|
||||
|
||||
# Use claim name with extension
|
||||
return f"{claim_name}.mp4" # Default to mp4
|
||||
|
||||
def _write_metadata(
|
||||
self, version_dir: Path, action: DownloadAction, file_info: Dict
|
||||
) -> None:
|
||||
@@ -336,10 +334,14 @@ class DirectDownloader:
|
||||
"title": action.metadata.get("title"),
|
||||
"version_token": action.version_token,
|
||||
"permanent_url": action.uri,
|
||||
"download_source": "odysee_cdn",
|
||||
"download_source": (
|
||||
"odysee_stream" if self._is_streaming_media(action) else "odysee_download"
|
||||
),
|
||||
"downloaded_at": time.strftime("%Y-%m-%dT%H:%M:%S"),
|
||||
"file_name": file_info.get("file_name"),
|
||||
"file_size": file_info.get("size"),
|
||||
"stream_type": action.metadata.get("stream_type"),
|
||||
"source_media_type": action.metadata.get("source_media_type"),
|
||||
}
|
||||
|
||||
metadata_path = version_dir / "metadata.json"
|
||||
|
||||
+12
-1
@@ -49,7 +49,15 @@ class Downloader:
|
||||
DownloadError: If the download fails.
|
||||
"""
|
||||
if action.action == "skip_existing":
|
||||
logger.debug(f"Skipping already downloaded: {action.claim_name}")
|
||||
existing_path = action.metadata.get("existing_file_relpath")
|
||||
if existing_path:
|
||||
logger.info(
|
||||
"Skipping existing file: %s (%s)",
|
||||
action.claim_name,
|
||||
self.base_dir / existing_path,
|
||||
)
|
||||
else:
|
||||
logger.info(f"Skipping existing file: {action.claim_name}")
|
||||
return True
|
||||
|
||||
if dry_run:
|
||||
@@ -61,6 +69,7 @@ class Downloader:
|
||||
version_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
logger.info(f"Downloading: {action.claim_name}")
|
||||
logger.info(f"Saving to directory: {version_dir}")
|
||||
|
||||
try:
|
||||
# Call daemon to start P2P download
|
||||
@@ -281,6 +290,7 @@ class Downloader:
|
||||
streaming_url = f"http://localhost:5280/stream/{sd_hash}"
|
||||
|
||||
logger.info(f"Downloading via daemon streaming endpoint...")
|
||||
logger.info(f"Streaming destination: {target_path}")
|
||||
|
||||
try:
|
||||
response = requests.get(streaming_url, stream=True, timeout=300)
|
||||
@@ -291,6 +301,7 @@ class Downloader:
|
||||
chunk_size = 8192
|
||||
|
||||
print(f"\n📥 Downloading: {action.claim_name}")
|
||||
print(f" Saving to: {target_path}")
|
||||
print(f" Size: {total_size / (1024 * 1024):.2f} MB")
|
||||
print()
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Initialize LBRY Downloader configuration.
|
||||
Initialize Odysee/LBRY Downloader configuration.
|
||||
|
||||
This script creates the initial configuration file and directory structure.
|
||||
"""
|
||||
@@ -14,7 +14,7 @@ from utils import expand_path
|
||||
|
||||
|
||||
def main():
|
||||
print("LBRY Downloader - Initialization")
|
||||
print("Odysee/LBRY Downloader - Initialization")
|
||||
print("=" * 40)
|
||||
|
||||
# Default paths
|
||||
@@ -51,12 +51,13 @@ def main():
|
||||
print("Next steps:")
|
||||
print("1. Edit the configuration file to add your channels")
|
||||
print(f" nano {config_file}")
|
||||
print("\n2. Ensure lbrynet daemon is running:")
|
||||
print(" lbrynet start")
|
||||
print("\n3. Run the downloader:")
|
||||
print(" python main.py")
|
||||
print("\n4. Or test first with dry-run:")
|
||||
print("\n2. Test first with dry-run:")
|
||||
print(" python main.py --dry-run")
|
||||
print("\n3. Recommended first real run:")
|
||||
print(" python main.py --non-video-only")
|
||||
print("\n4. Optional P2P mode if you want the local node path:")
|
||||
print(" lbrynet start")
|
||||
print(" python main.py --p2p")
|
||||
print("=" * 40)
|
||||
|
||||
|
||||
|
||||
+11
-52
@@ -12,7 +12,7 @@
|
||||
set -e
|
||||
|
||||
echo "================================================"
|
||||
echo " LBRY Downloader - Installation"
|
||||
echo " Odysee/LBRY Downloader - Installation"
|
||||
echo "================================================"
|
||||
echo ""
|
||||
|
||||
@@ -117,51 +117,14 @@ pip install --upgrade pip
|
||||
pip install -r requirements.txt
|
||||
print_status "Python dependencies installed"
|
||||
|
||||
# Check for lbrynet
|
||||
echo ""
|
||||
echo "Checking for LBRY SDK (lbrynet)..."
|
||||
echo "Checking for optional LBRY SDK (lbrynet)..."
|
||||
if command -v lbrynet &> /dev/null; then
|
||||
print_status "lbrynet found at: $(which lbrynet)"
|
||||
print_status "Optional P2P dependency found at: $(which lbrynet)"
|
||||
lbrynet --version
|
||||
else
|
||||
print_warning "lbrynet not found in PATH"
|
||||
echo ""
|
||||
echo "The LBRY SDK (lbrynet) is required to download content."
|
||||
echo ""
|
||||
read -p "Would you like to download and install lbrynet now? (y/N): " -n 1 -r
|
||||
echo
|
||||
|
||||
if [[ $REPLY =~ ^[Yy]$ ]]; then
|
||||
echo "Downloading lbrynet..."
|
||||
mkdir -p ~/.local/bin
|
||||
|
||||
# Get latest release
|
||||
LBRY_VERSION="0.113.0"
|
||||
DOWNLOAD_URL="https://github.com/lbryio/lbry-sdk/releases/download/v${LBRY_VERSION}/lbry-sdk-linux.zip"
|
||||
|
||||
cd ~/.local/bin
|
||||
if wget -q "$DOWNLOAD_URL"; then
|
||||
unzip -q lbry-sdk-linux.zip
|
||||
rm lbry-sdk-linux.zip
|
||||
chmod +x lbrynet
|
||||
print_status "lbrynet installed to ~/.local/bin"
|
||||
|
||||
# Check if ~/.local/bin is in PATH
|
||||
if [[ ":$PATH:" != *":$HOME/.local/bin:"* ]]; then
|
||||
echo 'export PATH="$HOME/.local/bin:$PATH"' >> ~/.bashrc
|
||||
print_status "Added ~/.local/bin to PATH in ~/.bashrc"
|
||||
print_warning "Please run: source ~/.bashrc"
|
||||
fi
|
||||
|
||||
cd "$SCRIPT_DIR"
|
||||
else
|
||||
print_error "Failed to download lbrynet"
|
||||
echo "Please install manually from: https://github.com/lbryio/lbry-sdk/releases"
|
||||
fi
|
||||
else
|
||||
print_warning "Skipping lbrynet installation"
|
||||
echo "You'll need to install it manually before using the downloader"
|
||||
fi
|
||||
print_info "lbrynet not found. Default direct mode does not require it."
|
||||
print_info "Install it later only if you want to run with --p2p."
|
||||
fi
|
||||
|
||||
# Initialize configuration
|
||||
@@ -201,13 +164,6 @@ fi
|
||||
cd "$DOWNLOADER_DIR"
|
||||
source venv/bin/activate 2>/dev/null || true
|
||||
|
||||
# Check if lbrynet is running
|
||||
if ! curl -s http://127.0.0.1:5279 > /dev/null 2>&1; then
|
||||
echo "Starting lbrynet daemon..."
|
||||
lbrynet start &
|
||||
sleep 5
|
||||
fi
|
||||
|
||||
python main.py "$@"
|
||||
EOF
|
||||
|
||||
@@ -233,7 +189,7 @@ echo "1. Edit configuration to add your channels:"
|
||||
echo " nano ~/Documents/lbry-downloads/config.yaml"
|
||||
echo ""
|
||||
echo "2. Start the LBRY daemon:"
|
||||
echo " lbrynet start"
|
||||
echo " Optional only for P2P mode: lbrynet start"
|
||||
echo ""
|
||||
echo "3. Test with dry-run:"
|
||||
echo " python main.py --dry-run"
|
||||
@@ -241,9 +197,12 @@ echo " # OR:"
|
||||
echo " ~/bin/lbry-sync --dry-run"
|
||||
echo ""
|
||||
echo "4. Run the downloader:"
|
||||
echo " python main.py"
|
||||
echo " python main.py --non-video-only"
|
||||
echo " # OR:"
|
||||
echo " ~/bin/lbry-sync"
|
||||
echo " ~/bin/lbry-sync --non-video-only"
|
||||
echo ""
|
||||
echo "5. Optional local-node mode:"
|
||||
echo " python main.py --p2p"
|
||||
echo ""
|
||||
echo "For detailed instructions, see:"
|
||||
echo " - LINUX_SETUP.md"
|
||||
|
||||
@@ -14,6 +14,7 @@ from typing import List, Optional
|
||||
|
||||
from config_loader import ConfigError, ensure_directories, load_config
|
||||
from downloader import DownloadError, Downloader
|
||||
from direct_downloader import DirectDownloadError
|
||||
from lbry_client import LbryClient, LbryClientError, check_daemon_health
|
||||
from models import Channel, Config, DownloadAction
|
||||
from planner import Planner
|
||||
@@ -26,6 +27,20 @@ from utils import (
|
||||
)
|
||||
|
||||
|
||||
VIDEO_EXTENSIONS = {
|
||||
".mp4",
|
||||
".webm",
|
||||
".mkv",
|
||||
".mov",
|
||||
".avi",
|
||||
".m4v",
|
||||
".mpeg",
|
||||
".mpg",
|
||||
".wmv",
|
||||
".flv",
|
||||
}
|
||||
|
||||
|
||||
def setup_logging(log_level: str) -> None:
|
||||
"""Configure logging."""
|
||||
logging.basicConfig(
|
||||
@@ -83,6 +98,24 @@ def resolve_channel(
|
||||
channel_claim_id = claim.get("claim_id")
|
||||
channel_name = claim.get("name")
|
||||
|
||||
# Odysee's public proxy can resolve vanity channel URLs to older
|
||||
# non-controlling channel claims. Enumerating streams from those claim IDs
|
||||
# returns zero items, so normalize to the controlling claim before scanning.
|
||||
if not client.supports_file_ops and channel_name:
|
||||
controlling_claim = client.find_controlling_channel(channel_name)
|
||||
if controlling_claim and controlling_claim.get("claim_id"):
|
||||
controlling_claim_id = controlling_claim.get("claim_id")
|
||||
if controlling_claim_id != channel_claim_id:
|
||||
logger.debug(
|
||||
"Normalized proxy-resolved channel %s from %s to controlling claim %s",
|
||||
channel_name,
|
||||
channel_claim_id,
|
||||
controlling_claim_id,
|
||||
)
|
||||
claim = controlling_claim
|
||||
channel_claim_id = controlling_claim_id
|
||||
channel_name = claim.get("name")
|
||||
|
||||
if not channel_claim_id:
|
||||
logger.error(f"No claim_id in resolved channel: {channel_input}")
|
||||
return None
|
||||
@@ -164,14 +197,96 @@ def enumerate_channel_claims(
|
||||
return all_claims
|
||||
|
||||
|
||||
def run_sync(config: Config, dry_run: bool = False, direct: bool = False) -> dict:
|
||||
def _action_is_video(action: DownloadAction) -> bool:
|
||||
"""Return True when a planned action looks like a video download."""
|
||||
media_type = (action.metadata.get("source_media_type") or "").lower()
|
||||
if media_type.startswith("video/"):
|
||||
return True
|
||||
|
||||
stream_type = (action.metadata.get("stream_type") or "").lower()
|
||||
if stream_type == "video":
|
||||
return True
|
||||
|
||||
source_name = action.metadata.get("source_name") or action.metadata.get("file_name")
|
||||
if source_name and Path(source_name).suffix.lower() in VIDEO_EXTENSIONS:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def filter_actions_by_content_type(
|
||||
actions: List[DownloadAction],
|
||||
*,
|
||||
video_only: bool = False,
|
||||
non_video_only: bool = False,
|
||||
) -> List[DownloadAction]:
|
||||
"""Filter planned actions by content type before limits and execution."""
|
||||
if not video_only and not non_video_only:
|
||||
return actions
|
||||
|
||||
filtered_actions = []
|
||||
for action in actions:
|
||||
is_video = _action_is_video(action)
|
||||
if video_only and is_video:
|
||||
filtered_actions.append(action)
|
||||
elif non_video_only and not is_video:
|
||||
filtered_actions.append(action)
|
||||
|
||||
return filtered_actions
|
||||
|
||||
|
||||
def resolve_content_filter_mode(
|
||||
channel_config,
|
||||
*,
|
||||
video_only: bool = False,
|
||||
non_video_only: bool = False,
|
||||
) -> tuple[bool, bool, str]:
|
||||
"""Resolve the effective content filter for a channel."""
|
||||
if video_only:
|
||||
return True, False, "video-only (CLI override)"
|
||||
if non_video_only:
|
||||
return False, True, "non-video-only (CLI override)"
|
||||
|
||||
content_mode = getattr(channel_config, "content_mode", "all")
|
||||
if content_mode == "video_only":
|
||||
return True, False, "video-only (channel config)"
|
||||
if content_mode == "non_video_only":
|
||||
return False, True, "non-video-only (channel config)"
|
||||
|
||||
return False, False, "all content"
|
||||
|
||||
|
||||
def resolve_transport_mode(
|
||||
*,
|
||||
direct: bool = False,
|
||||
p2p: bool = False,
|
||||
) -> tuple[bool, str]:
|
||||
"""Resolve the effective transport mode."""
|
||||
if p2p:
|
||||
return False, "p2p"
|
||||
return True, "direct"
|
||||
|
||||
|
||||
def _is_rate_limit_error(exc: Exception) -> bool:
|
||||
"""Return True when the error indicates direct-mode rate limiting."""
|
||||
return "rate-limited" in str(exc).lower()
|
||||
|
||||
|
||||
def run_sync(
|
||||
config: Config,
|
||||
dry_run: bool = False,
|
||||
direct: bool = True,
|
||||
*,
|
||||
video_only: bool = False,
|
||||
non_video_only: bool = False,
|
||||
) -> dict:
|
||||
"""
|
||||
Run the main synchronization process.
|
||||
|
||||
Args:
|
||||
config: Application configuration.
|
||||
dry_run: If True, don't actually download.
|
||||
direct: If True, use Odysee CDN instead of P2P.
|
||||
direct: If True, use Odysee proxy/CDN instead of P2P.
|
||||
|
||||
Returns:
|
||||
Statistics dictionary.
|
||||
@@ -195,30 +310,28 @@ def run_sync(config: Config, dry_run: bool = False, direct: bool = False) -> dic
|
||||
base_path = Path(config.general.base_dir)
|
||||
planner = Planner(config, state_db.data, str(base_path))
|
||||
|
||||
# Initialize LBRY client (needed for channel/claim resolution in both modes)
|
||||
if direct:
|
||||
logger.info("Using DIRECT mode (Odysee proxy for metadata, Odysee CDN for downloads)")
|
||||
client = LbryClient.create_odysee_proxy(timeout=config.lbrynet.timeout_seconds)
|
||||
from direct_downloader import DirectDownloader
|
||||
|
||||
downloader = DirectDownloader(config, base_path)
|
||||
p2p_fallback_client = None
|
||||
p2p_fallback_downloader = None
|
||||
p2p_fallback_available = None
|
||||
else:
|
||||
client = LbryClient(
|
||||
api_url=config.lbrynet.api_url,
|
||||
timeout=config.lbrynet.timeout_seconds,
|
||||
)
|
||||
|
||||
# Check daemon health (needed for metadata even in direct mode)
|
||||
logger.info(f"Checking daemon at {config.lbrynet.api_url}")
|
||||
try:
|
||||
check_daemon_health(client)
|
||||
logger.info("Daemon is healthy")
|
||||
except LbryClientError as e:
|
||||
logger.error(str(e))
|
||||
logger.error(
|
||||
"Daemon is required for channel/claim resolution even in --direct mode"
|
||||
)
|
||||
return stats
|
||||
|
||||
if direct:
|
||||
logger.info("Using DIRECT mode (Odysee CDN for downloads)")
|
||||
from direct_downloader import DirectDownloader
|
||||
|
||||
downloader = DirectDownloader(config, base_path)
|
||||
else:
|
||||
logger.info("Using P2P mode (LBRY network)")
|
||||
downloader = Downloader(client, config, base_path)
|
||||
|
||||
@@ -251,6 +364,26 @@ def run_sync(config: Config, dry_run: bool = False, direct: bool = False) -> dic
|
||||
|
||||
# Plan downloads
|
||||
actions = planner.process_channel_claims(channel, claims)
|
||||
actions_before_filter = len(actions)
|
||||
channel_video_only, channel_non_video_only, mode_label = (
|
||||
resolve_content_filter_mode(
|
||||
channel_config,
|
||||
video_only=video_only,
|
||||
non_video_only=non_video_only,
|
||||
)
|
||||
)
|
||||
actions = filter_actions_by_content_type(
|
||||
actions,
|
||||
video_only=channel_video_only,
|
||||
non_video_only=channel_non_video_only,
|
||||
)
|
||||
|
||||
if len(actions) != actions_before_filter:
|
||||
logger.info(
|
||||
"Content filter (%s) skipped %s planned claim(s).",
|
||||
mode_label,
|
||||
actions_before_filter - len(actions),
|
||||
)
|
||||
|
||||
# Apply download limit if configured
|
||||
# Separate skip actions from download actions
|
||||
@@ -314,7 +447,61 @@ def run_sync(config: Config, dry_run: bool = False, direct: bool = False) -> dic
|
||||
"success" if success else "failure",
|
||||
)
|
||||
|
||||
except DownloadError as e:
|
||||
except (DownloadError, DirectDownloadError) as e:
|
||||
if (
|
||||
direct
|
||||
and config.general.direct_auto_fallback_to_p2p
|
||||
and isinstance(e, DirectDownloadError)
|
||||
and _is_rate_limit_error(e)
|
||||
and not dry_run
|
||||
):
|
||||
if p2p_fallback_available is None:
|
||||
try:
|
||||
p2p_fallback_client = LbryClient(
|
||||
api_url=config.lbrynet.api_url,
|
||||
timeout=config.lbrynet.timeout_seconds,
|
||||
)
|
||||
check_daemon_health(p2p_fallback_client)
|
||||
p2p_fallback_downloader = Downloader(
|
||||
p2p_fallback_client, config, base_path
|
||||
)
|
||||
p2p_fallback_available = True
|
||||
logger.warning(
|
||||
"Direct mode hit rate limiting. Local P2P fallback is available."
|
||||
)
|
||||
except LbryClientError as fallback_init_error:
|
||||
p2p_fallback_available = False
|
||||
logger.warning(
|
||||
"Direct mode hit rate limiting, but P2P fallback is unavailable: %s",
|
||||
fallback_init_error,
|
||||
)
|
||||
|
||||
if p2p_fallback_available and p2p_fallback_downloader is not None:
|
||||
try:
|
||||
logger.warning(
|
||||
"Retrying %s via P2P fallback after direct-mode rate limiting.",
|
||||
action.claim_name,
|
||||
)
|
||||
success = p2p_fallback_downloader.download(
|
||||
action, dry_run=False
|
||||
)
|
||||
if success:
|
||||
planner.update_state_from_action(action, success=True)
|
||||
history_file = base_path / "state" / "run-history.jsonl"
|
||||
state_db.log_run_history(
|
||||
str(history_file),
|
||||
channel.channel_claim_id,
|
||||
action.claim_id,
|
||||
action.action,
|
||||
action.version_token,
|
||||
"success",
|
||||
)
|
||||
continue
|
||||
except DownloadError as fallback_error:
|
||||
e = DirectDownloadError(
|
||||
f"{e}; P2P fallback also failed: {fallback_error}"
|
||||
)
|
||||
|
||||
logger.error(f"Download failed: {e}")
|
||||
stats["failures"] += 1
|
||||
|
||||
@@ -352,10 +539,10 @@ def main():
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
Examples:
|
||||
python main.py # Run with default config (P2P mode)
|
||||
python main.py # Run with default config (DIRECT mode)
|
||||
python main.py --config ./config.yaml # Use custom config
|
||||
python main.py --dry-run # Plan actions but don't download
|
||||
python main.py --direct # Download from Odysee CDN (more reliable)
|
||||
python main.py --p2p # Use the local LBRY node instead
|
||||
""",
|
||||
)
|
||||
|
||||
@@ -372,16 +559,39 @@ Examples:
|
||||
help="Show what would be downloaded without actually downloading",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
transport_group = parser.add_mutually_exclusive_group()
|
||||
transport_group.add_argument(
|
||||
"--direct",
|
||||
"-d",
|
||||
action="store_true",
|
||||
help="Download directly from Odysee CDN instead of using P2P network",
|
||||
help="Use direct mode explicitly (default: Odysee public proxy + CDN)",
|
||||
)
|
||||
transport_group.add_argument(
|
||||
"--p2p",
|
||||
action="store_true",
|
||||
help="Use the local LBRY daemon and peer-to-peer network instead of direct mode",
|
||||
)
|
||||
|
||||
content_filter_group = parser.add_mutually_exclusive_group()
|
||||
content_filter_group.add_argument(
|
||||
"--video-only",
|
||||
action="store_true",
|
||||
help="Only include video downloads such as mp4/webm/mkv",
|
||||
)
|
||||
content_filter_group.add_argument(
|
||||
"--non-video-only",
|
||||
action="store_true",
|
||||
help="Only include non-video downloads such as zip files",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
direct_mode, transport_label = resolve_transport_mode(
|
||||
direct=args.direct,
|
||||
p2p=args.p2p,
|
||||
)
|
||||
|
||||
# Load config
|
||||
config = load_config(args.config)
|
||||
|
||||
@@ -390,12 +600,19 @@ Examples:
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
logger.info("LBRY Downloader starting")
|
||||
logger.info("Effective transport mode: %s", transport_label)
|
||||
|
||||
# Ensure directories exist
|
||||
ensure_directories(config)
|
||||
|
||||
# Run sync
|
||||
stats = run_sync(config, dry_run=args.dry_run, direct=args.direct)
|
||||
stats = run_sync(
|
||||
config,
|
||||
dry_run=args.dry_run,
|
||||
direct=direct_mode,
|
||||
video_only=args.video_only,
|
||||
non_video_only=args.non_video_only,
|
||||
)
|
||||
|
||||
# Print summary
|
||||
summary = format_summary(
|
||||
|
||||
@@ -12,6 +12,7 @@ class ChannelConfig:
|
||||
input: str
|
||||
enabled: bool = True
|
||||
download_path: Optional[str] = None
|
||||
content_mode: str = "all"
|
||||
tags_include: List[str] = field(default_factory=list)
|
||||
tags_exclude: List[str] = field(default_factory=list)
|
||||
|
||||
@@ -34,6 +35,12 @@ class GeneralConfig:
|
||||
download_limit: int = (
|
||||
10 # Number of most recent downloads per channel, or "all" (0)
|
||||
)
|
||||
direct_base_urls: List[str] = field(
|
||||
default_factory=lambda: ["https://odysee.com"]
|
||||
)
|
||||
direct_max_retries_per_url: int = 2
|
||||
direct_retry_backoff_seconds: float = 2.0
|
||||
direct_auto_fallback_to_p2p: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
@@ -183,6 +183,8 @@ class Planner:
|
||||
target_dir=str(version_dir),
|
||||
metadata=metadata,
|
||||
)
|
||||
if version.file_relpath:
|
||||
action.metadata["existing_file_relpath"] = version.file_relpath
|
||||
logger.debug(f"Skipping existing: {claim_name}")
|
||||
return action
|
||||
else:
|
||||
|
||||
+9
-70
@@ -20,7 +20,7 @@ REPO_URL="https://github.com/PR0M3TH3AN/LBRY-Downloader.git"
|
||||
INSTALL_DIR="${HOME}/Documents/LBRY-Downloader"
|
||||
|
||||
echo "=========================================="
|
||||
echo " LBRY Downloader - Remote Installer"
|
||||
echo " Odysee/LBRY Downloader - Remote Installer"
|
||||
echo "=========================================="
|
||||
echo ""
|
||||
|
||||
@@ -74,70 +74,11 @@ fi
|
||||
|
||||
print_status "Python $PYTHON_VERSION found"
|
||||
|
||||
# Check if lbrynet is installed
|
||||
if ! command -v lbrynet &> /dev/null; then
|
||||
print_warning "LBRY SDK (lbrynet) not found"
|
||||
echo ""
|
||||
echo "The LBRY SDK is required to download content from the LBRY network."
|
||||
echo "It can be downloaded from: https://github.com/lbryio/lbry-sdk/releases"
|
||||
echo ""
|
||||
read -p "Download and install lbrynet automatically? (Y/n): " -n 1 -r
|
||||
echo
|
||||
if [[ ! $REPLY =~ ^[Nn]$ ]]; then
|
||||
print_status "Installing LBRY SDK..."
|
||||
|
||||
# Create bin directory
|
||||
mkdir -p ~/.local/bin
|
||||
|
||||
# Download latest lbrynet for Linux
|
||||
LBRY_VERSION="0.113.0"
|
||||
DOWNLOAD_URL="https://github.com/lbryio/lbry-sdk/releases/download/v${LBRY_VERSION}/lbrynet-linux.zip"
|
||||
|
||||
cd ~/.local/bin
|
||||
|
||||
# Download with progress
|
||||
echo "Downloading lbrynet v${LBRY_VERSION}..."
|
||||
if wget --progress=bar:force "$DOWNLOAD_URL" -O lbrynet-linux.zip 2>&1; then
|
||||
print_status "Downloaded lbrynet"
|
||||
|
||||
# Check if file exists and has content
|
||||
if [ -f "lbrynet-linux.zip" ] && [ -s "lbrynet-linux.zip" ]; then
|
||||
# Extract
|
||||
if unzip -q lbrynet-linux.zip; then
|
||||
rm lbrynet-linux.zip
|
||||
chmod +x lbrynet
|
||||
print_status "Extracted lbrynet"
|
||||
|
||||
# Add to PATH if not already there
|
||||
if ! echo "$PATH" | grep -q "$HOME/.local/bin"; then
|
||||
echo 'export PATH="$HOME/.local/bin:$PATH"' >> ~/.bashrc
|
||||
print_status "Added ~/.local/bin to PATH"
|
||||
export PATH="$HOME/.local/bin:$PATH"
|
||||
fi
|
||||
|
||||
print_status "LBRY SDK installed successfully!"
|
||||
print_info "lbrynet installed to: ~/.local/bin/lbrynet"
|
||||
else
|
||||
print_error "Failed to extract lbrynet"
|
||||
rm -f lbrynet-linux.zip
|
||||
fi
|
||||
else
|
||||
print_error "Download failed - file not found or empty"
|
||||
fi
|
||||
else
|
||||
print_error "Failed to download lbrynet"
|
||||
echo "Please install manually from:"
|
||||
echo " https://github.com/lbryio/lbry-sdk/releases"
|
||||
fi
|
||||
|
||||
# Return to original directory
|
||||
cd - > /dev/null
|
||||
else
|
||||
print_warning "Skipping lbrynet installation"
|
||||
echo "You'll need to install it manually before using the downloader"
|
||||
fi
|
||||
if command -v lbrynet &> /dev/null; then
|
||||
print_status "Optional P2P dependency found: $(which lbrynet)"
|
||||
else
|
||||
print_status "lbrynet found at: $(which lbrynet)"
|
||||
print_info "lbrynet not found. That's fine for default direct mode."
|
||||
print_info "Install it later only if you want to run with --p2p."
|
||||
fi
|
||||
|
||||
# Check if running in interactive mode
|
||||
@@ -187,19 +128,17 @@ echo ""
|
||||
echo "1️⃣ Reload your shell (IMPORTANT):"
|
||||
echo " source ~/.bashrc"
|
||||
echo ""
|
||||
echo "2️⃣ Test the downloader (daemon starts automatically):"
|
||||
echo "2️⃣ Test the downloader:"
|
||||
echo " lbry-downloader --dry-run"
|
||||
echo ""
|
||||
echo "3️⃣ Download content:"
|
||||
echo " lbry-downloader"
|
||||
echo " lbry-downloader --non-video-only"
|
||||
echo ""
|
||||
echo "💡 If 'lbry-downloader' command not found after reload, use:"
|
||||
echo " ~/Documents/LBRY-Downloader/bin/lbry-downloader"
|
||||
echo ""
|
||||
echo "⚠️ If lbrynet was not installed automatically, install it from:"
|
||||
echo " https://github.com/lbryio/lbry-sdk/releases"
|
||||
echo " Then run: lbrynet start"
|
||||
echo " ~/Documents/LBRY-Downloader/bin/lbry-downloader"
|
||||
echo "⚙️ Optional P2P mode:"
|
||||
echo " Install lbrynet only if you want: ~/Documents/LBRY-Downloader/bin/lbry-downloader --p2p"
|
||||
echo ""
|
||||
echo "📁 Config file: ~/Documents/lbry-downloads/config.yaml"
|
||||
echo ""
|
||||
@@ -76,7 +76,9 @@ if current_dir != Path(INSTALL_DIR):
|
||||
config_content = f"""# LBRY Downloader Configuration
|
||||
|
||||
lbrynet:
|
||||
# Only used when running with --p2p
|
||||
api_url: "http://127.0.0.1:5279"
|
||||
# Only used when running with --p2p
|
||||
timeout_seconds: 60
|
||||
|
||||
general:
|
||||
@@ -97,6 +99,7 @@ channels:
|
||||
# Add your channels here:
|
||||
# - input: "https://odysee.com/@ChannelName:1"
|
||||
# enabled: true
|
||||
# content_mode: "non_video_only"
|
||||
"""
|
||||
|
||||
config_path = Path(DOWNLOAD_DIR) / "config.yaml"
|
||||
@@ -126,3 +129,7 @@ print(f"To run: {launcher}")
|
||||
print()
|
||||
print("Edit the config to add channels, then run:")
|
||||
print(f" {launcher} --dry-run")
|
||||
print("Recommended first real run:")
|
||||
print(f" {launcher} --non-video-only")
|
||||
print("Optional local-node mode:")
|
||||
print(f" {launcher} --p2p")
|
||||
@@ -350,8 +350,10 @@ def create_config(
|
||||
|
||||
lbrynet:
|
||||
# URL of the local LBRY daemon (lbrynet)
|
||||
# Only used when running with --p2p
|
||||
api_url: "http://127.0.0.1:5279"
|
||||
# Timeout for daemon requests in seconds
|
||||
# Only used when running with --p2p
|
||||
timeout_seconds: 60
|
||||
|
||||
general:
|
||||
@@ -386,12 +388,14 @@ channels:
|
||||
for channel in channels:
|
||||
config += f''' - input: "{channel["input"]}"
|
||||
enabled: true
|
||||
content_mode: "non_video_only"
|
||||
'''
|
||||
|
||||
if not channels:
|
||||
config += """ # Add your channels here:
|
||||
# - input: "https://odysee.com/@ChannelName:1"
|
||||
# enabled: true
|
||||
# content_mode: "non_video_only"
|
||||
"""
|
||||
|
||||
return config
|
||||
@@ -405,54 +409,13 @@ def create_launcher_script(install_dir: Path, venv_path: Path) -> None:
|
||||
bin_dir = install_dir / "bin"
|
||||
bin_dir.mkdir(exist_ok=True)
|
||||
|
||||
# Create main launcher with daemon auto-check
|
||||
# Create main launcher
|
||||
launcher = bin_dir / "lbry-downloader"
|
||||
launcher_content = f"""#!/bin/bash
|
||||
# LBRY Downloader Launcher
|
||||
# Auto-checks and starts lbrynet daemon if needed
|
||||
|
||||
INSTALL_DIR="{install_dir}"
|
||||
VENV_DIR="{venv_path}"
|
||||
DAEMON_URL="http://127.0.0.1:5279"
|
||||
|
||||
# Check if daemon is running
|
||||
if ! curl -s "$DAEMON_URL" > /dev/null 2>&1; then
|
||||
echo "LBRY daemon not running. Starting..."
|
||||
|
||||
# Try to start daemon
|
||||
if command -v lbrynet &> /dev/null; then
|
||||
lbrynet start &
|
||||
echo "Waiting for daemon to start..."
|
||||
|
||||
# Wait up to 30 seconds for daemon
|
||||
for i in $(seq 1 30); do
|
||||
if curl -s "$DAEMON_URL" > /dev/null 2>&1; then
|
||||
echo "✓ Daemon started successfully"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
# Check if daemon started
|
||||
if ! curl -s "$DAEMON_URL" > /dev/null 2>&1; then
|
||||
echo "✗ Failed to start daemon automatically"
|
||||
echo ""
|
||||
echo "Please start it manually:"
|
||||
echo " lbrynet start"
|
||||
echo ""
|
||||
echo "Or see LINUX_SETUP.md for installation instructions"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
echo "✗ lbrynet not found in PATH"
|
||||
echo ""
|
||||
echo "Please install LBRY SDK:"
|
||||
echo " https://github.com/lbryio/lbry-sdk/releases"
|
||||
echo ""
|
||||
echo "Or see LINUX_SETUP.md for detailed instructions"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
cd "$INSTALL_DIR"
|
||||
source "$VENV_DIR/bin/activate"
|
||||
@@ -464,37 +427,13 @@ python main.py "$@"
|
||||
launcher.chmod(0o755)
|
||||
print_success(f"Created launcher: {launcher}")
|
||||
|
||||
# Create quick test script with daemon auto-check
|
||||
# Create quick test script
|
||||
test_script = bin_dir / "lbry-test"
|
||||
test_content = f"""#!/bin/bash
|
||||
# Quick test script
|
||||
|
||||
INSTALL_DIR="{install_dir}"
|
||||
VENV_DIR="{venv_path}"
|
||||
DAEMON_URL="http://127.0.0.1:5279"
|
||||
|
||||
# Check if daemon is running
|
||||
if ! curl -s "$DAEMON_URL" > /dev/null 2>&1; then
|
||||
echo "LBRY daemon not running. Starting..."
|
||||
|
||||
if command -v lbrynet &> /dev/null; then
|
||||
lbrynet start &
|
||||
echo "Waiting for daemon to start..."
|
||||
sleep 5
|
||||
|
||||
if curl -s "$DAEMON_URL" > /dev/null 2>&1; then
|
||||
echo "✓ Daemon started"
|
||||
else
|
||||
echo "✗ Failed to start daemon"
|
||||
echo "Run manually: lbrynet start"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
echo "✗ lbrynet not found. Install from:"
|
||||
echo " https://github.com/lbryio/lbry-sdk/releases"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
cd "$INSTALL_DIR"
|
||||
source "$VENV_DIR/bin/activate"
|
||||
@@ -533,10 +472,10 @@ def run_setup() -> None:
|
||||
"""Main setup function."""
|
||||
print(f"""
|
||||
{Colors.BOLD}╔══════════════════════════════════════════════════════════════╗
|
||||
║ LBRY Downloader - Interactive Setup ║
|
||||
║ Odysee/LBRY Downloader - Interactive Setup ║
|
||||
║ ║
|
||||
║ This script will help you install and configure the ║
|
||||
║ LBRY Downloader tool. ║
|
||||
║ direct-first archive downloader. ║
|
||||
╚══════════════════════════════════════════════════════════════╝{Colors.END}
|
||||
""")
|
||||
|
||||
@@ -662,20 +601,20 @@ def run_setup() -> None:
|
||||
║ Setup Complete! ║
|
||||
╚══════════════════════════════════════════════════════════════╝{Colors.END}
|
||||
|
||||
🎉 LBRY Downloader is ready to use!
|
||||
🎉 Odysee/LBRY Downloader is ready to use!
|
||||
|
||||
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
📋 QUICK START (Run these commands):
|
||||
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
|
||||
1️⃣ Start the LBRY daemon:
|
||||
lbrynet start
|
||||
1️⃣ Optional P2P prerequisite:
|
||||
Optional only for P2P mode: lbrynet start
|
||||
|
||||
2️⃣ Test your configuration:
|
||||
lbry-downloader --dry-run
|
||||
|
||||
3️⃣ Download content:
|
||||
lbry-downloader
|
||||
lbry-downloader --non-video-only
|
||||
|
||||
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
ℹ️ IMPORTANT NOTES:
|
||||
@@ -688,7 +627,7 @@ def run_setup() -> None:
|
||||
If {Colors.BOLD}lbry-downloader{Colors.END} command not found, run:
|
||||
source ~/.bashrc
|
||||
|
||||
If lbrynet is not installed, see LINUX_SETUP.md for installation.
|
||||
Direct mode is the default. Use {Colors.BOLD}--p2p{Colors.END} only if you explicitly want the local node path.
|
||||
|
||||
To add more channels later:
|
||||
nano {config_path}
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
"""Tests for per-channel content mode config and resolution."""
|
||||
|
||||
import pytest
|
||||
|
||||
from config_loader import ConfigError, _parse_config
|
||||
from main import resolve_content_filter_mode
|
||||
from models import ChannelConfig
|
||||
|
||||
|
||||
def test_channel_content_mode_parses_from_config():
|
||||
config = _parse_config(
|
||||
{
|
||||
"channels": [
|
||||
{
|
||||
"input": "https://odysee.com/@Files:1",
|
||||
"enabled": True,
|
||||
"content_mode": "non_video_only",
|
||||
}
|
||||
]
|
||||
}
|
||||
)
|
||||
|
||||
assert config.channels[0].content_mode == "non_video_only"
|
||||
|
||||
|
||||
def test_invalid_channel_content_mode_is_rejected():
|
||||
with pytest.raises(ConfigError) as exc_info:
|
||||
_parse_config(
|
||||
{
|
||||
"channels": [
|
||||
{
|
||||
"input": "https://odysee.com/@Files:1",
|
||||
"enabled": True,
|
||||
"content_mode": "zip_only",
|
||||
}
|
||||
]
|
||||
}
|
||||
)
|
||||
|
||||
assert "content_mode" in str(exc_info.value)
|
||||
|
||||
|
||||
def test_channel_content_mode_defaults_to_all():
|
||||
channel_config = ChannelConfig(input="https://odysee.com/@Mixed:1")
|
||||
|
||||
assert resolve_content_filter_mode(channel_config) == (
|
||||
False,
|
||||
False,
|
||||
"all content",
|
||||
)
|
||||
|
||||
|
||||
def test_channel_content_mode_applies_when_no_cli_override():
|
||||
channel_config = ChannelConfig(
|
||||
input="https://odysee.com/@Videos:1",
|
||||
content_mode="video_only",
|
||||
)
|
||||
|
||||
assert resolve_content_filter_mode(channel_config) == (
|
||||
True,
|
||||
False,
|
||||
"video-only (channel config)",
|
||||
)
|
||||
|
||||
|
||||
def test_cli_override_beats_channel_content_mode():
|
||||
channel_config = ChannelConfig(
|
||||
input="https://odysee.com/@Files:1",
|
||||
content_mode="non_video_only",
|
||||
)
|
||||
|
||||
assert resolve_content_filter_mode(channel_config, video_only=True) == (
|
||||
True,
|
||||
False,
|
||||
"video-only (CLI override)",
|
||||
)
|
||||
@@ -122,6 +122,50 @@ channels:
|
||||
|
||||
assert "filename_mode" in str(exc_info.value)
|
||||
|
||||
def test_valid_direct_retry_config(self, tmp_path):
|
||||
config_file = tmp_path / "config.yaml"
|
||||
config_file.write_text("""
|
||||
general:
|
||||
base_dir: "~/test"
|
||||
direct_base_urls:
|
||||
- "https://odysee.com/"
|
||||
- "https://mirror.example/"
|
||||
direct_max_retries_per_url: 3
|
||||
direct_retry_backoff_seconds: 1.5
|
||||
direct_auto_fallback_to_p2p: true
|
||||
|
||||
channels:
|
||||
- input: "https://odysee.com/@Test:1"
|
||||
enabled: true
|
||||
""")
|
||||
|
||||
config = load_config(str(config_file))
|
||||
|
||||
assert config.general.direct_base_urls == [
|
||||
"https://odysee.com",
|
||||
"https://mirror.example",
|
||||
]
|
||||
assert config.general.direct_max_retries_per_url == 3
|
||||
assert config.general.direct_retry_backoff_seconds == 1.5
|
||||
assert config.general.direct_auto_fallback_to_p2p is True
|
||||
|
||||
def test_invalid_direct_retry_backoff(self, tmp_path):
|
||||
config_file = tmp_path / "config.yaml"
|
||||
config_file.write_text("""
|
||||
general:
|
||||
base_dir: "~/test"
|
||||
direct_retry_backoff_seconds: 0
|
||||
|
||||
channels:
|
||||
- input: "https://odysee.com/@Test:1"
|
||||
enabled: true
|
||||
""")
|
||||
|
||||
with pytest.raises(ConfigError) as exc_info:
|
||||
load_config(str(config_file))
|
||||
|
||||
assert "direct_retry_backoff_seconds" in str(exc_info.value)
|
||||
|
||||
|
||||
class TestDefaultConfig:
|
||||
def test_create_default_config(self):
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
"""Tests for CLI content-type filtering."""
|
||||
|
||||
from main import filter_actions_by_content_type
|
||||
from models import DownloadAction
|
||||
|
||||
|
||||
def make_action(name: str, **metadata):
|
||||
base_metadata = {
|
||||
"source_name": name,
|
||||
"file_name": name,
|
||||
}
|
||||
base_metadata.update(metadata)
|
||||
return DownloadAction(
|
||||
claim_id=f"claim-{name}",
|
||||
claim_name=name,
|
||||
channel_claim_id="channel-1",
|
||||
version_token=f"version-{name}",
|
||||
action="download_new",
|
||||
uri=f"lbry://{name}",
|
||||
target_dir="/tmp",
|
||||
metadata=base_metadata,
|
||||
)
|
||||
|
||||
|
||||
def test_video_only_keeps_only_video_actions():
|
||||
video_action = make_action("clip.mp4", source_media_type="video/mp4")
|
||||
zip_action = make_action("bundle.zip", source_media_type="application/zip")
|
||||
|
||||
actions = filter_actions_by_content_type(
|
||||
[video_action, zip_action],
|
||||
video_only=True,
|
||||
)
|
||||
|
||||
assert actions == [video_action]
|
||||
|
||||
|
||||
def test_non_video_only_keeps_zip_and_skips_video():
|
||||
video_action = make_action("clip.mp4", source_media_type="video/mp4")
|
||||
zip_action = make_action("bundle.zip", source_media_type="application/zip")
|
||||
|
||||
actions = filter_actions_by_content_type(
|
||||
[video_action, zip_action],
|
||||
non_video_only=True,
|
||||
)
|
||||
|
||||
assert actions == [zip_action]
|
||||
|
||||
|
||||
def test_video_filter_falls_back_to_filename_extension():
|
||||
video_action = make_action("clip.mkv")
|
||||
other_action = make_action("bundle.zip")
|
||||
|
||||
actions = filter_actions_by_content_type(
|
||||
[video_action, other_action],
|
||||
video_only=True,
|
||||
)
|
||||
|
||||
assert actions == [video_action]
|
||||
@@ -0,0 +1,183 @@
|
||||
"""Tests for the direct Odysee downloader."""
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import requests
|
||||
from direct_downloader import DirectDownloader
|
||||
from models import Config, DownloadAction
|
||||
|
||||
|
||||
def make_action(**metadata_overrides):
|
||||
metadata = {
|
||||
"normalized_name": "veritasium-trailer",
|
||||
"name": "veritasium-trailer",
|
||||
"source_name": "veritasium-trailer.mp4",
|
||||
}
|
||||
metadata.update(metadata_overrides)
|
||||
|
||||
return DownloadAction(
|
||||
claim_id="19201da2f75357d70b151f7d87ddd94227884126",
|
||||
claim_name="veritasium-trailer",
|
||||
channel_claim_id="fb364ef587872515f545a5b4b3182b58073f230f",
|
||||
version_token="sd_deadbeefdeadbeef",
|
||||
action="download_new",
|
||||
uri="lbry://veritasium-trailer#19201da2f75357d70b151f7d87ddd94227884126",
|
||||
target_dir="/tmp/lbry-test",
|
||||
metadata=metadata,
|
||||
)
|
||||
|
||||
|
||||
class TestDirectDownloader:
|
||||
def test_build_stream_url_uses_normalized_name(self):
|
||||
downloader = DirectDownloader(Config(), Path("/tmp"))
|
||||
action = make_action()
|
||||
|
||||
assert (
|
||||
downloader.build_stream_url(action)
|
||||
== "https://odysee.com/$/stream/veritasium-trailer/"
|
||||
"19201da2f75357d70b151f7d87ddd94227884126"
|
||||
)
|
||||
|
||||
def test_build_stream_url_quotes_special_slug(self):
|
||||
downloader = DirectDownloader(Config(), Path("/tmp"))
|
||||
action = make_action(normalized_name="Title With Spaces & Symbols")
|
||||
|
||||
assert downloader.build_stream_url(action).startswith(
|
||||
"https://odysee.com/$/stream/Title%20With%20Spaces%20%26%20Symbols/"
|
||||
)
|
||||
|
||||
def test_build_download_url_uses_claim_id(self):
|
||||
downloader = DirectDownloader(Config(), Path("/tmp"))
|
||||
action = make_action()
|
||||
|
||||
assert (
|
||||
downloader.build_download_url(action)
|
||||
== "https://odysee.com/$/download/veritasium-trailer/"
|
||||
"19201da2f75357d70b151f7d87ddd94227884126"
|
||||
)
|
||||
|
||||
def test_file_claims_prefer_download_endpoint(self):
|
||||
downloader = DirectDownloader(Config(), Path("/tmp"))
|
||||
action = make_action(
|
||||
source_name="archive.zip",
|
||||
source_media_type="application/zip",
|
||||
stream_type="binary",
|
||||
)
|
||||
|
||||
assert downloader.build_candidate_urls(action) == [
|
||||
"https://odysee.com/$/download/veritasium-trailer/"
|
||||
"19201da2f75357d70b151f7d87ddd94227884126",
|
||||
"https://odysee.com/$/stream/veritasium-trailer/"
|
||||
"19201da2f75357d70b151f7d87ddd94227884126",
|
||||
]
|
||||
|
||||
def test_video_claims_prefer_stream_endpoint(self):
|
||||
downloader = DirectDownloader(Config(), Path("/tmp"))
|
||||
action = make_action(
|
||||
source_media_type="video/mp4",
|
||||
stream_type="video",
|
||||
)
|
||||
|
||||
assert downloader.build_candidate_urls(action) == [
|
||||
"https://odysee.com/$/stream/veritasium-trailer/"
|
||||
"19201da2f75357d70b151f7d87ddd94227884126",
|
||||
"https://odysee.com/$/download/veritasium-trailer/"
|
||||
"19201da2f75357d70b151f7d87ddd94227884126",
|
||||
]
|
||||
|
||||
def test_candidate_urls_include_configured_mirrors(self):
|
||||
config = Config()
|
||||
config.general.direct_base_urls = [
|
||||
"https://odysee.com",
|
||||
"https://mirror.example",
|
||||
]
|
||||
downloader = DirectDownloader(config, Path("/tmp"))
|
||||
action = make_action(
|
||||
source_media_type="video/mp4",
|
||||
stream_type="video",
|
||||
)
|
||||
|
||||
assert downloader.build_candidate_urls(action) == [
|
||||
"https://odysee.com/$/stream/veritasium-trailer/"
|
||||
"19201da2f75357d70b151f7d87ddd94227884126",
|
||||
"https://odysee.com/$/download/veritasium-trailer/"
|
||||
"19201da2f75357d70b151f7d87ddd94227884126",
|
||||
"https://mirror.example/$/stream/veritasium-trailer/"
|
||||
"19201da2f75357d70b151f7d87ddd94227884126",
|
||||
"https://mirror.example/$/download/veritasium-trailer/"
|
||||
"19201da2f75357d70b151f7d87ddd94227884126",
|
||||
]
|
||||
|
||||
def test_429_retries_use_backoff_budget(self, monkeypatch, tmp_path):
|
||||
config = Config()
|
||||
config.general.direct_max_retries_per_url = 2
|
||||
config.general.direct_retry_backoff_seconds = 0.01
|
||||
downloader = DirectDownloader(config, Path("/tmp"))
|
||||
file_path = tmp_path / "retry-test.mp4"
|
||||
|
||||
calls = {"count": 0}
|
||||
sleeps = []
|
||||
|
||||
class FakeResponse:
|
||||
headers = {}
|
||||
|
||||
def raise_for_status(self):
|
||||
http_error = requests.exceptions.HTTPError("429")
|
||||
http_error.response = SimpleNamespace(status_code=429)
|
||||
raise http_error
|
||||
|
||||
def fake_get(*args, **kwargs):
|
||||
calls["count"] += 1
|
||||
return FakeResponse()
|
||||
|
||||
monkeypatch.setattr(downloader.session, "get", fake_get)
|
||||
monkeypatch.setattr("direct_downloader.time.sleep", lambda seconds: sleeps.append(seconds))
|
||||
|
||||
success, reason = downloader._download_file(
|
||||
"https://odysee.com/$/download/test/abc",
|
||||
file_path,
|
||||
"retry-test",
|
||||
)
|
||||
|
||||
assert success is False
|
||||
assert "rate-limited" in reason.lower()
|
||||
assert calls["count"] == 3
|
||||
assert sleeps == [0.01, 0.02]
|
||||
|
||||
def test_skip_existing_logs_existing_file_path(self, caplog):
|
||||
downloader = DirectDownloader(Config(), Path("/tmp"))
|
||||
action = make_action(
|
||||
existing_file_relpath="channels/test/claims/item/versions/sd/file.zip",
|
||||
)
|
||||
action.action = "skip_existing"
|
||||
|
||||
with caplog.at_level(logging.INFO):
|
||||
success = downloader.download(action)
|
||||
|
||||
assert success is True
|
||||
assert "Skipping existing file" in caplog.text
|
||||
assert "/tmp/channels/test/claims/item/versions/sd/file.zip" in caplog.text
|
||||
|
||||
def test_download_logs_destination_path(self, monkeypatch, caplog, tmp_path):
|
||||
downloader = DirectDownloader(Config(), Path("/tmp"))
|
||||
action = make_action()
|
||||
action.target_dir = str(tmp_path)
|
||||
|
||||
monkeypatch.setattr(
|
||||
downloader,
|
||||
"_download_file",
|
||||
lambda url, file_path, claim_name: (True, None),
|
||||
)
|
||||
monkeypatch.setattr(downloader, "_write_metadata", lambda *args, **kwargs: None)
|
||||
monkeypatch.setattr(downloader, "_write_checksum", lambda *args, **kwargs: None)
|
||||
target_file = tmp_path / "veritasium-trailer.mp4"
|
||||
target_file.write_bytes(b"data")
|
||||
|
||||
with caplog.at_level(logging.INFO):
|
||||
success = downloader.download(action)
|
||||
|
||||
assert success is True
|
||||
assert "Saving to:" in caplog.text
|
||||
assert str(target_file) in caplog.text
|
||||
@@ -104,6 +104,10 @@ class TestPlanner:
|
||||
|
||||
assert len(actions) == 1
|
||||
assert actions[0].action == "skip_existing"
|
||||
assert (
|
||||
actions[0].metadata["existing_file_relpath"]
|
||||
== f"channels/Test__channel123/claims/{claim_folder}/versions/{version_token}/video.mp4"
|
||||
)
|
||||
|
||||
def test_new_version_detected(self):
|
||||
channel = Channel(
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
"""Tests for transport mode selection."""
|
||||
|
||||
from main import resolve_transport_mode
|
||||
|
||||
|
||||
def test_transport_mode_defaults_to_direct():
|
||||
assert resolve_transport_mode() == (True, "direct")
|
||||
|
||||
|
||||
def test_transport_mode_p2p_override_wins():
|
||||
assert resolve_transport_mode(p2p=True) == (False, "p2p")
|
||||
|
||||
|
||||
def test_transport_mode_explicit_direct():
|
||||
assert resolve_transport_mode(direct=True) == (True, "direct")
|
||||
@@ -256,6 +256,7 @@ def extract_claim_metadata(claim_data: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"normalized_name": claim.get("normalized_name"),
|
||||
"claim_type": claim.get("claim_type"),
|
||||
"value_type": claim.get("value_type"),
|
||||
"stream_type": value.get("stream_type"),
|
||||
"permanent_url": claim.get("permanent_url"),
|
||||
"short_url": claim.get("short_url"),
|
||||
"canonical_url": claim.get("canonical_url"),
|
||||
|
||||
Reference in new issue
Block a user