mirror of
https://github.com/PR0M3TH3AN/LBRY-Downloader.git
synced 2026-10-27 08:22:24 +00:00
Add direct-first download modes and resilience controls
This commit is contained in:
1 parent
66d7a02458
commit
7b50110f2a
22 files changed
+1275
-488
No files matched your search
+65
-22
@@ -32,7 +32,7 @@ cd LBRY-Downloader
|
|||||||
|
|
||||||
This script will:
|
This script will:
|
||||||
1. Check and install system dependencies
|
1. Check and install system dependencies
|
||||||
2. Download and install LBRY SDK (lbrynet) - **No blockchain sync required!**
|
2. Set up the downloader for direct mode by default
|
||||||
3. Set up a Python virtual environment
|
3. Set up a Python virtual environment
|
||||||
4. Install Python packages
|
4. Install Python packages
|
||||||
5. Ask for your download location
|
5. Ask for your download location
|
||||||
@@ -42,11 +42,13 @@ This script will:
|
|||||||
|
|
||||||
**Want to uninstall later?** Just run `./uninstall.py`
|
**Want to uninstall later?** Just run `./uninstall.py`
|
||||||
|
|
||||||
### About LBRY SDK (lbrynet) - IMPORTANT
|
### About LBRY SDK (lbrynet) - Optional P2P Mode Only
|
||||||
|
|
||||||
**Q: Does this download the LBRY blockchain?**
|
**Q: Does this download the LBRY blockchain?**
|
||||||
|
|
||||||
**A:** The LBRY SDK downloads **blockchain headers** (not full blocks), which is required to function:
|
**A:** Only if you choose to use `--p2p`. The default direct mode does not require this.
|
||||||
|
|
||||||
|
When you do use `--p2p`, the LBRY SDK downloads **blockchain headers** (not full blocks):
|
||||||
|
|
||||||
**First-time setup (one-time):**
|
**First-time setup (one-time):**
|
||||||
- Downloads ~1.7 million block headers
|
- Downloads ~1.7 million block headers
|
||||||
@@ -61,7 +63,7 @@ This script will:
|
|||||||
**What this means:**
|
**What this means:**
|
||||||
- The LBRY SDK needs this data to verify which content claims are valid
|
- The LBRY SDK needs this data to verify which content claims are valid
|
||||||
- This is lighter than Bitcoin's full blockchain sync
|
- This is lighter than Bitcoin's full blockchain sync
|
||||||
- But it IS required and will take time on first launch
|
- But it is only required for the optional P2P path
|
||||||
- You can stop it with `Ctrl+C` or `lbrynet stop`
|
- You can stop it with `Ctrl+C` or `lbrynet stop`
|
||||||
|
|
||||||
**Privacy note:** This connects to LBRY's network and syncs blockchain data to verify content. This is how the LBRY protocol works.
|
**Privacy note:** This connects to LBRY's network and syncs blockchain data to verify content. This is how the LBRY protocol works.
|
||||||
@@ -94,9 +96,9 @@ python3 --version # Should be 3.8 or higher
|
|||||||
pip3 --version
|
pip3 --version
|
||||||
```
|
```
|
||||||
|
|
||||||
### 2. Install LBRY SDK (lbrynet daemon)
|
### 2. Optional: Install LBRY SDK (`lbrynet`) for P2P Mode
|
||||||
|
|
||||||
The LBRY SDK is required to communicate with the LBRY network.
|
You only need the LBRY SDK if you want to run the downloader with `--p2p`. The default direct mode does not require it.
|
||||||
|
|
||||||
#### Option A: Download Pre-built Binary (Recommended)
|
#### Option A: Download Pre-built Binary (Recommended)
|
||||||
|
|
||||||
@@ -250,18 +252,22 @@ channels:
|
|||||||
# Technology channels
|
# Technology channels
|
||||||
- input: "https://odysee.com/@LinusTechTips:1"
|
- input: "https://odysee.com/@LinusTechTips:1"
|
||||||
enabled: true
|
enabled: true
|
||||||
|
content_mode: "video_only"
|
||||||
|
|
||||||
# Educational content
|
# Educational content
|
||||||
- input: "https://odysee.com/@veritasium:1"
|
- input: "https://odysee.com/@veritasium:1"
|
||||||
enabled: true
|
enabled: true
|
||||||
|
content_mode: "video_only"
|
||||||
|
|
||||||
# News/Commentary
|
# News/Commentary
|
||||||
- input: "https://odysee.com/@Timcast:1"
|
- input: "https://odysee.com/@Timcast:1"
|
||||||
enabled: true
|
enabled: true
|
||||||
|
content_mode: "video_only"
|
||||||
|
|
||||||
# Documentaries
|
# File / document archives
|
||||||
- input: "https://odysee.com/@DocumentaryLab:1"
|
- input: "https://odysee.com/@DocumentaryLab:1"
|
||||||
enabled: true
|
enabled: true
|
||||||
|
content_mode: "non_video_only"
|
||||||
```
|
```
|
||||||
|
|
||||||
#### URL Formats Supported
|
#### URL Formats Supported
|
||||||
@@ -300,15 +306,25 @@ Add your channels under the `channels:` section:
|
|||||||
channels:
|
channels:
|
||||||
- input: "https://odysee.com/@YourFavoriteChannel:1"
|
- input: "https://odysee.com/@YourFavoriteChannel:1"
|
||||||
enabled: true
|
enabled: true
|
||||||
|
content_mode: "non_video_only"
|
||||||
tags_include: []
|
tags_include: []
|
||||||
tags_exclude: []
|
tags_exclude: []
|
||||||
|
|
||||||
- input: "https://odysee.com/@AnotherChannel:5"
|
- input: "https://odysee.com/@AnotherChannel:5"
|
||||||
enabled: true
|
enabled: true
|
||||||
|
content_mode: "video_only"
|
||||||
tags_include: []
|
tags_include: []
|
||||||
tags_exclude: []
|
tags_exclude: []
|
||||||
```
|
```
|
||||||
|
|
||||||
|
`content_mode` controls what each channel downloads:
|
||||||
|
|
||||||
|
- `all`: video and non-video files
|
||||||
|
- `video_only`: only videos such as mp4/webm/mkv
|
||||||
|
- `non_video_only`: only file bundles/docs such as zip/pdf
|
||||||
|
|
||||||
|
If you omit it, the default is `all`.
|
||||||
|
|
||||||
### 4. Configure Per-Channel Download Locations (Optional)
|
### 4. Configure Per-Channel Download Locations (Optional)
|
||||||
|
|
||||||
By default, all downloads go to `~/Documents/lbry-downloads/channels/`. You can specify custom locations for each channel:
|
By default, all downloads go to `~/Documents/lbry-downloads/channels/`. You can specify custom locations for each channel:
|
||||||
@@ -318,16 +334,19 @@ channels:
|
|||||||
# Downloads to default location
|
# Downloads to default location
|
||||||
- input: "https://odysee.com/@RegularChannel:1"
|
- input: "https://odysee.com/@RegularChannel:1"
|
||||||
enabled: true
|
enabled: true
|
||||||
|
content_mode: "all"
|
||||||
|
|
||||||
# Downloads to custom location (external drive)
|
# Downloads only videos to a custom location (external drive)
|
||||||
- input: "https://odysee.com/@LargeVideos:1"
|
- input: "https://odysee.com/@LargeVideos:1"
|
||||||
enabled: true
|
enabled: true
|
||||||
download_path: "/mnt/media/videos"
|
download_path: "/mnt/media/videos"
|
||||||
|
content_mode: "video_only"
|
||||||
|
|
||||||
# Downloads to custom location (home directory)
|
# Downloads only non-video files to a custom location
|
||||||
- input: "https://odysee.com/@Music:1"
|
- input: "https://odysee.com/@Music:1"
|
||||||
enabled: true
|
enabled: true
|
||||||
download_path: "~/Music/LBRY"
|
download_path: "~/Music/LBRY"
|
||||||
|
content_mode: "non_video_only"
|
||||||
```
|
```
|
||||||
|
|
||||||
**Why use custom locations?**
|
**Why use custom locations?**
|
||||||
@@ -363,7 +382,7 @@ general:
|
|||||||
```
|
```
|
||||||
|
|
||||||
The tool downloads **most recent items first** (sorted by publish date), so:
|
The tool downloads **most recent items first** (sorted by publish date), so:
|
||||||
- `download_limit: 10` = Download the 10 newest videos
|
- `download_limit: 10` = Download the 10 newest matching items for that channel
|
||||||
- Older items remain in the channel but won't be downloaded yet
|
- Older items remain in the channel but won't be downloaded yet
|
||||||
- On next run, it will download the next 10 (if new items were published)
|
- On next run, it will download the next 10 (if new items were published)
|
||||||
|
|
||||||
@@ -382,7 +401,7 @@ Once you're happy with how it works, increase the limit or set to 0.
|
|||||||
|
|
||||||
## Running the Daemon
|
## Running the Daemon
|
||||||
|
|
||||||
The LBRY daemon (`lbrynet`) must be running before using the downloader.
|
The LBRY daemon (`lbrynet`) is only required if you want to use `--p2p`. The default direct mode does not require it.
|
||||||
|
|
||||||
### Start the Daemon
|
### Start the Daemon
|
||||||
|
|
||||||
@@ -456,16 +475,40 @@ python3 main.py --dry-run
|
|||||||
|
|
||||||
This shows what would be downloaded without actually downloading anything.
|
This shows what would be downloaded without actually downloading anything.
|
||||||
|
|
||||||
|
If you mostly want zip/doc/file claims and want to skip videos, the most useful first test is:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python3 main.py --dry-run --non-video-only
|
||||||
|
```
|
||||||
|
|
||||||
### Normal Run
|
### Normal Run
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Make sure lbrynet is running
|
# Run the downloader using direct mode (default) and each channel's configured content_mode
|
||||||
lbrynet status
|
|
||||||
|
|
||||||
# Run the downloader
|
|
||||||
python3 main.py
|
python3 main.py
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### Useful Run Variants
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Recommended default if you mainly want non-video files such as zip bundles
|
||||||
|
python3 main.py --non-video-only
|
||||||
|
|
||||||
|
# Force video-only mode for every channel for this run
|
||||||
|
python3 main.py --video-only
|
||||||
|
|
||||||
|
# Force non-video-only mode for every channel for this run
|
||||||
|
python3 main.py --non-video-only
|
||||||
|
|
||||||
|
# Use the local LBRY daemon and peer-to-peer mode instead of direct mode
|
||||||
|
python3 main.py --p2p
|
||||||
|
|
||||||
|
# Use P2P mode with a run-wide content override
|
||||||
|
python3 main.py --p2p --non-video-only
|
||||||
|
```
|
||||||
|
|
||||||
|
CLI flags `--video-only` and `--non-video-only` override each channel's `content_mode` for that run only. `--p2p` overrides the default direct transport mode for that run only.
|
||||||
|
|
||||||
### Custom Config Location
|
### Custom Config Location
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
@@ -590,7 +633,7 @@ lbc_mainnet: added BlockHeightEvent(height=1665210, change=2001)
|
|||||||
|
|
||||||
**What this means:**
|
**What this means:**
|
||||||
- The LBRY SDK is downloading blockchain headers (not full blocks)
|
- The LBRY SDK is downloading blockchain headers (not full blocks)
|
||||||
- This is required to verify content claims
|
- This is only required when using `--p2p` to verify content claims
|
||||||
- Takes 5-10 minutes on first run
|
- Takes 5-10 minutes on first run
|
||||||
- Subsequent runs are instant
|
- Subsequent runs are instant
|
||||||
|
|
||||||
@@ -859,9 +902,9 @@ cd ~/Documents/LBRY-Downloader
|
|||||||
./bin/lbry-test # Automatically runs with --dry-run
|
./bin/lbry-test # Automatically runs with --dry-run
|
||||||
```
|
```
|
||||||
|
|
||||||
### Prerequisites: Start the Daemon
|
### Optional Prerequisite: Start the Daemon for P2P Mode
|
||||||
|
|
||||||
**Important:** Always start the LBRY daemon before running the downloader:
|
Only do this if you plan to run with `--p2p`:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Check if daemon is running
|
# Check if daemon is running
|
||||||
@@ -886,14 +929,14 @@ pip3 install --user -r requirements.txt
|
|||||||
python3 init.py
|
python3 init.py
|
||||||
|
|
||||||
# Daily use
|
# Daily use
|
||||||
lbrynet start # Start daemon
|
|
||||||
~/Documents/LBRY-Downloader/bin/lbry-downloader --dry-run # Test first
|
~/Documents/LBRY-Downloader/bin/lbry-downloader --dry-run # Test first
|
||||||
~/Documents/LBRY-Downloader/bin/lbry-downloader # Run sync
|
~/Documents/LBRY-Downloader/bin/lbry-downloader --non-video-only # Recommended direct-mode run
|
||||||
lbrynet stop # Stop daemon (optional)
|
~/Documents/LBRY-Downloader/bin/lbry-downloader --p2p # Optional local-node mode
|
||||||
|
|
||||||
# Or if in PATH
|
# Or if in PATH
|
||||||
lbry-downloader --dry-run
|
lbry-downloader --dry-run
|
||||||
lbry-downloader
|
lbry-downloader --non-video-only
|
||||||
|
lbry-downloader --p2p
|
||||||
|
|
||||||
# Locations
|
# Locations
|
||||||
Install: ~/Documents/LBRY-Downloader/
|
Install: ~/Documents/LBRY-Downloader/
|
||||||
|
|||||||
+38
-3
@@ -39,17 +39,36 @@ Edit `~/Documents/lbry-downloads/config.yaml`:
|
|||||||
channels:
|
channels:
|
||||||
- input: "https://odysee.com/@YourFavoriteChannel:1"
|
- input: "https://odysee.com/@YourFavoriteChannel:1"
|
||||||
enabled: true
|
enabled: true
|
||||||
|
content_mode: "non_video_only"
|
||||||
- input: "https://odysee.com/@AnotherChannel:5"
|
- input: "https://odysee.com/@AnotherChannel:5"
|
||||||
enabled: true
|
enabled: true
|
||||||
|
content_mode: "video_only"
|
||||||
```
|
```
|
||||||
|
|
||||||
**Tip:** Just paste the Odysee URL exactly as it appears in your browser!
|
**Tip:** Just paste the Odysee URL exactly as it appears in your browser!
|
||||||
|
|
||||||
|
### Per-Channel Content Mode
|
||||||
|
|
||||||
|
You can choose what each channel downloads:
|
||||||
|
|
||||||
|
- `content_mode: "all"` downloads video and non-video files
|
||||||
|
- `content_mode: "video_only"` downloads only video files
|
||||||
|
- `content_mode: "non_video_only"` downloads only zip/PDF/other non-video files
|
||||||
|
|
||||||
|
Most users who want file bundles and archives should start with:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
channels:
|
||||||
|
- input: "https://odysee.com/@YourFavoriteChannel:1"
|
||||||
|
enabled: true
|
||||||
|
content_mode: "non_video_only"
|
||||||
|
```
|
||||||
|
|
||||||
## Launching
|
## Launching
|
||||||
|
|
||||||
### Start the LBRY Daemon First
|
### Optional: Start the LBRY Daemon First
|
||||||
|
|
||||||
**Important:** The LBRY daemon must be running before using the downloader:
|
You only need the LBRY daemon if you plan to use `--p2p`. The default direct mode does not require it.
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Start the daemon
|
# Start the daemon
|
||||||
@@ -90,9 +109,21 @@ python main.py --dry-run
|
|||||||
# Test what would be downloaded (dry run) - ALWAYS DO THIS FIRST
|
# Test what would be downloaded (dry run) - ALWAYS DO THIS FIRST
|
||||||
lbry-downloader --dry-run
|
lbry-downloader --dry-run
|
||||||
|
|
||||||
# Actually download content
|
# Recommended default: download only non-video files such as zip bundles
|
||||||
|
lbry-downloader --non-video-only
|
||||||
|
|
||||||
|
# Actually download using each channel's configured content_mode
|
||||||
lbry-downloader
|
lbry-downloader
|
||||||
|
|
||||||
|
# Download only videos for every channel for this run
|
||||||
|
lbry-downloader --video-only
|
||||||
|
|
||||||
|
# Download only non-video files for every channel for this run
|
||||||
|
lbry-downloader --non-video-only
|
||||||
|
|
||||||
|
# Use the local LBRY node instead of the default direct mode
|
||||||
|
lbry-downloader --p2p
|
||||||
|
|
||||||
# Use custom config file
|
# Use custom config file
|
||||||
lbry-downloader --config ./my-config.yaml
|
lbry-downloader --config ./my-config.yaml
|
||||||
|
|
||||||
@@ -100,6 +131,8 @@ lbry-downloader --config ./my-config.yaml
|
|||||||
lbry-downloader --help
|
lbry-downloader --help
|
||||||
```
|
```
|
||||||
|
|
||||||
|
CLI flags `--video-only` and `--non-video-only` override per-channel `content_mode` for that run only. `--p2p` overrides the default direct transport mode for that run only.
|
||||||
|
|
||||||
## Understanding Output
|
## Understanding Output
|
||||||
|
|
||||||
```
|
```
|
||||||
@@ -160,6 +193,8 @@ Downloads are saved to:
|
|||||||
lbrynet start
|
lbrynet start
|
||||||
```
|
```
|
||||||
|
|
||||||
|
This only applies when you are using `--p2p`. Default direct mode does not require `lbrynet`.
|
||||||
|
|
||||||
**"Config file not found"**
|
**"Config file not found"**
|
||||||
```bash
|
```bash
|
||||||
python init.py
|
python init.py
|
||||||
|
|||||||
@@ -2,13 +2,23 @@
|
|||||||
|
|
||||||
A Python tool that incrementally syncs downloadable file claims from configured LBRY/Odysee channels into a structured local archive.
|
A Python tool that incrementally syncs downloadable file claims from configured LBRY/Odysee channels into a structured local archive.
|
||||||
|
|
||||||
|
## Recommended Defaults
|
||||||
|
|
||||||
|
For most users:
|
||||||
|
|
||||||
|
- Files only: `lbry-downloader --non-video-only`
|
||||||
|
- Videos only: `lbry-downloader --video-only`
|
||||||
|
- Mixed per-channel behavior: set `content_mode` in `config.yaml` and run `lbry-downloader`
|
||||||
|
- Optional local-node mode: `lbry-downloader --p2p`
|
||||||
|
|
||||||
## Features
|
## Features
|
||||||
|
|
||||||
- **Incremental Sync**: Only downloads new claims and updated versions
|
- **Incremental Sync**: Only downloads new claims and updated versions
|
||||||
- **Version Preservation**: Keeps old versions instead of overwriting them
|
- **Version Preservation**: Keeps old versions instead of overwriting them
|
||||||
- **Robust Identity**: Uses stable LBRY claim IDs, not display names or titles
|
- **Robust Identity**: Uses stable LBRY claim IDs, not display names or titles
|
||||||
- **Atomic State**: Database writes are atomic to prevent corruption
|
- **Atomic State**: Database writes are atomic to prevent corruption
|
||||||
- **Daemon-Based**: Talks directly to local `lbrynet` daemon via JSON-RPC
|
- **Direct Mode by Default**: Uses Odysee's public proxy + CDN without requiring a local node
|
||||||
|
- **Optional P2P Mode**: Can use a local `lbrynet` daemon when explicitly requested
|
||||||
- **Streaming Fallback**: Automatically uses daemon's streaming endpoint when P2P peers are unavailable
|
- **Streaming Fallback**: Automatically uses daemon's streaming endpoint when P2P peers are unavailable
|
||||||
- **Flexible Input**: Accepts Odysee URLs, LBRY URIs, or claim IDs
|
- **Flexible Input**: Accepts Odysee URLs, LBRY URIs, or claim IDs
|
||||||
- **Audit Trail**: Maintains run history in JSONL format
|
- **Audit Trail**: Maintains run history in JSONL format
|
||||||
@@ -41,21 +51,22 @@ cd LBRY-Downloader
|
|||||||
|
|
||||||
The setup script will:
|
The setup script will:
|
||||||
- ✅ Check and install system dependencies
|
- ✅ Check and install system dependencies
|
||||||
- ✅ Download and install LBRY SDK (lbrynet) - **No blockchain sync required!**
|
|
||||||
- ✅ Set up a Python virtual environment
|
- ✅ Set up a Python virtual environment
|
||||||
- ✅ Install Python packages
|
- ✅ Install Python packages
|
||||||
- ✅ Ask for your download location
|
- ✅ Ask for your download location
|
||||||
- ✅ Configure channels to download from
|
- ✅ Configure channels to download from
|
||||||
- ✅ Set download limits
|
- ✅ Set download limits
|
||||||
- ✅ Create launcher scripts
|
- ✅ Create launcher scripts
|
||||||
|
- ✅ Prepare a direct-first downloader setup that works without a local node
|
||||||
|
- ✅ Leave `lbrynet` as an optional install only for users who want `--p2p`
|
||||||
|
|
||||||
**Important:** The LBRY SDK (lbrynet) requires syncing blockchain **headers** (not full blocks) to function. On first run:
|
**Important:** Direct mode is now the default and does not require `lbrynet`.
|
||||||
|
|
||||||
|
If you choose to use `--p2p`, then `lbrynet` requires syncing blockchain **headers** (not full blocks) on first use:
|
||||||
- Downloads ~1.7 million block headers (takes 5-10 minutes)
|
- Downloads ~1.7 million block headers (takes 5-10 minutes)
|
||||||
- Uses minimal storage (headers only, not full blockchain)
|
- Uses minimal storage (headers only, not full blockchain)
|
||||||
- Subsequent starts are instant
|
- Subsequent starts are much faster
|
||||||
- This is required to verify content claims on the network
|
- This is only needed for the optional node-backed mode
|
||||||
|
|
||||||
This is lighter than Bitcoin's full blockchain sync, but still necessary for the downloader to work.
|
|
||||||
|
|
||||||
### Manual Install
|
### Manual Install
|
||||||
|
|
||||||
@@ -71,7 +82,9 @@ To remove LBRY Downloader:
|
|||||||
|
|
||||||
## Prerequisites
|
## Prerequisites
|
||||||
|
|
||||||
You need to have the LBRY SDK daemon (`lbrynet`) installed and running:
|
Direct mode is the default, so a local LBRY daemon is not required for normal use.
|
||||||
|
|
||||||
|
If you want to use P2P mode explicitly with `--p2p`, then you need to have the LBRY SDK daemon (`lbrynet`) installed and running:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# The daemon should be accessible at http://127.0.0.1:5279
|
# The daemon should be accessible at http://127.0.0.1:5279
|
||||||
@@ -104,12 +117,19 @@ general:
|
|||||||
max_workers: 2
|
max_workers: 2
|
||||||
dry_run: false
|
dry_run: false
|
||||||
include_reposts: false
|
include_reposts: false
|
||||||
|
direct_base_urls:
|
||||||
|
- "https://odysee.com"
|
||||||
|
direct_max_retries_per_url: 2
|
||||||
|
direct_retry_backoff_seconds: 2.0
|
||||||
|
direct_auto_fallback_to_p2p: false
|
||||||
|
|
||||||
channels:
|
channels:
|
||||||
- input: "https://odysee.com/@SomeChannel:1"
|
- input: "https://odysee.com/@SomeChannel:1"
|
||||||
enabled: true
|
enabled: true
|
||||||
|
content_mode: "all"
|
||||||
- input: "lbry://@AnotherChannel#5"
|
- input: "lbry://@AnotherChannel#5"
|
||||||
enabled: true
|
enabled: true
|
||||||
|
content_mode: "non_video_only"
|
||||||
```
|
```
|
||||||
|
|
||||||
### Finding Channel URLs
|
### Finding Channel URLs
|
||||||
@@ -183,6 +203,36 @@ channels:
|
|||||||
# No download_path - uses general.base_dir/channels/
|
# No download_path - uses general.base_dir/channels/
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### Per-Channel Content Mode
|
||||||
|
|
||||||
|
You can control the download type for each channel independently.
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
channels:
|
||||||
|
- input: "https://odysee.com/@VideoChannel:1"
|
||||||
|
enabled: true
|
||||||
|
content_mode: "video_only"
|
||||||
|
|
||||||
|
- input: "https://odysee.com/@FileArchive:1"
|
||||||
|
enabled: true
|
||||||
|
content_mode: "non_video_only"
|
||||||
|
|
||||||
|
- input: "https://odysee.com/@MixedChannel:1"
|
||||||
|
enabled: true
|
||||||
|
content_mode: "all"
|
||||||
|
```
|
||||||
|
|
||||||
|
Available values:
|
||||||
|
|
||||||
|
- `all`: Download both video and non-video files for that channel.
|
||||||
|
- `video_only`: Download only video files such as `.mp4`, `.webm`, and `.mkv`.
|
||||||
|
- `non_video_only`: Download only non-video files such as `.zip`, PDFs, and similar file claims.
|
||||||
|
|
||||||
|
Precedence:
|
||||||
|
|
||||||
|
- Per-channel `content_mode` is the normal default for that channel.
|
||||||
|
- CLI flags `--video-only` and `--non-video-only` override all channels for a single run.
|
||||||
|
|
||||||
### Download Limit
|
### Download Limit
|
||||||
|
|
||||||
Limit how many new downloads per channel (downloads most recent first):
|
Limit how many new downloads per channel (downloads most recent first):
|
||||||
@@ -201,6 +251,33 @@ This is useful for:
|
|||||||
- **Large channels**: Gradually sync channels with thousands of videos
|
- **Large channels**: Gradually sync channels with thousands of videos
|
||||||
- **Bandwidth**: Control how much data is downloaded per run
|
- **Bandwidth**: Control how much data is downloaded per run
|
||||||
|
|
||||||
|
### Direct-Mode Rate Limit Fallbacks
|
||||||
|
|
||||||
|
If Odysee starts returning `429 Too Many Requests`, you can tune the direct downloader instead of hardcoding random public mirrors.
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
general:
|
||||||
|
direct_base_urls:
|
||||||
|
- "https://odysee.com"
|
||||||
|
# - "https://your-mirror.example"
|
||||||
|
direct_max_retries_per_url: 2
|
||||||
|
direct_retry_backoff_seconds: 2.0
|
||||||
|
direct_auto_fallback_to_p2p: false
|
||||||
|
```
|
||||||
|
|
||||||
|
What these do:
|
||||||
|
|
||||||
|
- `direct_base_urls`: direct download bases to try in order. Odysee should normally stay first.
|
||||||
|
- `direct_max_retries_per_url`: how many retries to allow after HTTP `429` for the same direct URL.
|
||||||
|
- `direct_retry_backoff_seconds`: base delay for exponential backoff after HTTP `429`.
|
||||||
|
- `direct_auto_fallback_to_p2p`: if `true`, a direct-mode run will try the local `lbrynet` daemon after repeated rate-limit failures.
|
||||||
|
|
||||||
|
Recommended pattern:
|
||||||
|
|
||||||
|
- Keep `https://odysee.com` first.
|
||||||
|
- Add your own mirror only if you actually control or trust it.
|
||||||
|
- Turn on `direct_auto_fallback_to_p2p: true` only if you also maintain a working local node.
|
||||||
|
|
||||||
## Launching the Application
|
## Launching the Application
|
||||||
|
|
||||||
After installation, you can launch LBRY Downloader in several ways:
|
After installation, you can launch LBRY Downloader in several ways:
|
||||||
@@ -248,7 +325,7 @@ Now you can run `lbry-downloader` from any directory!
|
|||||||
|
|
||||||
### Prerequisites Before Running
|
### Prerequisites Before Running
|
||||||
|
|
||||||
**Important:** Make sure the LBRY daemon is running first:
|
**Important:** If you are using `--p2p`, make sure the LBRY daemon is running first:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Start the daemon
|
# Start the daemon
|
||||||
@@ -260,15 +337,70 @@ lbrynet status
|
|||||||
|
|
||||||
## Usage
|
## Usage
|
||||||
|
|
||||||
|
### Recommended Default: Direct + Non-Video Downloads Only
|
||||||
|
|
||||||
|
If your main goal is to archive file attachments such as `.zip` bundles and skip video files such as `.mp4`, this should be the primary command you use:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Run with default config
|
python3 main.py --non-video-only
|
||||||
python main.py
|
```
|
||||||
|
|
||||||
# Use custom config
|
This mode keeps non-video claim downloads and skips video claims before the per-channel download limit is applied.
|
||||||
python main.py --config ./my-config.yaml
|
|
||||||
|
|
||||||
# Dry run - see what would happen
|
### Common Commands
|
||||||
python main.py --dry-run
|
|
||||||
|
```bash
|
||||||
|
# Recommended default: download only non-video files such as zip archives
|
||||||
|
python3 main.py --non-video-only
|
||||||
|
|
||||||
|
# Download only video files such as mp4/webm/mkv
|
||||||
|
python3 main.py --video-only
|
||||||
|
|
||||||
|
# Download everything supported by the tool
|
||||||
|
python3 main.py
|
||||||
|
|
||||||
|
# Preview what would be downloaded without downloading anything
|
||||||
|
python3 main.py --dry-run
|
||||||
|
|
||||||
|
# Preview only non-video downloads
|
||||||
|
python3 main.py --dry-run --non-video-only
|
||||||
|
|
||||||
|
# Use a custom config file
|
||||||
|
python3 main.py --config ./my-config.yaml
|
||||||
|
|
||||||
|
# Run explicitly in direct mode (normally unnecessary because this is the default)
|
||||||
|
python3 main.py --direct
|
||||||
|
|
||||||
|
# Use the local LBRY daemon and P2P network instead of direct mode
|
||||||
|
python3 main.py --p2p
|
||||||
|
|
||||||
|
# Use P2P mode but keep only non-video files
|
||||||
|
python3 main.py --p2p --non-video-only
|
||||||
|
|
||||||
|
# Use P2P mode but keep only video files
|
||||||
|
python3 main.py --p2p --video-only
|
||||||
|
```
|
||||||
|
|
||||||
|
### Flag Reference
|
||||||
|
|
||||||
|
- `--non-video-only`: Download only non-video claims such as `.zip`, PDFs, and other non-video files.
|
||||||
|
- `--video-only`: Download only video claims such as `.mp4`, `.webm`, and `.mkv`.
|
||||||
|
- `--dry-run`: Show what would be downloaded without changing anything.
|
||||||
|
- `--direct`: Use direct mode explicitly. This is already the default.
|
||||||
|
- `--p2p`: Use the local LBRY daemon and peer-to-peer network instead of the default direct mode.
|
||||||
|
- `--config PATH`: Use a config file other than `~/Documents/lbry-downloads/config.yaml`.
|
||||||
|
|
||||||
|
If you configure `content_mode` per channel, you usually only need the CLI content flags when you want a temporary run-wide override.
|
||||||
|
|
||||||
|
### Running Through the Launcher
|
||||||
|
|
||||||
|
If you installed the launcher script, the same commands work there too:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
lbry-downloader --non-video-only
|
||||||
|
lbry-downloader --video-only
|
||||||
|
lbry-downloader --dry-run --non-video-only
|
||||||
|
lbry-downloader --p2p --non-video-only
|
||||||
```
|
```
|
||||||
|
|
||||||
## Directory Structure
|
## Directory Structure
|
||||||
@@ -319,15 +451,27 @@ Make sure `lbrynet` is running:
|
|||||||
lbrynet start
|
lbrynet start
|
||||||
```
|
```
|
||||||
|
|
||||||
|
This only applies when you are using `--p2p`. Direct mode does not require a local daemon.
|
||||||
|
|
||||||
### "Config file not found"
|
### "Config file not found"
|
||||||
Create a config at the default location or specify a path:
|
Create a config at the default location or specify a path:
|
||||||
```bash
|
```bash
|
||||||
python main.py --config ./config.yaml
|
python3 main.py --config ./config.yaml
|
||||||
```
|
```
|
||||||
|
|
||||||
### Dry run shows no downloads
|
### Dry run shows no downloads
|
||||||
Check that your channels are enabled and have downloadable content (streams, not just reposts).
|
Check that your channels are enabled and have downloadable content (streams, not just reposts).
|
||||||
|
|
||||||
|
### HTTP 429 / Too Many Requests
|
||||||
|
|
||||||
|
If direct mode is being rate-limited:
|
||||||
|
|
||||||
|
1. Reduce how aggressively you rerun the downloader.
|
||||||
|
2. Increase `direct_retry_backoff_seconds`.
|
||||||
|
3. Keep `direct_max_retries_per_url` modest, such as `2` or `3`.
|
||||||
|
4. Add a trusted mirror to `direct_base_urls` if you actually have one.
|
||||||
|
5. Enable `direct_auto_fallback_to_p2p: true` if you want automatic fallback to the local node when direct mode keeps hitting `429`.
|
||||||
|
|
||||||
## Architecture
|
## Architecture
|
||||||
|
|
||||||
- `main.py` - Entry point and orchestration
|
- `main.py` - Entry point and orchestration
|
||||||
@@ -341,12 +485,12 @@ Check that your channels are enabled and have downloadable content (streams, not
|
|||||||
|
|
||||||
## How Downloads Work
|
## How Downloads Work
|
||||||
|
|
||||||
The downloader uses a two-stage approach for maximum reliability:
|
The downloader supports two transport modes:
|
||||||
|
|
||||||
1. **P2P Attempt**: First tries to download from LBRY's peer-to-peer network
|
1. **Direct mode (default)**: Uses Odysee's public proxy for metadata and Odysee CDN for the actual download
|
||||||
2. **Streaming Fallback**: If P2P peers are unavailable (common for older content), automatically falls back to the daemon's local streaming endpoint
|
2. **P2P mode (`--p2p`)**: Uses the local LBRY daemon and peer-to-peer network, with daemon streaming fallback when peers are unavailable
|
||||||
|
|
||||||
This ensures content downloads successfully even when the P2P network has no seeders.
|
Direct mode is the intended default for most users. P2P mode remains available when you specifically want node-backed downloading.
|
||||||
|
|
||||||
## License
|
## License
|
||||||
|
|
||||||
|
|||||||
+32
-2
@@ -4,8 +4,10 @@
|
|||||||
|
|
||||||
lbrynet:
|
lbrynet:
|
||||||
# URL of the local LBRY daemon (lbrynet)
|
# URL of the local LBRY daemon (lbrynet)
|
||||||
|
# Only used when running with --p2p
|
||||||
api_url: "http://127.0.0.1:5279"
|
api_url: "http://127.0.0.1:5279"
|
||||||
# Timeout for daemon requests in seconds
|
# Timeout for daemon requests in seconds
|
||||||
|
# Only used when running with --p2p
|
||||||
timeout_seconds: 60
|
timeout_seconds: 60
|
||||||
|
|
||||||
general:
|
general:
|
||||||
@@ -31,6 +33,18 @@ general:
|
|||||||
channel_page_size: 50
|
channel_page_size: 50
|
||||||
# Keep records of claims even if they disappear from the channel
|
# Keep records of claims even if they disappear from the channel
|
||||||
keep_missing_claim_records: true
|
keep_missing_claim_records: true
|
||||||
|
# Direct download base URLs to try in order. Keep Odysee first unless you
|
||||||
|
# explicitly want a mirror-first strategy.
|
||||||
|
direct_base_urls:
|
||||||
|
- "https://odysee.com"
|
||||||
|
# - "https://your-mirror.example"
|
||||||
|
# How many times to retry the same direct URL after HTTP 429
|
||||||
|
direct_max_retries_per_url: 2
|
||||||
|
# Base delay in seconds for exponential backoff after HTTP 429
|
||||||
|
direct_retry_backoff_seconds: 2.0
|
||||||
|
# If true, direct-mode runs will try the local lbrynet daemon as a fallback
|
||||||
|
# after repeated rate-limit failures. Requires a working local daemon.
|
||||||
|
direct_auto_fallback_to_p2p: false
|
||||||
|
|
||||||
# Download limit: Maximum number of downloads per channel per run
|
# Download limit: Maximum number of downloads per channel per run
|
||||||
#
|
#
|
||||||
@@ -67,20 +81,36 @@ channels:
|
|||||||
# - enabled: (optional) true/false - default: true
|
# - enabled: (optional) true/false - default: true
|
||||||
# - download_path: (optional) Custom download location for this channel
|
# - download_path: (optional) Custom download location for this channel
|
||||||
# If not set, uses general.base_dir/channels/
|
# If not set, uses general.base_dir/channels/
|
||||||
|
# - content_mode: (optional) What file types to download for this channel
|
||||||
|
# Values:
|
||||||
|
# - all = video and non-video files
|
||||||
|
# - video_only = only video files such as mp4/webm/mkv
|
||||||
|
# - non_video_only = only non-video files such as zip/pdf
|
||||||
|
# Default: all
|
||||||
|
# Note: CLI flags --video-only and --non-video-only override
|
||||||
|
# all channel content_mode settings for that run.
|
||||||
# - tags_include: (optional) List of tags to include (empty = all)
|
# - tags_include: (optional) List of tags to include (empty = all)
|
||||||
# - tags_exclude: (optional) List of tags to exclude
|
# - tags_exclude: (optional) List of tags to exclude
|
||||||
#
|
#
|
||||||
# Example 1: Basic channel
|
# Example 1: Basic channel
|
||||||
# - input: "https://odysee.com/@SomeChannel:1"
|
# - input: "https://odysee.com/@SomeChannel:1"
|
||||||
# enabled: true
|
# enabled: true
|
||||||
|
# content_mode: "all"
|
||||||
#
|
#
|
||||||
# Example 2: Channel with custom download location
|
# Example 2: Channel with custom download location and only videos
|
||||||
# - input: "https://odysee.com/@TechChannel:1"
|
# - input: "https://odysee.com/@TechChannel:1"
|
||||||
# enabled: true
|
# enabled: true
|
||||||
# download_path: "/mnt/media/videos/tech"
|
# download_path: "/mnt/media/videos/tech"
|
||||||
|
# content_mode: "video_only"
|
||||||
#
|
#
|
||||||
# Example 3: Channel with tag filtering
|
# Example 3: Channel that downloads only file bundles, docs, and other non-video claims
|
||||||
|
# - input: "https://odysee.com/@FileArchive:1"
|
||||||
|
# enabled: true
|
||||||
|
# content_mode: "non_video_only"
|
||||||
|
#
|
||||||
|
# Example 4: Channel with tag filtering
|
||||||
# - input: "lbry://@AnotherChannel#5"
|
# - input: "lbry://@AnotherChannel#5"
|
||||||
# enabled: true
|
# enabled: true
|
||||||
|
# content_mode: "all"
|
||||||
# tags_include: ["documentary", "science"]
|
# tags_include: ["documentary", "science"]
|
||||||
# tags_exclude: ["shorts"]
|
# tags_exclude: ["shorts"]
|
||||||
@@ -20,6 +20,9 @@ class ConfigError(Exception):
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
VALID_CONTENT_MODES = {"all", "video_only", "non_video_only"}
|
||||||
|
|
||||||
|
|
||||||
def load_config(config_path: Optional[str] = None) -> Config:
|
def load_config(config_path: Optional[str] = None) -> Config:
|
||||||
"""
|
"""
|
||||||
Load and validate the configuration file.
|
Load and validate the configuration file.
|
||||||
@@ -102,6 +105,21 @@ def _parse_config(raw: Dict[str, Any]) -> Config:
|
|||||||
download_limit=general_raw.get(
|
download_limit=general_raw.get(
|
||||||
"download_limit", config.general.download_limit
|
"download_limit", config.general.download_limit
|
||||||
),
|
),
|
||||||
|
direct_base_urls=general_raw.get(
|
||||||
|
"direct_base_urls", config.general.direct_base_urls
|
||||||
|
),
|
||||||
|
direct_max_retries_per_url=general_raw.get(
|
||||||
|
"direct_max_retries_per_url",
|
||||||
|
config.general.direct_max_retries_per_url,
|
||||||
|
),
|
||||||
|
direct_retry_backoff_seconds=general_raw.get(
|
||||||
|
"direct_retry_backoff_seconds",
|
||||||
|
config.general.direct_retry_backoff_seconds,
|
||||||
|
),
|
||||||
|
direct_auto_fallback_to_p2p=general_raw.get(
|
||||||
|
"direct_auto_fallback_to_p2p",
|
||||||
|
config.general.direct_auto_fallback_to_p2p,
|
||||||
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
# Validate and expand paths
|
# Validate and expand paths
|
||||||
@@ -121,6 +139,22 @@ def _parse_config(raw: Dict[str, Any]) -> Config:
|
|||||||
# 0 means "all" (no limit), negative values are not allowed
|
# 0 means "all" (no limit), negative values are not allowed
|
||||||
if config.general.download_limit < 0:
|
if config.general.download_limit < 0:
|
||||||
raise ConfigError("download_limit must be 0 (for all) or a positive number")
|
raise ConfigError("download_limit must be 0 (for all) or a positive number")
|
||||||
|
if config.general.direct_max_retries_per_url < 0:
|
||||||
|
raise ConfigError("direct_max_retries_per_url must be 0 or a positive number")
|
||||||
|
if config.general.direct_retry_backoff_seconds <= 0:
|
||||||
|
raise ConfigError("direct_retry_backoff_seconds must be greater than 0")
|
||||||
|
if not isinstance(config.general.direct_base_urls, list):
|
||||||
|
raise ConfigError("direct_base_urls must be a list")
|
||||||
|
if not config.general.direct_base_urls:
|
||||||
|
raise ConfigError("direct_base_urls must contain at least one base URL")
|
||||||
|
normalized_direct_base_urls = []
|
||||||
|
for idx, base_url in enumerate(config.general.direct_base_urls, start=1):
|
||||||
|
if not isinstance(base_url, str) or not base_url.strip():
|
||||||
|
raise ConfigError(
|
||||||
|
f"direct_base_urls entry {idx} must be a non-empty string"
|
||||||
|
)
|
||||||
|
normalized_direct_base_urls.append(base_url.rstrip("/"))
|
||||||
|
config.general.direct_base_urls = normalized_direct_base_urls
|
||||||
|
|
||||||
# Validate log_level
|
# Validate log_level
|
||||||
valid_levels = ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]
|
valid_levels = ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]
|
||||||
@@ -175,10 +209,18 @@ def _parse_channels(channels_raw: List[Dict[str, Any]]) -> List[ChannelConfig]:
|
|||||||
input=input_val,
|
input=input_val,
|
||||||
enabled=channel_raw.get("enabled", True),
|
enabled=channel_raw.get("enabled", True),
|
||||||
download_path=download_path,
|
download_path=download_path,
|
||||||
|
content_mode=channel_raw.get("content_mode", "all"),
|
||||||
tags_include=channel_raw.get("tags_include", []),
|
tags_include=channel_raw.get("tags_include", []),
|
||||||
tags_exclude=channel_raw.get("tags_exclude", []),
|
tags_exclude=channel_raw.get("tags_exclude", []),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
if channel.content_mode not in VALID_CONTENT_MODES:
|
||||||
|
raise ConfigError(
|
||||||
|
f"Channel {i + 1} has invalid content_mode "
|
||||||
|
f"'{channel.content_mode}'. Must be one of: "
|
||||||
|
f"{', '.join(sorted(VALID_CONTENT_MODES))}"
|
||||||
|
)
|
||||||
|
|
||||||
channels.append(channel)
|
channels.append(channel)
|
||||||
|
|
||||||
return channels
|
return channels
|
||||||
@@ -190,30 +232,58 @@ def create_default_config() -> str:
|
|||||||
# Place this file at ~/Documents/lbry-downloads/config.yaml
|
# Place this file at ~/Documents/lbry-downloads/config.yaml
|
||||||
|
|
||||||
lbrynet:
|
lbrynet:
|
||||||
|
# URL of the local LBRY daemon (lbrynet)
|
||||||
|
# Only used when running with --p2p
|
||||||
api_url: "http://127.0.0.1:5279"
|
api_url: "http://127.0.0.1:5279"
|
||||||
|
# Timeout for daemon requests in seconds
|
||||||
|
# Only used when running with --p2p
|
||||||
timeout_seconds: 60
|
timeout_seconds: 60
|
||||||
|
|
||||||
general:
|
general:
|
||||||
|
# Base directory for all downloads and state
|
||||||
base_dir: "~/Documents/lbry-downloads"
|
base_dir: "~/Documents/lbry-downloads"
|
||||||
|
# Path to the state database
|
||||||
state_file: "~/Documents/lbry-downloads/state/database.json"
|
state_file: "~/Documents/lbry-downloads/state/database.json"
|
||||||
|
# Number of concurrent downloads
|
||||||
max_workers: 2
|
max_workers: 2
|
||||||
|
# Logging level: DEBUG, INFO, WARNING, ERROR, CRITICAL
|
||||||
log_level: "INFO"
|
log_level: "INFO"
|
||||||
|
# If true, show what would be downloaded without actually downloading
|
||||||
dry_run: false
|
dry_run: false
|
||||||
|
# Verify that previously downloaded files still exist
|
||||||
verify_existing_files: true
|
verify_existing_files: true
|
||||||
|
# Write SHA256 checksums for downloaded files
|
||||||
write_checksums: true
|
write_checksums: true
|
||||||
|
# Filename handling: 'original' or 'safe'
|
||||||
filename_mode: "original"
|
filename_mode: "original"
|
||||||
|
# Include reposts in downloads
|
||||||
include_reposts: false
|
include_reposts: false
|
||||||
|
# Number of claims to fetch per page
|
||||||
channel_page_size: 50
|
channel_page_size: 50
|
||||||
|
# Keep records of claims even if they disappear from the channel
|
||||||
keep_missing_claim_records: true
|
keep_missing_claim_records: true
|
||||||
|
# Direct download bases to try in order
|
||||||
|
direct_base_urls:
|
||||||
|
- "https://odysee.com"
|
||||||
|
# Retries per direct URL after HTTP 429
|
||||||
|
direct_max_retries_per_url: 2
|
||||||
|
# Base backoff in seconds after HTTP 429 (uses exponential backoff)
|
||||||
|
direct_retry_backoff_seconds: 2.0
|
||||||
|
# If true, direct-mode runs will try the local node after repeated 429s
|
||||||
|
direct_auto_fallback_to_p2p: false
|
||||||
|
# Download limit: Number of most recent matching downloads per channel (0 = all)
|
||||||
|
download_limit: 10
|
||||||
|
|
||||||
channels:
|
channels:
|
||||||
# Add your channels here. Examples:
|
# Add your channels here. Examples:
|
||||||
# - input: "https://odysee.com/@SomeChannel:1"
|
# - input: "https://odysee.com/@SomeChannel:1"
|
||||||
# enabled: true
|
# enabled: true
|
||||||
|
# content_mode: "non_video_only"
|
||||||
# tags_include: []
|
# tags_include: []
|
||||||
# tags_exclude: []
|
# tags_exclude: []
|
||||||
# - input: "lbry://@AnotherChannel#5"
|
# - input: "lbry://@AnotherChannel#5"
|
||||||
# enabled: true
|
# enabled: true
|
||||||
|
# content_mode: "all"
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+212
-210
@@ -1,12 +1,11 @@
|
|||||||
"""Direct downloader module - downloads from Odysee CDN instead of P2P."""
|
"""Direct downloader module - downloads from Odysee's direct web endpoints."""
|
||||||
|
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
import re
|
|
||||||
import shutil
|
|
||||||
import time
|
import time
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Dict, Optional
|
from typing import Dict
|
||||||
|
from urllib.parse import quote
|
||||||
|
|
||||||
try:
|
try:
|
||||||
import requests
|
import requests
|
||||||
@@ -26,59 +25,127 @@ class DirectDownloadError(Exception):
|
|||||||
|
|
||||||
|
|
||||||
class DirectDownloader:
|
class DirectDownloader:
|
||||||
"""Downloads content directly from Odysee CDN."""
|
"""Downloads content directly from Odysee's web-exposed endpoints."""
|
||||||
|
|
||||||
|
stream_path = "/$/stream"
|
||||||
|
download_path = "/$/download"
|
||||||
|
|
||||||
def __init__(self, config: Config, base_dir: Path):
|
def __init__(self, config: Config, base_dir: Path):
|
||||||
self.config = config
|
self.config = config
|
||||||
self.base_dir = base_dir
|
self.base_dir = base_dir
|
||||||
|
self.direct_base_urls = [
|
||||||
|
base_url.rstrip("/") for base_url in config.general.direct_base_urls
|
||||||
|
]
|
||||||
self.session = requests.Session()
|
self.session = requests.Session()
|
||||||
self.session.headers.update(
|
self.session.headers.update(
|
||||||
{"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36"}
|
{
|
||||||
|
"User-Agent": (
|
||||||
|
"Mozilla/5.0 (X11; Linux x86_64) "
|
||||||
|
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||||
|
"Chrome/123.0.0.0 Safari/537.36"
|
||||||
|
)
|
||||||
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def _claim_slug(self, action: DownloadAction) -> str:
|
||||||
|
"""Choose the best available slug for Odysee URLs."""
|
||||||
|
slug = (
|
||||||
|
action.metadata.get("normalized_name")
|
||||||
|
or action.metadata.get("name")
|
||||||
|
or action.claim_name
|
||||||
|
)
|
||||||
|
return slug
|
||||||
|
|
||||||
|
def build_stream_url(self, action: DownloadAction, base_url: str | None = None) -> str:
|
||||||
|
"""Build Odysee's public direct-stream URL for a claim."""
|
||||||
|
slug = self._claim_slug(action)
|
||||||
|
chosen_base = (base_url or self.direct_base_urls[0]).rstrip("/")
|
||||||
|
return f"{chosen_base}{self.stream_path}/{quote(slug, safe='')}/{action.claim_id}"
|
||||||
|
|
||||||
|
def build_download_url(self, action: DownloadAction, base_url: str | None = None) -> str:
|
||||||
|
"""Build Odysee's public download-entry URL for a claim."""
|
||||||
|
slug = self._claim_slug(action)
|
||||||
|
chosen_base = (base_url or self.direct_base_urls[0]).rstrip("/")
|
||||||
|
return (
|
||||||
|
f"{chosen_base}{self.download_path}/{quote(slug, safe='')}/{action.claim_id}"
|
||||||
|
)
|
||||||
|
|
||||||
|
def _is_streaming_media(self, action: DownloadAction) -> bool:
|
||||||
|
"""Return True when the claim looks like audio/video media."""
|
||||||
|
stream_type = (action.metadata.get("stream_type") or "").lower()
|
||||||
|
media_type = (action.metadata.get("source_media_type") or "").lower()
|
||||||
|
|
||||||
|
if stream_type in {"video", "audio"}:
|
||||||
|
return True
|
||||||
|
if media_type.startswith("video/") or media_type.startswith("audio/"):
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
def build_candidate_urls(self, action: DownloadAction) -> list[str]:
|
||||||
|
"""Build Odysee direct-download URL candidates in priority order."""
|
||||||
|
candidate_urls = []
|
||||||
|
for base_url in self.direct_base_urls:
|
||||||
|
stream_url = self.build_stream_url(action, base_url=base_url)
|
||||||
|
download_url = self.build_download_url(action, base_url=base_url)
|
||||||
|
if self._is_streaming_media(action):
|
||||||
|
candidate_urls.extend([stream_url, download_url])
|
||||||
|
else:
|
||||||
|
candidate_urls.extend([download_url, stream_url])
|
||||||
|
return candidate_urls
|
||||||
|
|
||||||
def download(self, action: DownloadAction, dry_run: bool = False) -> bool:
|
def download(self, action: DownloadAction, dry_run: bool = False) -> bool:
|
||||||
"""
|
"""
|
||||||
Download a claim directly from Odysee CDN.
|
Download a claim directly from Odysee.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
action: The download action
|
action: The download action.
|
||||||
dry_run: If True, don't actually download
|
dry_run: If True, don't actually download.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
True if download succeeded
|
True if download succeeded.
|
||||||
"""
|
"""
|
||||||
if action.action == "skip_existing":
|
if action.action == "skip_existing":
|
||||||
logger.debug(f"Skipping already downloaded: {action.claim_name}")
|
existing_path = action.metadata.get("existing_file_relpath")
|
||||||
|
if existing_path:
|
||||||
|
logger.info(
|
||||||
|
"Skipping existing file: %s (%s)",
|
||||||
|
action.claim_name,
|
||||||
|
self.base_dir / existing_path,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
logger.info(f"Skipping existing file: {action.claim_name}")
|
||||||
return True
|
return True
|
||||||
|
|
||||||
if dry_run:
|
if dry_run:
|
||||||
logger.info(f"[DRY-RUN] Would download from CDN: {action.claim_name}")
|
logger.info(f"[DRY-RUN] Would download from Odysee: {action.claim_name}")
|
||||||
return True
|
return True
|
||||||
|
|
||||||
# Create directories
|
|
||||||
version_dir = Path(action.target_dir)
|
version_dir = Path(action.target_dir)
|
||||||
version_dir.mkdir(parents=True, exist_ok=True)
|
version_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
logger.info(f"Downloading from CDN: {action.claim_name}")
|
file_name = action.metadata.get("source_name") or f"{action.claim_name}.bin"
|
||||||
|
file_path = version_dir / file_name
|
||||||
|
|
||||||
|
logger.info(f"Downloading from Odysee: {action.claim_name}")
|
||||||
|
logger.info(f"Saving to: {file_path}")
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# Get download URL from Odysee
|
last_error = None
|
||||||
download_url = self._get_download_url(action.uri)
|
for index, download_url in enumerate(
|
||||||
if not download_url:
|
self.build_candidate_urls(action), start=1
|
||||||
raise DirectDownloadError("Could not get download URL from Odysee")
|
):
|
||||||
|
logger.info(f"Direct URL candidate {index}: {download_url}")
|
||||||
logger.info(f"Download URL: {download_url}")
|
success, failure_reason = self._download_file(
|
||||||
|
download_url, file_path, action.claim_name
|
||||||
# Download the file
|
)
|
||||||
file_path = self._download_file(
|
if success:
|
||||||
download_url, version_dir, action.claim_name
|
break
|
||||||
)
|
last_error = failure_reason
|
||||||
if not file_path:
|
else:
|
||||||
raise DirectDownloadError("File download failed")
|
raise DirectDownloadError(last_error or "Odysee direct download failed")
|
||||||
|
|
||||||
logger.info(f"Downloaded: {action.claim_name} -> {file_path.name}")
|
logger.info(f"Downloaded: {action.claim_name} -> {file_path.name}")
|
||||||
|
|
||||||
# Write metadata
|
|
||||||
file_info = {
|
file_info = {
|
||||||
"download_path": str(file_path),
|
"download_path": str(file_path),
|
||||||
"file_name": file_path.name,
|
"file_name": file_path.name,
|
||||||
@@ -86,11 +153,9 @@ class DirectDownloader:
|
|||||||
}
|
}
|
||||||
self._write_metadata(version_dir, action, file_info)
|
self._write_metadata(version_dir, action, file_info)
|
||||||
|
|
||||||
# Write checksum if enabled
|
|
||||||
if self.config.general.write_checksums:
|
if self.config.general.write_checksums:
|
||||||
self._write_checksum(version_dir, file_path)
|
self._write_checksum(version_dir, file_path)
|
||||||
|
|
||||||
# Update action metadata
|
|
||||||
try:
|
try:
|
||||||
rel_path = file_path.relative_to(self.base_dir)
|
rel_path = file_path.relative_to(self.base_dir)
|
||||||
action.metadata["local_file_path"] = str(rel_path)
|
action.metadata["local_file_path"] = str(rel_path)
|
||||||
@@ -99,204 +164,149 @@ class DirectDownloader:
|
|||||||
|
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
except DirectDownloadError:
|
||||||
|
raise
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Direct download failed for {action.claim_name}: {e}")
|
logger.error(f"Direct download failed for {action.claim_name}: {e}")
|
||||||
raise DirectDownloadError(f"Download failed: {e}")
|
raise DirectDownloadError(f"Download failed: {e}")
|
||||||
|
|
||||||
def _get_download_url(self, lbry_uri: str) -> Optional[str]:
|
|
||||||
"""
|
|
||||||
Get direct download URL from Odysee API.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
lbry_uri: LBRY URI like lbry://@channel/claim#claim_id
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Direct download URL or None
|
|
||||||
"""
|
|
||||||
try:
|
|
||||||
# First, resolve the claim to get the permanent URL
|
|
||||||
# Odysee API endpoint for streaming
|
|
||||||
api_url = "https://api.lbry.tv/api/v1/proxy"
|
|
||||||
|
|
||||||
# Get the claim info
|
|
||||||
claim_name = lbry_uri.split("/")[-1].split("#")[0]
|
|
||||||
|
|
||||||
# Try to get streaming URL from Odysee
|
|
||||||
# This uses their API to get CDN links
|
|
||||||
headers = {
|
|
||||||
"Content-Type": "application/json",
|
|
||||||
}
|
|
||||||
|
|
||||||
# Method 1: Try to get via Odysee's API
|
|
||||||
payload = {
|
|
||||||
"jsonrpc": "2.0",
|
|
||||||
"method": "get",
|
|
||||||
"params": {
|
|
||||||
"uri": lbry_uri,
|
|
||||||
"save_file": False,
|
|
||||||
},
|
|
||||||
"id": 1,
|
|
||||||
}
|
|
||||||
|
|
||||||
logger.debug(f"Querying Odysee API for: {lbry_uri}")
|
|
||||||
response = self.session.post(
|
|
||||||
api_url, json=payload, headers=headers, timeout=30
|
|
||||||
)
|
|
||||||
|
|
||||||
if response.status_code == 200:
|
|
||||||
data = response.json()
|
|
||||||
if "result" in data and data["result"]:
|
|
||||||
result = data["result"]
|
|
||||||
|
|
||||||
# Try to get streaming_url
|
|
||||||
if "streaming_url" in result and result["streaming_url"]:
|
|
||||||
return result["streaming_url"]
|
|
||||||
|
|
||||||
# Try to get download URL from value
|
|
||||||
if "download_path" in result:
|
|
||||||
return result["download_path"]
|
|
||||||
|
|
||||||
# Try to construct from lbry.tv
|
|
||||||
if "permanent_url" in result:
|
|
||||||
claim_id = result.get("claim_id", "")
|
|
||||||
if claim_id:
|
|
||||||
# Construct Odysee direct URL
|
|
||||||
return f"https://player.odycdn.com/v6/streams/{claim_id}/"
|
|
||||||
|
|
||||||
# Method 2: Try using lbry.tv player URL directly
|
|
||||||
# Extract claim info from URI
|
|
||||||
claim_parts = lbry_uri.replace("lbry://", "").split("/")
|
|
||||||
if len(claim_parts) >= 2:
|
|
||||||
claim_name_encoded = claim_parts[-1].split("#")[0]
|
|
||||||
# Try to construct direct download URL
|
|
||||||
# This is an educated guess based on Odysee's URL structure
|
|
||||||
odysee_url = f"https://odysee.com/{lbry_uri.replace('lbry://', '').replace('#', ':')}"
|
|
||||||
logger.debug(f"Trying Odysee URL: {odysee_url}")
|
|
||||||
|
|
||||||
# Get the page to find the download link
|
|
||||||
page_response = self.session.get(odysee_url, timeout=30)
|
|
||||||
if page_response.status_code == 200:
|
|
||||||
# Look for download URL in page
|
|
||||||
# Odysee embeds this in their JavaScript
|
|
||||||
match = re.search(r'"download_url":"([^"]+)"', page_response.text)
|
|
||||||
if match:
|
|
||||||
url = match.group(1).replace("\\u0026", "&")
|
|
||||||
return url
|
|
||||||
|
|
||||||
# Look for streaming URL
|
|
||||||
match = re.search(r'"streaming_url":"([^"]+)"', page_response.text)
|
|
||||||
if match:
|
|
||||||
url = match.group(1).replace("\\u0026", "&")
|
|
||||||
return url
|
|
||||||
|
|
||||||
logger.warning(f"Could not extract download URL for: {lbry_uri}")
|
|
||||||
return None
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"Error getting download URL: {e}")
|
|
||||||
return None
|
|
||||||
|
|
||||||
def _download_file(
|
def _download_file(
|
||||||
self, url: str, version_dir: Path, claim_name: str
|
self, url: str, file_path: Path, claim_name: str
|
||||||
) -> Optional[Path]:
|
) -> tuple[bool, str | None]:
|
||||||
"""
|
"""
|
||||||
Download file from URL to version directory with progress bar.
|
Download file from URL with progress display.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
url: Download URL
|
url: Download URL.
|
||||||
version_dir: Target directory
|
file_path: Target file path.
|
||||||
claim_name: Name of the claim
|
claim_name: Name of the claim for display.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Path to downloaded file or None
|
Tuple of success flag and optional failure reason.
|
||||||
"""
|
"""
|
||||||
try:
|
attempts = self.config.general.direct_max_retries_per_url + 1
|
||||||
# Determine filename from URL or claim name
|
for attempt in range(1, attempts + 1):
|
||||||
filename = self._extract_filename(url, claim_name)
|
try:
|
||||||
file_path = version_dir / filename
|
print(f"\n Downloading: {claim_name}")
|
||||||
|
print(f" Filename: {file_path.name}")
|
||||||
|
print(f" Saving to: {file_path}")
|
||||||
|
print(f" Source: {url}")
|
||||||
|
|
||||||
# Print clear message with filename
|
response = self.session.get(
|
||||||
print(f"\n📥 Downloading: {claim_name}")
|
url,
|
||||||
print(f" Filename: {filename}")
|
stream=True,
|
||||||
print(f" Destination: {file_path.parent}")
|
timeout=300,
|
||||||
|
headers={"Referer": "https://odysee.com/"},
|
||||||
# Stream download with progress
|
|
||||||
response = self.session.get(url, stream=True, timeout=300)
|
|
||||||
response.raise_for_status()
|
|
||||||
|
|
||||||
total_size = int(response.headers.get("content-length", 0))
|
|
||||||
|
|
||||||
if total_size > 0:
|
|
||||||
size_mb = total_size / (1024 * 1024)
|
|
||||||
print(f" Size: {size_mb:.2f} MB")
|
|
||||||
else:
|
|
||||||
print(f" Size: Unknown")
|
|
||||||
print()
|
|
||||||
|
|
||||||
start_time = time.time()
|
|
||||||
last_update = start_time
|
|
||||||
downloaded = 0
|
|
||||||
chunk_size = 8192
|
|
||||||
|
|
||||||
with open(file_path, "wb") as f:
|
|
||||||
for chunk in response.iter_content(chunk_size=chunk_size):
|
|
||||||
if chunk:
|
|
||||||
f.write(chunk)
|
|
||||||
downloaded += len(chunk)
|
|
||||||
|
|
||||||
# Update progress bar every 0.5 seconds
|
|
||||||
current_time = time.time()
|
|
||||||
if current_time - last_update >= 0.5:
|
|
||||||
self._print_progress_bar(
|
|
||||||
downloaded, total_size, start_time, claim_name
|
|
||||||
)
|
|
||||||
last_update = current_time
|
|
||||||
|
|
||||||
# Final progress update
|
|
||||||
self._print_progress_bar(
|
|
||||||
downloaded, total_size, start_time, claim_name, final=True
|
|
||||||
)
|
)
|
||||||
print() # New line after progress bar
|
response.raise_for_status()
|
||||||
|
|
||||||
# Verify file was downloaded
|
content_type = response.headers.get("content-type", "")
|
||||||
if file_path.exists() and file_path.stat().st_size > 0:
|
if "text/html" in content_type:
|
||||||
size_mb = file_path.stat().st_size / (1024 * 1024)
|
logger.warning("Got HTML response instead of file content")
|
||||||
elapsed = time.time() - start_time
|
return False, "Got HTML response instead of file content"
|
||||||
speed_mbps = (size_mb / elapsed) if elapsed > 0 else 0
|
|
||||||
print(f"✅ Complete: {size_mb:.2f} MB @ {speed_mbps:.2f} MB/s\n")
|
|
||||||
return file_path
|
|
||||||
else:
|
|
||||||
print(f"❌ Error: Downloaded file is empty\n")
|
|
||||||
return None
|
|
||||||
|
|
||||||
except Exception as e:
|
total_size = int(response.headers.get("content-length", 0))
|
||||||
print(f"❌ Error downloading {claim_name}: {e}\n")
|
if total_size > 0:
|
||||||
logger.error(f"Error downloading file: {e}")
|
size_mb = total_size / (1024 * 1024)
|
||||||
return None
|
print(f" Size: {size_mb:.2f} MB")
|
||||||
|
else:
|
||||||
|
print(" Size: Unknown")
|
||||||
|
print()
|
||||||
|
|
||||||
|
start_time = time.time()
|
||||||
|
last_update = start_time
|
||||||
|
downloaded = 0
|
||||||
|
|
||||||
|
with open(file_path, "wb") as f:
|
||||||
|
for chunk in response.iter_content(chunk_size=8192):
|
||||||
|
if chunk:
|
||||||
|
f.write(chunk)
|
||||||
|
downloaded += len(chunk)
|
||||||
|
|
||||||
|
current_time = time.time()
|
||||||
|
if current_time - last_update >= 0.5:
|
||||||
|
self._print_progress_bar(
|
||||||
|
downloaded, total_size, start_time
|
||||||
|
)
|
||||||
|
last_update = current_time
|
||||||
|
|
||||||
|
self._print_progress_bar(
|
||||||
|
downloaded, total_size, start_time, final=True
|
||||||
|
)
|
||||||
|
print()
|
||||||
|
|
||||||
|
if file_path.exists() and file_path.stat().st_size > 0:
|
||||||
|
size_mb = file_path.stat().st_size / (1024 * 1024)
|
||||||
|
elapsed = time.time() - start_time
|
||||||
|
speed_mbps = (size_mb / elapsed) if elapsed > 0 else 0
|
||||||
|
print(f" Complete: {size_mb:.2f} MB @ {speed_mbps:.2f} MB/s\n")
|
||||||
|
return True, None
|
||||||
|
|
||||||
|
if file_path.exists():
|
||||||
|
file_path.unlink()
|
||||||
|
print(" Error: Downloaded file is empty\n")
|
||||||
|
return False, "Downloaded file is empty"
|
||||||
|
|
||||||
|
except requests.exceptions.HTTPError as e:
|
||||||
|
status_code = getattr(e.response, "status_code", None)
|
||||||
|
if status_code == 401:
|
||||||
|
logger.error(f"HTTP 401 downloading file: {e}")
|
||||||
|
reason = "Odysee CDN rejected this direct URL as unauthorized"
|
||||||
|
elif status_code == 429:
|
||||||
|
logger.error(f"HTTP 429 downloading file: {e}")
|
||||||
|
if file_path.exists():
|
||||||
|
file_path.unlink()
|
||||||
|
if attempt < attempts:
|
||||||
|
backoff = (
|
||||||
|
self.config.general.direct_retry_backoff_seconds
|
||||||
|
* (2 ** (attempt - 1))
|
||||||
|
)
|
||||||
|
logger.warning(
|
||||||
|
"HTTP 429 downloading %s. Retrying in %.1fs (%s/%s)",
|
||||||
|
claim_name,
|
||||||
|
backoff,
|
||||||
|
attempt,
|
||||||
|
attempts - 1,
|
||||||
|
)
|
||||||
|
time.sleep(backoff)
|
||||||
|
continue
|
||||||
|
reason = (
|
||||||
|
"Odysee CDN rate-limited the direct download request after retries"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
logger.error(f"HTTP error downloading file: {e}")
|
||||||
|
reason = f"HTTP error: {e}"
|
||||||
|
if file_path.exists():
|
||||||
|
file_path.unlink()
|
||||||
|
return False, reason
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error downloading file: {e}")
|
||||||
|
if file_path.exists():
|
||||||
|
file_path.unlink()
|
||||||
|
return False, str(e)
|
||||||
|
return False, "Direct download failed"
|
||||||
|
|
||||||
def _print_progress_bar(
|
def _print_progress_bar(
|
||||||
self,
|
self,
|
||||||
downloaded: int,
|
downloaded: int,
|
||||||
total_size: int,
|
total_size: int,
|
||||||
start_time: float,
|
start_time: float,
|
||||||
claim_name: str,
|
|
||||||
final: bool = False,
|
final: bool = False,
|
||||||
):
|
) -> None:
|
||||||
"""Print a progress bar to the console."""
|
"""Print a progress bar to the console."""
|
||||||
if total_size > 0:
|
if total_size > 0:
|
||||||
percent = min(100, int((downloaded / total_size) * 100))
|
percent = min(100, int((downloaded / total_size) * 100))
|
||||||
filled = int(50 * percent / 100)
|
filled = int(50 * percent / 100)
|
||||||
bar = "█" * filled + "░" * (50 - filled)
|
bar = "\u2588" * filled + "\u2591" * (50 - filled)
|
||||||
|
|
||||||
# Calculate speed
|
|
||||||
elapsed = time.time() - start_time
|
elapsed = time.time() - start_time
|
||||||
|
speed_mbps = 0
|
||||||
if elapsed > 0:
|
if elapsed > 0:
|
||||||
speed_mbps = (downloaded / (1024 * 1024)) / elapsed
|
speed_mbps = (downloaded / (1024 * 1024)) / elapsed
|
||||||
speed_str = f"{speed_mbps:.2f} MB/s"
|
speed_str = f"{speed_mbps:.2f} MB/s"
|
||||||
else:
|
else:
|
||||||
speed_str = "calculating..."
|
speed_str = "calculating..."
|
||||||
|
|
||||||
# Calculate ETA
|
|
||||||
if not final and speed_mbps > 0:
|
if not final and speed_mbps > 0:
|
||||||
remaining = (total_size - downloaded) / (1024 * 1024)
|
remaining = (total_size - downloaded) / (1024 * 1024)
|
||||||
eta = remaining / speed_mbps
|
eta = remaining / speed_mbps
|
||||||
@@ -304,27 +314,15 @@ class DirectDownloader:
|
|||||||
else:
|
else:
|
||||||
eta_str = ""
|
eta_str = ""
|
||||||
|
|
||||||
# Clear line and print progress
|
|
||||||
print(
|
print(
|
||||||
f"\r [{bar}] {percent}% | {speed_str} | {eta_str}", end="", flush=True
|
f"\r [{bar}] {percent}% | {speed_str} | {eta_str}",
|
||||||
|
end="",
|
||||||
|
flush=True,
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
# Unknown size, just show downloaded amount
|
|
||||||
downloaded_mb = downloaded / (1024 * 1024)
|
downloaded_mb = downloaded / (1024 * 1024)
|
||||||
print(f"\r Downloaded: {downloaded_mb:.2f} MB", end="", flush=True)
|
print(f"\r Downloaded: {downloaded_mb:.2f} MB", end="", flush=True)
|
||||||
|
|
||||||
def _extract_filename(self, url: str, claim_name: str) -> str:
|
|
||||||
"""Extract filename from URL or use claim name."""
|
|
||||||
# Try to get from URL
|
|
||||||
if "." in url.split("/")[-1].split("?")[0]:
|
|
||||||
filename = url.split("/")[-1].split("?")[0]
|
|
||||||
# Sanitize
|
|
||||||
filename = re.sub(r'[<>"|?*]', "_", filename)
|
|
||||||
return filename[:255] # Max filename length
|
|
||||||
|
|
||||||
# Use claim name with extension
|
|
||||||
return f"{claim_name}.mp4" # Default to mp4
|
|
||||||
|
|
||||||
def _write_metadata(
|
def _write_metadata(
|
||||||
self, version_dir: Path, action: DownloadAction, file_info: Dict
|
self, version_dir: Path, action: DownloadAction, file_info: Dict
|
||||||
) -> None:
|
) -> None:
|
||||||
@@ -336,10 +334,14 @@ class DirectDownloader:
|
|||||||
"title": action.metadata.get("title"),
|
"title": action.metadata.get("title"),
|
||||||
"version_token": action.version_token,
|
"version_token": action.version_token,
|
||||||
"permanent_url": action.uri,
|
"permanent_url": action.uri,
|
||||||
"download_source": "odysee_cdn",
|
"download_source": (
|
||||||
|
"odysee_stream" if self._is_streaming_media(action) else "odysee_download"
|
||||||
|
),
|
||||||
"downloaded_at": time.strftime("%Y-%m-%dT%H:%M:%S"),
|
"downloaded_at": time.strftime("%Y-%m-%dT%H:%M:%S"),
|
||||||
"file_name": file_info.get("file_name"),
|
"file_name": file_info.get("file_name"),
|
||||||
"file_size": file_info.get("size"),
|
"file_size": file_info.get("size"),
|
||||||
|
"stream_type": action.metadata.get("stream_type"),
|
||||||
|
"source_media_type": action.metadata.get("source_media_type"),
|
||||||
}
|
}
|
||||||
|
|
||||||
metadata_path = version_dir / "metadata.json"
|
metadata_path = version_dir / "metadata.json"
|
||||||
|
|||||||
+12
-1
@@ -49,7 +49,15 @@ class Downloader:
|
|||||||
DownloadError: If the download fails.
|
DownloadError: If the download fails.
|
||||||
"""
|
"""
|
||||||
if action.action == "skip_existing":
|
if action.action == "skip_existing":
|
||||||
logger.debug(f"Skipping already downloaded: {action.claim_name}")
|
existing_path = action.metadata.get("existing_file_relpath")
|
||||||
|
if existing_path:
|
||||||
|
logger.info(
|
||||||
|
"Skipping existing file: %s (%s)",
|
||||||
|
action.claim_name,
|
||||||
|
self.base_dir / existing_path,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
logger.info(f"Skipping existing file: {action.claim_name}")
|
||||||
return True
|
return True
|
||||||
|
|
||||||
if dry_run:
|
if dry_run:
|
||||||
@@ -61,6 +69,7 @@ class Downloader:
|
|||||||
version_dir.mkdir(parents=True, exist_ok=True)
|
version_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
logger.info(f"Downloading: {action.claim_name}")
|
logger.info(f"Downloading: {action.claim_name}")
|
||||||
|
logger.info(f"Saving to directory: {version_dir}")
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# Call daemon to start P2P download
|
# Call daemon to start P2P download
|
||||||
@@ -281,6 +290,7 @@ class Downloader:
|
|||||||
streaming_url = f"http://localhost:5280/stream/{sd_hash}"
|
streaming_url = f"http://localhost:5280/stream/{sd_hash}"
|
||||||
|
|
||||||
logger.info(f"Downloading via daemon streaming endpoint...")
|
logger.info(f"Downloading via daemon streaming endpoint...")
|
||||||
|
logger.info(f"Streaming destination: {target_path}")
|
||||||
|
|
||||||
try:
|
try:
|
||||||
response = requests.get(streaming_url, stream=True, timeout=300)
|
response = requests.get(streaming_url, stream=True, timeout=300)
|
||||||
@@ -291,6 +301,7 @@ class Downloader:
|
|||||||
chunk_size = 8192
|
chunk_size = 8192
|
||||||
|
|
||||||
print(f"\n📥 Downloading: {action.claim_name}")
|
print(f"\n📥 Downloading: {action.claim_name}")
|
||||||
|
print(f" Saving to: {target_path}")
|
||||||
print(f" Size: {total_size / (1024 * 1024):.2f} MB")
|
print(f" Size: {total_size / (1024 * 1024):.2f} MB")
|
||||||
print()
|
print()
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
"""
|
"""
|
||||||
Initialize LBRY Downloader configuration.
|
Initialize Odysee/LBRY Downloader configuration.
|
||||||
|
|
||||||
This script creates the initial configuration file and directory structure.
|
This script creates the initial configuration file and directory structure.
|
||||||
"""
|
"""
|
||||||
@@ -14,7 +14,7 @@ from utils import expand_path
|
|||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
print("LBRY Downloader - Initialization")
|
print("Odysee/LBRY Downloader - Initialization")
|
||||||
print("=" * 40)
|
print("=" * 40)
|
||||||
|
|
||||||
# Default paths
|
# Default paths
|
||||||
@@ -51,12 +51,13 @@ def main():
|
|||||||
print("Next steps:")
|
print("Next steps:")
|
||||||
print("1. Edit the configuration file to add your channels")
|
print("1. Edit the configuration file to add your channels")
|
||||||
print(f" nano {config_file}")
|
print(f" nano {config_file}")
|
||||||
print("\n2. Ensure lbrynet daemon is running:")
|
print("\n2. Test first with dry-run:")
|
||||||
print(" lbrynet start")
|
|
||||||
print("\n3. Run the downloader:")
|
|
||||||
print(" python main.py")
|
|
||||||
print("\n4. Or test first with dry-run:")
|
|
||||||
print(" python main.py --dry-run")
|
print(" python main.py --dry-run")
|
||||||
|
print("\n3. Recommended first real run:")
|
||||||
|
print(" python main.py --non-video-only")
|
||||||
|
print("\n4. Optional P2P mode if you want the local node path:")
|
||||||
|
print(" lbrynet start")
|
||||||
|
print(" python main.py --p2p")
|
||||||
print("=" * 40)
|
print("=" * 40)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+11
-52
@@ -12,7 +12,7 @@
|
|||||||
set -e
|
set -e
|
||||||
|
|
||||||
echo "================================================"
|
echo "================================================"
|
||||||
echo " LBRY Downloader - Installation"
|
echo " Odysee/LBRY Downloader - Installation"
|
||||||
echo "================================================"
|
echo "================================================"
|
||||||
echo ""
|
echo ""
|
||||||
|
|
||||||
@@ -117,51 +117,14 @@ pip install --upgrade pip
|
|||||||
pip install -r requirements.txt
|
pip install -r requirements.txt
|
||||||
print_status "Python dependencies installed"
|
print_status "Python dependencies installed"
|
||||||
|
|
||||||
# Check for lbrynet
|
|
||||||
echo ""
|
echo ""
|
||||||
echo "Checking for LBRY SDK (lbrynet)..."
|
echo "Checking for optional LBRY SDK (lbrynet)..."
|
||||||
if command -v lbrynet &> /dev/null; then
|
if command -v lbrynet &> /dev/null; then
|
||||||
print_status "lbrynet found at: $(which lbrynet)"
|
print_status "Optional P2P dependency found at: $(which lbrynet)"
|
||||||
lbrynet --version
|
lbrynet --version
|
||||||
else
|
else
|
||||||
print_warning "lbrynet not found in PATH"
|
print_info "lbrynet not found. Default direct mode does not require it."
|
||||||
echo ""
|
print_info "Install it later only if you want to run with --p2p."
|
||||||
echo "The LBRY SDK (lbrynet) is required to download content."
|
|
||||||
echo ""
|
|
||||||
read -p "Would you like to download and install lbrynet now? (y/N): " -n 1 -r
|
|
||||||
echo
|
|
||||||
|
|
||||||
if [[ $REPLY =~ ^[Yy]$ ]]; then
|
|
||||||
echo "Downloading lbrynet..."
|
|
||||||
mkdir -p ~/.local/bin
|
|
||||||
|
|
||||||
# Get latest release
|
|
||||||
LBRY_VERSION="0.113.0"
|
|
||||||
DOWNLOAD_URL="https://github.com/lbryio/lbry-sdk/releases/download/v${LBRY_VERSION}/lbry-sdk-linux.zip"
|
|
||||||
|
|
||||||
cd ~/.local/bin
|
|
||||||
if wget -q "$DOWNLOAD_URL"; then
|
|
||||||
unzip -q lbry-sdk-linux.zip
|
|
||||||
rm lbry-sdk-linux.zip
|
|
||||||
chmod +x lbrynet
|
|
||||||
print_status "lbrynet installed to ~/.local/bin"
|
|
||||||
|
|
||||||
# Check if ~/.local/bin is in PATH
|
|
||||||
if [[ ":$PATH:" != *":$HOME/.local/bin:"* ]]; then
|
|
||||||
echo 'export PATH="$HOME/.local/bin:$PATH"' >> ~/.bashrc
|
|
||||||
print_status "Added ~/.local/bin to PATH in ~/.bashrc"
|
|
||||||
print_warning "Please run: source ~/.bashrc"
|
|
||||||
fi
|
|
||||||
|
|
||||||
cd "$SCRIPT_DIR"
|
|
||||||
else
|
|
||||||
print_error "Failed to download lbrynet"
|
|
||||||
echo "Please install manually from: https://github.com/lbryio/lbry-sdk/releases"
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
print_warning "Skipping lbrynet installation"
|
|
||||||
echo "You'll need to install it manually before using the downloader"
|
|
||||||
fi
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Initialize configuration
|
# Initialize configuration
|
||||||
@@ -201,13 +164,6 @@ fi
|
|||||||
cd "$DOWNLOADER_DIR"
|
cd "$DOWNLOADER_DIR"
|
||||||
source venv/bin/activate 2>/dev/null || true
|
source venv/bin/activate 2>/dev/null || true
|
||||||
|
|
||||||
# Check if lbrynet is running
|
|
||||||
if ! curl -s http://127.0.0.1:5279 > /dev/null 2>&1; then
|
|
||||||
echo "Starting lbrynet daemon..."
|
|
||||||
lbrynet start &
|
|
||||||
sleep 5
|
|
||||||
fi
|
|
||||||
|
|
||||||
python main.py "$@"
|
python main.py "$@"
|
||||||
EOF
|
EOF
|
||||||
|
|
||||||
@@ -233,7 +189,7 @@ echo "1. Edit configuration to add your channels:"
|
|||||||
echo " nano ~/Documents/lbry-downloads/config.yaml"
|
echo " nano ~/Documents/lbry-downloads/config.yaml"
|
||||||
echo ""
|
echo ""
|
||||||
echo "2. Start the LBRY daemon:"
|
echo "2. Start the LBRY daemon:"
|
||||||
echo " lbrynet start"
|
echo " Optional only for P2P mode: lbrynet start"
|
||||||
echo ""
|
echo ""
|
||||||
echo "3. Test with dry-run:"
|
echo "3. Test with dry-run:"
|
||||||
echo " python main.py --dry-run"
|
echo " python main.py --dry-run"
|
||||||
@@ -241,9 +197,12 @@ echo " # OR:"
|
|||||||
echo " ~/bin/lbry-sync --dry-run"
|
echo " ~/bin/lbry-sync --dry-run"
|
||||||
echo ""
|
echo ""
|
||||||
echo "4. Run the downloader:"
|
echo "4. Run the downloader:"
|
||||||
echo " python main.py"
|
echo " python main.py --non-video-only"
|
||||||
echo " # OR:"
|
echo " # OR:"
|
||||||
echo " ~/bin/lbry-sync"
|
echo " ~/bin/lbry-sync --non-video-only"
|
||||||
|
echo ""
|
||||||
|
echo "5. Optional local-node mode:"
|
||||||
|
echo " python main.py --p2p"
|
||||||
echo ""
|
echo ""
|
||||||
echo "For detailed instructions, see:"
|
echo "For detailed instructions, see:"
|
||||||
echo " - LINUX_SETUP.md"
|
echo " - LINUX_SETUP.md"
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ from typing import List, Optional
|
|||||||
|
|
||||||
from config_loader import ConfigError, ensure_directories, load_config
|
from config_loader import ConfigError, ensure_directories, load_config
|
||||||
from downloader import DownloadError, Downloader
|
from downloader import DownloadError, Downloader
|
||||||
|
from direct_downloader import DirectDownloadError
|
||||||
from lbry_client import LbryClient, LbryClientError, check_daemon_health
|
from lbry_client import LbryClient, LbryClientError, check_daemon_health
|
||||||
from models import Channel, Config, DownloadAction
|
from models import Channel, Config, DownloadAction
|
||||||
from planner import Planner
|
from planner import Planner
|
||||||
@@ -26,6 +27,20 @@ from utils import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
VIDEO_EXTENSIONS = {
|
||||||
|
".mp4",
|
||||||
|
".webm",
|
||||||
|
".mkv",
|
||||||
|
".mov",
|
||||||
|
".avi",
|
||||||
|
".m4v",
|
||||||
|
".mpeg",
|
||||||
|
".mpg",
|
||||||
|
".wmv",
|
||||||
|
".flv",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def setup_logging(log_level: str) -> None:
|
def setup_logging(log_level: str) -> None:
|
||||||
"""Configure logging."""
|
"""Configure logging."""
|
||||||
logging.basicConfig(
|
logging.basicConfig(
|
||||||
@@ -83,6 +98,24 @@ def resolve_channel(
|
|||||||
channel_claim_id = claim.get("claim_id")
|
channel_claim_id = claim.get("claim_id")
|
||||||
channel_name = claim.get("name")
|
channel_name = claim.get("name")
|
||||||
|
|
||||||
|
# Odysee's public proxy can resolve vanity channel URLs to older
|
||||||
|
# non-controlling channel claims. Enumerating streams from those claim IDs
|
||||||
|
# returns zero items, so normalize to the controlling claim before scanning.
|
||||||
|
if not client.supports_file_ops and channel_name:
|
||||||
|
controlling_claim = client.find_controlling_channel(channel_name)
|
||||||
|
if controlling_claim and controlling_claim.get("claim_id"):
|
||||||
|
controlling_claim_id = controlling_claim.get("claim_id")
|
||||||
|
if controlling_claim_id != channel_claim_id:
|
||||||
|
logger.debug(
|
||||||
|
"Normalized proxy-resolved channel %s from %s to controlling claim %s",
|
||||||
|
channel_name,
|
||||||
|
channel_claim_id,
|
||||||
|
controlling_claim_id,
|
||||||
|
)
|
||||||
|
claim = controlling_claim
|
||||||
|
channel_claim_id = controlling_claim_id
|
||||||
|
channel_name = claim.get("name")
|
||||||
|
|
||||||
if not channel_claim_id:
|
if not channel_claim_id:
|
||||||
logger.error(f"No claim_id in resolved channel: {channel_input}")
|
logger.error(f"No claim_id in resolved channel: {channel_input}")
|
||||||
return None
|
return None
|
||||||
@@ -164,14 +197,96 @@ def enumerate_channel_claims(
|
|||||||
return all_claims
|
return all_claims
|
||||||
|
|
||||||
|
|
||||||
def run_sync(config: Config, dry_run: bool = False, direct: bool = False) -> dict:
|
def _action_is_video(action: DownloadAction) -> bool:
|
||||||
|
"""Return True when a planned action looks like a video download."""
|
||||||
|
media_type = (action.metadata.get("source_media_type") or "").lower()
|
||||||
|
if media_type.startswith("video/"):
|
||||||
|
return True
|
||||||
|
|
||||||
|
stream_type = (action.metadata.get("stream_type") or "").lower()
|
||||||
|
if stream_type == "video":
|
||||||
|
return True
|
||||||
|
|
||||||
|
source_name = action.metadata.get("source_name") or action.metadata.get("file_name")
|
||||||
|
if source_name and Path(source_name).suffix.lower() in VIDEO_EXTENSIONS:
|
||||||
|
return True
|
||||||
|
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def filter_actions_by_content_type(
|
||||||
|
actions: List[DownloadAction],
|
||||||
|
*,
|
||||||
|
video_only: bool = False,
|
||||||
|
non_video_only: bool = False,
|
||||||
|
) -> List[DownloadAction]:
|
||||||
|
"""Filter planned actions by content type before limits and execution."""
|
||||||
|
if not video_only and not non_video_only:
|
||||||
|
return actions
|
||||||
|
|
||||||
|
filtered_actions = []
|
||||||
|
for action in actions:
|
||||||
|
is_video = _action_is_video(action)
|
||||||
|
if video_only and is_video:
|
||||||
|
filtered_actions.append(action)
|
||||||
|
elif non_video_only and not is_video:
|
||||||
|
filtered_actions.append(action)
|
||||||
|
|
||||||
|
return filtered_actions
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_content_filter_mode(
|
||||||
|
channel_config,
|
||||||
|
*,
|
||||||
|
video_only: bool = False,
|
||||||
|
non_video_only: bool = False,
|
||||||
|
) -> tuple[bool, bool, str]:
|
||||||
|
"""Resolve the effective content filter for a channel."""
|
||||||
|
if video_only:
|
||||||
|
return True, False, "video-only (CLI override)"
|
||||||
|
if non_video_only:
|
||||||
|
return False, True, "non-video-only (CLI override)"
|
||||||
|
|
||||||
|
content_mode = getattr(channel_config, "content_mode", "all")
|
||||||
|
if content_mode == "video_only":
|
||||||
|
return True, False, "video-only (channel config)"
|
||||||
|
if content_mode == "non_video_only":
|
||||||
|
return False, True, "non-video-only (channel config)"
|
||||||
|
|
||||||
|
return False, False, "all content"
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_transport_mode(
|
||||||
|
*,
|
||||||
|
direct: bool = False,
|
||||||
|
p2p: bool = False,
|
||||||
|
) -> tuple[bool, str]:
|
||||||
|
"""Resolve the effective transport mode."""
|
||||||
|
if p2p:
|
||||||
|
return False, "p2p"
|
||||||
|
return True, "direct"
|
||||||
|
|
||||||
|
|
||||||
|
def _is_rate_limit_error(exc: Exception) -> bool:
|
||||||
|
"""Return True when the error indicates direct-mode rate limiting."""
|
||||||
|
return "rate-limited" in str(exc).lower()
|
||||||
|
|
||||||
|
|
||||||
|
def run_sync(
|
||||||
|
config: Config,
|
||||||
|
dry_run: bool = False,
|
||||||
|
direct: bool = True,
|
||||||
|
*,
|
||||||
|
video_only: bool = False,
|
||||||
|
non_video_only: bool = False,
|
||||||
|
) -> dict:
|
||||||
"""
|
"""
|
||||||
Run the main synchronization process.
|
Run the main synchronization process.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
config: Application configuration.
|
config: Application configuration.
|
||||||
dry_run: If True, don't actually download.
|
dry_run: If True, don't actually download.
|
||||||
direct: If True, use Odysee CDN instead of P2P.
|
direct: If True, use Odysee proxy/CDN instead of P2P.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Statistics dictionary.
|
Statistics dictionary.
|
||||||
@@ -195,30 +310,28 @@ def run_sync(config: Config, dry_run: bool = False, direct: bool = False) -> dic
|
|||||||
base_path = Path(config.general.base_dir)
|
base_path = Path(config.general.base_dir)
|
||||||
planner = Planner(config, state_db.data, str(base_path))
|
planner = Planner(config, state_db.data, str(base_path))
|
||||||
|
|
||||||
# Initialize LBRY client (needed for channel/claim resolution in both modes)
|
|
||||||
client = LbryClient(
|
|
||||||
api_url=config.lbrynet.api_url,
|
|
||||||
timeout=config.lbrynet.timeout_seconds,
|
|
||||||
)
|
|
||||||
|
|
||||||
# Check daemon health (needed for metadata even in direct mode)
|
|
||||||
logger.info(f"Checking daemon at {config.lbrynet.api_url}")
|
|
||||||
try:
|
|
||||||
check_daemon_health(client)
|
|
||||||
logger.info("Daemon is healthy")
|
|
||||||
except LbryClientError as e:
|
|
||||||
logger.error(str(e))
|
|
||||||
logger.error(
|
|
||||||
"Daemon is required for channel/claim resolution even in --direct mode"
|
|
||||||
)
|
|
||||||
return stats
|
|
||||||
|
|
||||||
if direct:
|
if direct:
|
||||||
logger.info("Using DIRECT mode (Odysee CDN for downloads)")
|
logger.info("Using DIRECT mode (Odysee proxy for metadata, Odysee CDN for downloads)")
|
||||||
|
client = LbryClient.create_odysee_proxy(timeout=config.lbrynet.timeout_seconds)
|
||||||
from direct_downloader import DirectDownloader
|
from direct_downloader import DirectDownloader
|
||||||
|
|
||||||
downloader = DirectDownloader(config, base_path)
|
downloader = DirectDownloader(config, base_path)
|
||||||
|
p2p_fallback_client = None
|
||||||
|
p2p_fallback_downloader = None
|
||||||
|
p2p_fallback_available = None
|
||||||
else:
|
else:
|
||||||
|
client = LbryClient(
|
||||||
|
api_url=config.lbrynet.api_url,
|
||||||
|
timeout=config.lbrynet.timeout_seconds,
|
||||||
|
)
|
||||||
|
logger.info(f"Checking daemon at {config.lbrynet.api_url}")
|
||||||
|
try:
|
||||||
|
check_daemon_health(client)
|
||||||
|
logger.info("Daemon is healthy")
|
||||||
|
except LbryClientError as e:
|
||||||
|
logger.error(str(e))
|
||||||
|
return stats
|
||||||
|
|
||||||
logger.info("Using P2P mode (LBRY network)")
|
logger.info("Using P2P mode (LBRY network)")
|
||||||
downloader = Downloader(client, config, base_path)
|
downloader = Downloader(client, config, base_path)
|
||||||
|
|
||||||
@@ -251,6 +364,26 @@ def run_sync(config: Config, dry_run: bool = False, direct: bool = False) -> dic
|
|||||||
|
|
||||||
# Plan downloads
|
# Plan downloads
|
||||||
actions = planner.process_channel_claims(channel, claims)
|
actions = planner.process_channel_claims(channel, claims)
|
||||||
|
actions_before_filter = len(actions)
|
||||||
|
channel_video_only, channel_non_video_only, mode_label = (
|
||||||
|
resolve_content_filter_mode(
|
||||||
|
channel_config,
|
||||||
|
video_only=video_only,
|
||||||
|
non_video_only=non_video_only,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
actions = filter_actions_by_content_type(
|
||||||
|
actions,
|
||||||
|
video_only=channel_video_only,
|
||||||
|
non_video_only=channel_non_video_only,
|
||||||
|
)
|
||||||
|
|
||||||
|
if len(actions) != actions_before_filter:
|
||||||
|
logger.info(
|
||||||
|
"Content filter (%s) skipped %s planned claim(s).",
|
||||||
|
mode_label,
|
||||||
|
actions_before_filter - len(actions),
|
||||||
|
)
|
||||||
|
|
||||||
# Apply download limit if configured
|
# Apply download limit if configured
|
||||||
# Separate skip actions from download actions
|
# Separate skip actions from download actions
|
||||||
@@ -314,7 +447,61 @@ def run_sync(config: Config, dry_run: bool = False, direct: bool = False) -> dic
|
|||||||
"success" if success else "failure",
|
"success" if success else "failure",
|
||||||
)
|
)
|
||||||
|
|
||||||
except DownloadError as e:
|
except (DownloadError, DirectDownloadError) as e:
|
||||||
|
if (
|
||||||
|
direct
|
||||||
|
and config.general.direct_auto_fallback_to_p2p
|
||||||
|
and isinstance(e, DirectDownloadError)
|
||||||
|
and _is_rate_limit_error(e)
|
||||||
|
and not dry_run
|
||||||
|
):
|
||||||
|
if p2p_fallback_available is None:
|
||||||
|
try:
|
||||||
|
p2p_fallback_client = LbryClient(
|
||||||
|
api_url=config.lbrynet.api_url,
|
||||||
|
timeout=config.lbrynet.timeout_seconds,
|
||||||
|
)
|
||||||
|
check_daemon_health(p2p_fallback_client)
|
||||||
|
p2p_fallback_downloader = Downloader(
|
||||||
|
p2p_fallback_client, config, base_path
|
||||||
|
)
|
||||||
|
p2p_fallback_available = True
|
||||||
|
logger.warning(
|
||||||
|
"Direct mode hit rate limiting. Local P2P fallback is available."
|
||||||
|
)
|
||||||
|
except LbryClientError as fallback_init_error:
|
||||||
|
p2p_fallback_available = False
|
||||||
|
logger.warning(
|
||||||
|
"Direct mode hit rate limiting, but P2P fallback is unavailable: %s",
|
||||||
|
fallback_init_error,
|
||||||
|
)
|
||||||
|
|
||||||
|
if p2p_fallback_available and p2p_fallback_downloader is not None:
|
||||||
|
try:
|
||||||
|
logger.warning(
|
||||||
|
"Retrying %s via P2P fallback after direct-mode rate limiting.",
|
||||||
|
action.claim_name,
|
||||||
|
)
|
||||||
|
success = p2p_fallback_downloader.download(
|
||||||
|
action, dry_run=False
|
||||||
|
)
|
||||||
|
if success:
|
||||||
|
planner.update_state_from_action(action, success=True)
|
||||||
|
history_file = base_path / "state" / "run-history.jsonl"
|
||||||
|
state_db.log_run_history(
|
||||||
|
str(history_file),
|
||||||
|
channel.channel_claim_id,
|
||||||
|
action.claim_id,
|
||||||
|
action.action,
|
||||||
|
action.version_token,
|
||||||
|
"success",
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
except DownloadError as fallback_error:
|
||||||
|
e = DirectDownloadError(
|
||||||
|
f"{e}; P2P fallback also failed: {fallback_error}"
|
||||||
|
)
|
||||||
|
|
||||||
logger.error(f"Download failed: {e}")
|
logger.error(f"Download failed: {e}")
|
||||||
stats["failures"] += 1
|
stats["failures"] += 1
|
||||||
|
|
||||||
@@ -352,10 +539,10 @@ def main():
|
|||||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||||
epilog="""
|
epilog="""
|
||||||
Examples:
|
Examples:
|
||||||
python main.py # Run with default config (P2P mode)
|
python main.py # Run with default config (DIRECT mode)
|
||||||
python main.py --config ./config.yaml # Use custom config
|
python main.py --config ./config.yaml # Use custom config
|
||||||
python main.py --dry-run # Plan actions but don't download
|
python main.py --dry-run # Plan actions but don't download
|
||||||
python main.py --direct # Download from Odysee CDN (more reliable)
|
python main.py --p2p # Use the local LBRY node instead
|
||||||
""",
|
""",
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -372,16 +559,39 @@ Examples:
|
|||||||
help="Show what would be downloaded without actually downloading",
|
help="Show what would be downloaded without actually downloading",
|
||||||
)
|
)
|
||||||
|
|
||||||
parser.add_argument(
|
transport_group = parser.add_mutually_exclusive_group()
|
||||||
|
transport_group.add_argument(
|
||||||
"--direct",
|
"--direct",
|
||||||
"-d",
|
"-d",
|
||||||
action="store_true",
|
action="store_true",
|
||||||
help="Download directly from Odysee CDN instead of using P2P network",
|
help="Use direct mode explicitly (default: Odysee public proxy + CDN)",
|
||||||
|
)
|
||||||
|
transport_group.add_argument(
|
||||||
|
"--p2p",
|
||||||
|
action="store_true",
|
||||||
|
help="Use the local LBRY daemon and peer-to-peer network instead of direct mode",
|
||||||
|
)
|
||||||
|
|
||||||
|
content_filter_group = parser.add_mutually_exclusive_group()
|
||||||
|
content_filter_group.add_argument(
|
||||||
|
"--video-only",
|
||||||
|
action="store_true",
|
||||||
|
help="Only include video downloads such as mp4/webm/mkv",
|
||||||
|
)
|
||||||
|
content_filter_group.add_argument(
|
||||||
|
"--non-video-only",
|
||||||
|
action="store_true",
|
||||||
|
help="Only include non-video downloads such as zip files",
|
||||||
)
|
)
|
||||||
|
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
direct_mode, transport_label = resolve_transport_mode(
|
||||||
|
direct=args.direct,
|
||||||
|
p2p=args.p2p,
|
||||||
|
)
|
||||||
|
|
||||||
# Load config
|
# Load config
|
||||||
config = load_config(args.config)
|
config = load_config(args.config)
|
||||||
|
|
||||||
@@ -390,12 +600,19 @@ Examples:
|
|||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
logger.info("LBRY Downloader starting")
|
logger.info("LBRY Downloader starting")
|
||||||
|
logger.info("Effective transport mode: %s", transport_label)
|
||||||
|
|
||||||
# Ensure directories exist
|
# Ensure directories exist
|
||||||
ensure_directories(config)
|
ensure_directories(config)
|
||||||
|
|
||||||
# Run sync
|
# Run sync
|
||||||
stats = run_sync(config, dry_run=args.dry_run, direct=args.direct)
|
stats = run_sync(
|
||||||
|
config,
|
||||||
|
dry_run=args.dry_run,
|
||||||
|
direct=direct_mode,
|
||||||
|
video_only=args.video_only,
|
||||||
|
non_video_only=args.non_video_only,
|
||||||
|
)
|
||||||
|
|
||||||
# Print summary
|
# Print summary
|
||||||
summary = format_summary(
|
summary = format_summary(
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ class ChannelConfig:
|
|||||||
input: str
|
input: str
|
||||||
enabled: bool = True
|
enabled: bool = True
|
||||||
download_path: Optional[str] = None
|
download_path: Optional[str] = None
|
||||||
|
content_mode: str = "all"
|
||||||
tags_include: List[str] = field(default_factory=list)
|
tags_include: List[str] = field(default_factory=list)
|
||||||
tags_exclude: List[str] = field(default_factory=list)
|
tags_exclude: List[str] = field(default_factory=list)
|
||||||
|
|
||||||
@@ -34,6 +35,12 @@ class GeneralConfig:
|
|||||||
download_limit: int = (
|
download_limit: int = (
|
||||||
10 # Number of most recent downloads per channel, or "all" (0)
|
10 # Number of most recent downloads per channel, or "all" (0)
|
||||||
)
|
)
|
||||||
|
direct_base_urls: List[str] = field(
|
||||||
|
default_factory=lambda: ["https://odysee.com"]
|
||||||
|
)
|
||||||
|
direct_max_retries_per_url: int = 2
|
||||||
|
direct_retry_backoff_seconds: float = 2.0
|
||||||
|
direct_auto_fallback_to_p2p: bool = False
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
|
|||||||
@@ -183,6 +183,8 @@ class Planner:
|
|||||||
target_dir=str(version_dir),
|
target_dir=str(version_dir),
|
||||||
metadata=metadata,
|
metadata=metadata,
|
||||||
)
|
)
|
||||||
|
if version.file_relpath:
|
||||||
|
action.metadata["existing_file_relpath"] = version.file_relpath
|
||||||
logger.debug(f"Skipping existing: {claim_name}")
|
logger.debug(f"Skipping existing: {claim_name}")
|
||||||
return action
|
return action
|
||||||
else:
|
else:
|
||||||
|
|||||||
+9
-70
@@ -20,7 +20,7 @@ REPO_URL="https://github.com/PR0M3TH3AN/LBRY-Downloader.git"
|
|||||||
INSTALL_DIR="${HOME}/Documents/LBRY-Downloader"
|
INSTALL_DIR="${HOME}/Documents/LBRY-Downloader"
|
||||||
|
|
||||||
echo "=========================================="
|
echo "=========================================="
|
||||||
echo " LBRY Downloader - Remote Installer"
|
echo " Odysee/LBRY Downloader - Remote Installer"
|
||||||
echo "=========================================="
|
echo "=========================================="
|
||||||
echo ""
|
echo ""
|
||||||
|
|
||||||
@@ -74,70 +74,11 @@ fi
|
|||||||
|
|
||||||
print_status "Python $PYTHON_VERSION found"
|
print_status "Python $PYTHON_VERSION found"
|
||||||
|
|
||||||
# Check if lbrynet is installed
|
if command -v lbrynet &> /dev/null; then
|
||||||
if ! command -v lbrynet &> /dev/null; then
|
print_status "Optional P2P dependency found: $(which lbrynet)"
|
||||||
print_warning "LBRY SDK (lbrynet) not found"
|
|
||||||
echo ""
|
|
||||||
echo "The LBRY SDK is required to download content from the LBRY network."
|
|
||||||
echo "It can be downloaded from: https://github.com/lbryio/lbry-sdk/releases"
|
|
||||||
echo ""
|
|
||||||
read -p "Download and install lbrynet automatically? (Y/n): " -n 1 -r
|
|
||||||
echo
|
|
||||||
if [[ ! $REPLY =~ ^[Nn]$ ]]; then
|
|
||||||
print_status "Installing LBRY SDK..."
|
|
||||||
|
|
||||||
# Create bin directory
|
|
||||||
mkdir -p ~/.local/bin
|
|
||||||
|
|
||||||
# Download latest lbrynet for Linux
|
|
||||||
LBRY_VERSION="0.113.0"
|
|
||||||
DOWNLOAD_URL="https://github.com/lbryio/lbry-sdk/releases/download/v${LBRY_VERSION}/lbrynet-linux.zip"
|
|
||||||
|
|
||||||
cd ~/.local/bin
|
|
||||||
|
|
||||||
# Download with progress
|
|
||||||
echo "Downloading lbrynet v${LBRY_VERSION}..."
|
|
||||||
if wget --progress=bar:force "$DOWNLOAD_URL" -O lbrynet-linux.zip 2>&1; then
|
|
||||||
print_status "Downloaded lbrynet"
|
|
||||||
|
|
||||||
# Check if file exists and has content
|
|
||||||
if [ -f "lbrynet-linux.zip" ] && [ -s "lbrynet-linux.zip" ]; then
|
|
||||||
# Extract
|
|
||||||
if unzip -q lbrynet-linux.zip; then
|
|
||||||
rm lbrynet-linux.zip
|
|
||||||
chmod +x lbrynet
|
|
||||||
print_status "Extracted lbrynet"
|
|
||||||
|
|
||||||
# Add to PATH if not already there
|
|
||||||
if ! echo "$PATH" | grep -q "$HOME/.local/bin"; then
|
|
||||||
echo 'export PATH="$HOME/.local/bin:$PATH"' >> ~/.bashrc
|
|
||||||
print_status "Added ~/.local/bin to PATH"
|
|
||||||
export PATH="$HOME/.local/bin:$PATH"
|
|
||||||
fi
|
|
||||||
|
|
||||||
print_status "LBRY SDK installed successfully!"
|
|
||||||
print_info "lbrynet installed to: ~/.local/bin/lbrynet"
|
|
||||||
else
|
|
||||||
print_error "Failed to extract lbrynet"
|
|
||||||
rm -f lbrynet-linux.zip
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
print_error "Download failed - file not found or empty"
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
print_error "Failed to download lbrynet"
|
|
||||||
echo "Please install manually from:"
|
|
||||||
echo " https://github.com/lbryio/lbry-sdk/releases"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Return to original directory
|
|
||||||
cd - > /dev/null
|
|
||||||
else
|
|
||||||
print_warning "Skipping lbrynet installation"
|
|
||||||
echo "You'll need to install it manually before using the downloader"
|
|
||||||
fi
|
|
||||||
else
|
else
|
||||||
print_status "lbrynet found at: $(which lbrynet)"
|
print_info "lbrynet not found. That's fine for default direct mode."
|
||||||
|
print_info "Install it later only if you want to run with --p2p."
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Check if running in interactive mode
|
# Check if running in interactive mode
|
||||||
@@ -187,19 +128,17 @@ echo ""
|
|||||||
echo "1️⃣ Reload your shell (IMPORTANT):"
|
echo "1️⃣ Reload your shell (IMPORTANT):"
|
||||||
echo " source ~/.bashrc"
|
echo " source ~/.bashrc"
|
||||||
echo ""
|
echo ""
|
||||||
echo "2️⃣ Test the downloader (daemon starts automatically):"
|
echo "2️⃣ Test the downloader:"
|
||||||
echo " lbry-downloader --dry-run"
|
echo " lbry-downloader --dry-run"
|
||||||
echo ""
|
echo ""
|
||||||
echo "3️⃣ Download content:"
|
echo "3️⃣ Download content:"
|
||||||
echo " lbry-downloader"
|
echo " lbry-downloader --non-video-only"
|
||||||
echo ""
|
echo ""
|
||||||
echo "💡 If 'lbry-downloader' command not found after reload, use:"
|
echo "💡 If 'lbry-downloader' command not found after reload, use:"
|
||||||
echo " ~/Documents/LBRY-Downloader/bin/lbry-downloader"
|
echo " ~/Documents/LBRY-Downloader/bin/lbry-downloader"
|
||||||
echo ""
|
echo ""
|
||||||
echo "⚠️ If lbrynet was not installed automatically, install it from:"
|
echo "⚙️ Optional P2P mode:"
|
||||||
echo " https://github.com/lbryio/lbry-sdk/releases"
|
echo " Install lbrynet only if you want: ~/Documents/LBRY-Downloader/bin/lbry-downloader --p2p"
|
||||||
echo " Then run: lbrynet start"
|
|
||||||
echo " ~/Documents/LBRY-Downloader/bin/lbry-downloader"
|
|
||||||
echo ""
|
echo ""
|
||||||
echo "📁 Config file: ~/Documents/lbry-downloads/config.yaml"
|
echo "📁 Config file: ~/Documents/lbry-downloads/config.yaml"
|
||||||
echo ""
|
echo ""
|
||||||
@@ -76,7 +76,9 @@ if current_dir != Path(INSTALL_DIR):
|
|||||||
config_content = f"""# LBRY Downloader Configuration
|
config_content = f"""# LBRY Downloader Configuration
|
||||||
|
|
||||||
lbrynet:
|
lbrynet:
|
||||||
|
# Only used when running with --p2p
|
||||||
api_url: "http://127.0.0.1:5279"
|
api_url: "http://127.0.0.1:5279"
|
||||||
|
# Only used when running with --p2p
|
||||||
timeout_seconds: 60
|
timeout_seconds: 60
|
||||||
|
|
||||||
general:
|
general:
|
||||||
@@ -97,6 +99,7 @@ channels:
|
|||||||
# Add your channels here:
|
# Add your channels here:
|
||||||
# - input: "https://odysee.com/@ChannelName:1"
|
# - input: "https://odysee.com/@ChannelName:1"
|
||||||
# enabled: true
|
# enabled: true
|
||||||
|
# content_mode: "non_video_only"
|
||||||
"""
|
"""
|
||||||
|
|
||||||
config_path = Path(DOWNLOAD_DIR) / "config.yaml"
|
config_path = Path(DOWNLOAD_DIR) / "config.yaml"
|
||||||
@@ -126,3 +129,7 @@ print(f"To run: {launcher}")
|
|||||||
print()
|
print()
|
||||||
print("Edit the config to add channels, then run:")
|
print("Edit the config to add channels, then run:")
|
||||||
print(f" {launcher} --dry-run")
|
print(f" {launcher} --dry-run")
|
||||||
|
print("Recommended first real run:")
|
||||||
|
print(f" {launcher} --non-video-only")
|
||||||
|
print("Optional local-node mode:")
|
||||||
|
print(f" {launcher} --p2p")
|
||||||
@@ -350,8 +350,10 @@ def create_config(
|
|||||||
|
|
||||||
lbrynet:
|
lbrynet:
|
||||||
# URL of the local LBRY daemon (lbrynet)
|
# URL of the local LBRY daemon (lbrynet)
|
||||||
|
# Only used when running with --p2p
|
||||||
api_url: "http://127.0.0.1:5279"
|
api_url: "http://127.0.0.1:5279"
|
||||||
# Timeout for daemon requests in seconds
|
# Timeout for daemon requests in seconds
|
||||||
|
# Only used when running with --p2p
|
||||||
timeout_seconds: 60
|
timeout_seconds: 60
|
||||||
|
|
||||||
general:
|
general:
|
||||||
@@ -386,12 +388,14 @@ channels:
|
|||||||
for channel in channels:
|
for channel in channels:
|
||||||
config += f''' - input: "{channel["input"]}"
|
config += f''' - input: "{channel["input"]}"
|
||||||
enabled: true
|
enabled: true
|
||||||
|
content_mode: "non_video_only"
|
||||||
'''
|
'''
|
||||||
|
|
||||||
if not channels:
|
if not channels:
|
||||||
config += """ # Add your channels here:
|
config += """ # Add your channels here:
|
||||||
# - input: "https://odysee.com/@ChannelName:1"
|
# - input: "https://odysee.com/@ChannelName:1"
|
||||||
# enabled: true
|
# enabled: true
|
||||||
|
# content_mode: "non_video_only"
|
||||||
"""
|
"""
|
||||||
|
|
||||||
return config
|
return config
|
||||||
@@ -405,54 +409,13 @@ def create_launcher_script(install_dir: Path, venv_path: Path) -> None:
|
|||||||
bin_dir = install_dir / "bin"
|
bin_dir = install_dir / "bin"
|
||||||
bin_dir.mkdir(exist_ok=True)
|
bin_dir.mkdir(exist_ok=True)
|
||||||
|
|
||||||
# Create main launcher with daemon auto-check
|
# Create main launcher
|
||||||
launcher = bin_dir / "lbry-downloader"
|
launcher = bin_dir / "lbry-downloader"
|
||||||
launcher_content = f"""#!/bin/bash
|
launcher_content = f"""#!/bin/bash
|
||||||
# LBRY Downloader Launcher
|
# LBRY Downloader Launcher
|
||||||
# Auto-checks and starts lbrynet daemon if needed
|
|
||||||
|
|
||||||
INSTALL_DIR="{install_dir}"
|
INSTALL_DIR="{install_dir}"
|
||||||
VENV_DIR="{venv_path}"
|
VENV_DIR="{venv_path}"
|
||||||
DAEMON_URL="http://127.0.0.1:5279"
|
|
||||||
|
|
||||||
# Check if daemon is running
|
|
||||||
if ! curl -s "$DAEMON_URL" > /dev/null 2>&1; then
|
|
||||||
echo "LBRY daemon not running. Starting..."
|
|
||||||
|
|
||||||
# Try to start daemon
|
|
||||||
if command -v lbrynet &> /dev/null; then
|
|
||||||
lbrynet start &
|
|
||||||
echo "Waiting for daemon to start..."
|
|
||||||
|
|
||||||
# Wait up to 30 seconds for daemon
|
|
||||||
for i in $(seq 1 30); do
|
|
||||||
if curl -s "$DAEMON_URL" > /dev/null 2>&1; then
|
|
||||||
echo "✓ Daemon started successfully"
|
|
||||||
break
|
|
||||||
fi
|
|
||||||
sleep 1
|
|
||||||
done
|
|
||||||
|
|
||||||
# Check if daemon started
|
|
||||||
if ! curl -s "$DAEMON_URL" > /dev/null 2>&1; then
|
|
||||||
echo "✗ Failed to start daemon automatically"
|
|
||||||
echo ""
|
|
||||||
echo "Please start it manually:"
|
|
||||||
echo " lbrynet start"
|
|
||||||
echo ""
|
|
||||||
echo "Or see LINUX_SETUP.md for installation instructions"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
echo "✗ lbrynet not found in PATH"
|
|
||||||
echo ""
|
|
||||||
echo "Please install LBRY SDK:"
|
|
||||||
echo " https://github.com/lbryio/lbry-sdk/releases"
|
|
||||||
echo ""
|
|
||||||
echo "Or see LINUX_SETUP.md for detailed instructions"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
cd "$INSTALL_DIR"
|
cd "$INSTALL_DIR"
|
||||||
source "$VENV_DIR/bin/activate"
|
source "$VENV_DIR/bin/activate"
|
||||||
@@ -464,37 +427,13 @@ python main.py "$@"
|
|||||||
launcher.chmod(0o755)
|
launcher.chmod(0o755)
|
||||||
print_success(f"Created launcher: {launcher}")
|
print_success(f"Created launcher: {launcher}")
|
||||||
|
|
||||||
# Create quick test script with daemon auto-check
|
# Create quick test script
|
||||||
test_script = bin_dir / "lbry-test"
|
test_script = bin_dir / "lbry-test"
|
||||||
test_content = f"""#!/bin/bash
|
test_content = f"""#!/bin/bash
|
||||||
# Quick test script
|
# Quick test script
|
||||||
|
|
||||||
INSTALL_DIR="{install_dir}"
|
INSTALL_DIR="{install_dir}"
|
||||||
VENV_DIR="{venv_path}"
|
VENV_DIR="{venv_path}"
|
||||||
DAEMON_URL="http://127.0.0.1:5279"
|
|
||||||
|
|
||||||
# Check if daemon is running
|
|
||||||
if ! curl -s "$DAEMON_URL" > /dev/null 2>&1; then
|
|
||||||
echo "LBRY daemon not running. Starting..."
|
|
||||||
|
|
||||||
if command -v lbrynet &> /dev/null; then
|
|
||||||
lbrynet start &
|
|
||||||
echo "Waiting for daemon to start..."
|
|
||||||
sleep 5
|
|
||||||
|
|
||||||
if curl -s "$DAEMON_URL" > /dev/null 2>&1; then
|
|
||||||
echo "✓ Daemon started"
|
|
||||||
else
|
|
||||||
echo "✗ Failed to start daemon"
|
|
||||||
echo "Run manually: lbrynet start"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
echo "✗ lbrynet not found. Install from:"
|
|
||||||
echo " https://github.com/lbryio/lbry-sdk/releases"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
cd "$INSTALL_DIR"
|
cd "$INSTALL_DIR"
|
||||||
source "$VENV_DIR/bin/activate"
|
source "$VENV_DIR/bin/activate"
|
||||||
@@ -533,10 +472,10 @@ def run_setup() -> None:
|
|||||||
"""Main setup function."""
|
"""Main setup function."""
|
||||||
print(f"""
|
print(f"""
|
||||||
{Colors.BOLD}╔══════════════════════════════════════════════════════════════╗
|
{Colors.BOLD}╔══════════════════════════════════════════════════════════════╗
|
||||||
║ LBRY Downloader - Interactive Setup ║
|
║ Odysee/LBRY Downloader - Interactive Setup ║
|
||||||
║ ║
|
║ ║
|
||||||
║ This script will help you install and configure the ║
|
║ This script will help you install and configure the ║
|
||||||
║ LBRY Downloader tool. ║
|
║ direct-first archive downloader. ║
|
||||||
╚══════════════════════════════════════════════════════════════╝{Colors.END}
|
╚══════════════════════════════════════════════════════════════╝{Colors.END}
|
||||||
""")
|
""")
|
||||||
|
|
||||||
@@ -662,20 +601,20 @@ def run_setup() -> None:
|
|||||||
║ Setup Complete! ║
|
║ Setup Complete! ║
|
||||||
╚══════════════════════════════════════════════════════════════╝{Colors.END}
|
╚══════════════════════════════════════════════════════════════╝{Colors.END}
|
||||||
|
|
||||||
🎉 LBRY Downloader is ready to use!
|
🎉 Odysee/LBRY Downloader is ready to use!
|
||||||
|
|
||||||
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||||
📋 QUICK START (Run these commands):
|
📋 QUICK START (Run these commands):
|
||||||
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||||
|
|
||||||
1️⃣ Start the LBRY daemon:
|
1️⃣ Optional P2P prerequisite:
|
||||||
lbrynet start
|
Optional only for P2P mode: lbrynet start
|
||||||
|
|
||||||
2️⃣ Test your configuration:
|
2️⃣ Test your configuration:
|
||||||
lbry-downloader --dry-run
|
lbry-downloader --dry-run
|
||||||
|
|
||||||
3️⃣ Download content:
|
3️⃣ Download content:
|
||||||
lbry-downloader
|
lbry-downloader --non-video-only
|
||||||
|
|
||||||
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||||
ℹ️ IMPORTANT NOTES:
|
ℹ️ IMPORTANT NOTES:
|
||||||
@@ -688,7 +627,7 @@ def run_setup() -> None:
|
|||||||
If {Colors.BOLD}lbry-downloader{Colors.END} command not found, run:
|
If {Colors.BOLD}lbry-downloader{Colors.END} command not found, run:
|
||||||
source ~/.bashrc
|
source ~/.bashrc
|
||||||
|
|
||||||
If lbrynet is not installed, see LINUX_SETUP.md for installation.
|
Direct mode is the default. Use {Colors.BOLD}--p2p{Colors.END} only if you explicitly want the local node path.
|
||||||
|
|
||||||
To add more channels later:
|
To add more channels later:
|
||||||
nano {config_path}
|
nano {config_path}
|
||||||
|
|||||||
@@ -0,0 +1,76 @@
|
|||||||
|
"""Tests for per-channel content mode config and resolution."""
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from config_loader import ConfigError, _parse_config
|
||||||
|
from main import resolve_content_filter_mode
|
||||||
|
from models import ChannelConfig
|
||||||
|
|
||||||
|
|
||||||
|
def test_channel_content_mode_parses_from_config():
|
||||||
|
config = _parse_config(
|
||||||
|
{
|
||||||
|
"channels": [
|
||||||
|
{
|
||||||
|
"input": "https://odysee.com/@Files:1",
|
||||||
|
"enabled": True,
|
||||||
|
"content_mode": "non_video_only",
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
assert config.channels[0].content_mode == "non_video_only"
|
||||||
|
|
||||||
|
|
||||||
|
def test_invalid_channel_content_mode_is_rejected():
|
||||||
|
with pytest.raises(ConfigError) as exc_info:
|
||||||
|
_parse_config(
|
||||||
|
{
|
||||||
|
"channels": [
|
||||||
|
{
|
||||||
|
"input": "https://odysee.com/@Files:1",
|
||||||
|
"enabled": True,
|
||||||
|
"content_mode": "zip_only",
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
assert "content_mode" in str(exc_info.value)
|
||||||
|
|
||||||
|
|
||||||
|
def test_channel_content_mode_defaults_to_all():
|
||||||
|
channel_config = ChannelConfig(input="https://odysee.com/@Mixed:1")
|
||||||
|
|
||||||
|
assert resolve_content_filter_mode(channel_config) == (
|
||||||
|
False,
|
||||||
|
False,
|
||||||
|
"all content",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_channel_content_mode_applies_when_no_cli_override():
|
||||||
|
channel_config = ChannelConfig(
|
||||||
|
input="https://odysee.com/@Videos:1",
|
||||||
|
content_mode="video_only",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert resolve_content_filter_mode(channel_config) == (
|
||||||
|
True,
|
||||||
|
False,
|
||||||
|
"video-only (channel config)",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_cli_override_beats_channel_content_mode():
|
||||||
|
channel_config = ChannelConfig(
|
||||||
|
input="https://odysee.com/@Files:1",
|
||||||
|
content_mode="non_video_only",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert resolve_content_filter_mode(channel_config, video_only=True) == (
|
||||||
|
True,
|
||||||
|
False,
|
||||||
|
"video-only (CLI override)",
|
||||||
|
)
|
||||||
@@ -122,6 +122,50 @@ channels:
|
|||||||
|
|
||||||
assert "filename_mode" in str(exc_info.value)
|
assert "filename_mode" in str(exc_info.value)
|
||||||
|
|
||||||
|
def test_valid_direct_retry_config(self, tmp_path):
|
||||||
|
config_file = tmp_path / "config.yaml"
|
||||||
|
config_file.write_text("""
|
||||||
|
general:
|
||||||
|
base_dir: "~/test"
|
||||||
|
direct_base_urls:
|
||||||
|
- "https://odysee.com/"
|
||||||
|
- "https://mirror.example/"
|
||||||
|
direct_max_retries_per_url: 3
|
||||||
|
direct_retry_backoff_seconds: 1.5
|
||||||
|
direct_auto_fallback_to_p2p: true
|
||||||
|
|
||||||
|
channels:
|
||||||
|
- input: "https://odysee.com/@Test:1"
|
||||||
|
enabled: true
|
||||||
|
""")
|
||||||
|
|
||||||
|
config = load_config(str(config_file))
|
||||||
|
|
||||||
|
assert config.general.direct_base_urls == [
|
||||||
|
"https://odysee.com",
|
||||||
|
"https://mirror.example",
|
||||||
|
]
|
||||||
|
assert config.general.direct_max_retries_per_url == 3
|
||||||
|
assert config.general.direct_retry_backoff_seconds == 1.5
|
||||||
|
assert config.general.direct_auto_fallback_to_p2p is True
|
||||||
|
|
||||||
|
def test_invalid_direct_retry_backoff(self, tmp_path):
|
||||||
|
config_file = tmp_path / "config.yaml"
|
||||||
|
config_file.write_text("""
|
||||||
|
general:
|
||||||
|
base_dir: "~/test"
|
||||||
|
direct_retry_backoff_seconds: 0
|
||||||
|
|
||||||
|
channels:
|
||||||
|
- input: "https://odysee.com/@Test:1"
|
||||||
|
enabled: true
|
||||||
|
""")
|
||||||
|
|
||||||
|
with pytest.raises(ConfigError) as exc_info:
|
||||||
|
load_config(str(config_file))
|
||||||
|
|
||||||
|
assert "direct_retry_backoff_seconds" in str(exc_info.value)
|
||||||
|
|
||||||
|
|
||||||
class TestDefaultConfig:
|
class TestDefaultConfig:
|
||||||
def test_create_default_config(self):
|
def test_create_default_config(self):
|
||||||
|
|||||||
@@ -0,0 +1,58 @@
|
|||||||
|
"""Tests for CLI content-type filtering."""
|
||||||
|
|
||||||
|
from main import filter_actions_by_content_type
|
||||||
|
from models import DownloadAction
|
||||||
|
|
||||||
|
|
||||||
|
def make_action(name: str, **metadata):
|
||||||
|
base_metadata = {
|
||||||
|
"source_name": name,
|
||||||
|
"file_name": name,
|
||||||
|
}
|
||||||
|
base_metadata.update(metadata)
|
||||||
|
return DownloadAction(
|
||||||
|
claim_id=f"claim-{name}",
|
||||||
|
claim_name=name,
|
||||||
|
channel_claim_id="channel-1",
|
||||||
|
version_token=f"version-{name}",
|
||||||
|
action="download_new",
|
||||||
|
uri=f"lbry://{name}",
|
||||||
|
target_dir="/tmp",
|
||||||
|
metadata=base_metadata,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_video_only_keeps_only_video_actions():
|
||||||
|
video_action = make_action("clip.mp4", source_media_type="video/mp4")
|
||||||
|
zip_action = make_action("bundle.zip", source_media_type="application/zip")
|
||||||
|
|
||||||
|
actions = filter_actions_by_content_type(
|
||||||
|
[video_action, zip_action],
|
||||||
|
video_only=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert actions == [video_action]
|
||||||
|
|
||||||
|
|
||||||
|
def test_non_video_only_keeps_zip_and_skips_video():
|
||||||
|
video_action = make_action("clip.mp4", source_media_type="video/mp4")
|
||||||
|
zip_action = make_action("bundle.zip", source_media_type="application/zip")
|
||||||
|
|
||||||
|
actions = filter_actions_by_content_type(
|
||||||
|
[video_action, zip_action],
|
||||||
|
non_video_only=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert actions == [zip_action]
|
||||||
|
|
||||||
|
|
||||||
|
def test_video_filter_falls_back_to_filename_extension():
|
||||||
|
video_action = make_action("clip.mkv")
|
||||||
|
other_action = make_action("bundle.zip")
|
||||||
|
|
||||||
|
actions = filter_actions_by_content_type(
|
||||||
|
[video_action, other_action],
|
||||||
|
video_only=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert actions == [video_action]
|
||||||
@@ -0,0 +1,183 @@
|
|||||||
|
"""Tests for the direct Odysee downloader."""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
from pathlib import Path
|
||||||
|
from types import SimpleNamespace
|
||||||
|
|
||||||
|
import requests
|
||||||
|
from direct_downloader import DirectDownloader
|
||||||
|
from models import Config, DownloadAction
|
||||||
|
|
||||||
|
|
||||||
|
def make_action(**metadata_overrides):
|
||||||
|
metadata = {
|
||||||
|
"normalized_name": "veritasium-trailer",
|
||||||
|
"name": "veritasium-trailer",
|
||||||
|
"source_name": "veritasium-trailer.mp4",
|
||||||
|
}
|
||||||
|
metadata.update(metadata_overrides)
|
||||||
|
|
||||||
|
return DownloadAction(
|
||||||
|
claim_id="19201da2f75357d70b151f7d87ddd94227884126",
|
||||||
|
claim_name="veritasium-trailer",
|
||||||
|
channel_claim_id="fb364ef587872515f545a5b4b3182b58073f230f",
|
||||||
|
version_token="sd_deadbeefdeadbeef",
|
||||||
|
action="download_new",
|
||||||
|
uri="lbry://veritasium-trailer#19201da2f75357d70b151f7d87ddd94227884126",
|
||||||
|
target_dir="/tmp/lbry-test",
|
||||||
|
metadata=metadata,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TestDirectDownloader:
|
||||||
|
def test_build_stream_url_uses_normalized_name(self):
|
||||||
|
downloader = DirectDownloader(Config(), Path("/tmp"))
|
||||||
|
action = make_action()
|
||||||
|
|
||||||
|
assert (
|
||||||
|
downloader.build_stream_url(action)
|
||||||
|
== "https://odysee.com/$/stream/veritasium-trailer/"
|
||||||
|
"19201da2f75357d70b151f7d87ddd94227884126"
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_build_stream_url_quotes_special_slug(self):
|
||||||
|
downloader = DirectDownloader(Config(), Path("/tmp"))
|
||||||
|
action = make_action(normalized_name="Title With Spaces & Symbols")
|
||||||
|
|
||||||
|
assert downloader.build_stream_url(action).startswith(
|
||||||
|
"https://odysee.com/$/stream/Title%20With%20Spaces%20%26%20Symbols/"
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_build_download_url_uses_claim_id(self):
|
||||||
|
downloader = DirectDownloader(Config(), Path("/tmp"))
|
||||||
|
action = make_action()
|
||||||
|
|
||||||
|
assert (
|
||||||
|
downloader.build_download_url(action)
|
||||||
|
== "https://odysee.com/$/download/veritasium-trailer/"
|
||||||
|
"19201da2f75357d70b151f7d87ddd94227884126"
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_file_claims_prefer_download_endpoint(self):
|
||||||
|
downloader = DirectDownloader(Config(), Path("/tmp"))
|
||||||
|
action = make_action(
|
||||||
|
source_name="archive.zip",
|
||||||
|
source_media_type="application/zip",
|
||||||
|
stream_type="binary",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert downloader.build_candidate_urls(action) == [
|
||||||
|
"https://odysee.com/$/download/veritasium-trailer/"
|
||||||
|
"19201da2f75357d70b151f7d87ddd94227884126",
|
||||||
|
"https://odysee.com/$/stream/veritasium-trailer/"
|
||||||
|
"19201da2f75357d70b151f7d87ddd94227884126",
|
||||||
|
]
|
||||||
|
|
||||||
|
def test_video_claims_prefer_stream_endpoint(self):
|
||||||
|
downloader = DirectDownloader(Config(), Path("/tmp"))
|
||||||
|
action = make_action(
|
||||||
|
source_media_type="video/mp4",
|
||||||
|
stream_type="video",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert downloader.build_candidate_urls(action) == [
|
||||||
|
"https://odysee.com/$/stream/veritasium-trailer/"
|
||||||
|
"19201da2f75357d70b151f7d87ddd94227884126",
|
||||||
|
"https://odysee.com/$/download/veritasium-trailer/"
|
||||||
|
"19201da2f75357d70b151f7d87ddd94227884126",
|
||||||
|
]
|
||||||
|
|
||||||
|
def test_candidate_urls_include_configured_mirrors(self):
|
||||||
|
config = Config()
|
||||||
|
config.general.direct_base_urls = [
|
||||||
|
"https://odysee.com",
|
||||||
|
"https://mirror.example",
|
||||||
|
]
|
||||||
|
downloader = DirectDownloader(config, Path("/tmp"))
|
||||||
|
action = make_action(
|
||||||
|
source_media_type="video/mp4",
|
||||||
|
stream_type="video",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert downloader.build_candidate_urls(action) == [
|
||||||
|
"https://odysee.com/$/stream/veritasium-trailer/"
|
||||||
|
"19201da2f75357d70b151f7d87ddd94227884126",
|
||||||
|
"https://odysee.com/$/download/veritasium-trailer/"
|
||||||
|
"19201da2f75357d70b151f7d87ddd94227884126",
|
||||||
|
"https://mirror.example/$/stream/veritasium-trailer/"
|
||||||
|
"19201da2f75357d70b151f7d87ddd94227884126",
|
||||||
|
"https://mirror.example/$/download/veritasium-trailer/"
|
||||||
|
"19201da2f75357d70b151f7d87ddd94227884126",
|
||||||
|
]
|
||||||
|
|
||||||
|
def test_429_retries_use_backoff_budget(self, monkeypatch, tmp_path):
|
||||||
|
config = Config()
|
||||||
|
config.general.direct_max_retries_per_url = 2
|
||||||
|
config.general.direct_retry_backoff_seconds = 0.01
|
||||||
|
downloader = DirectDownloader(config, Path("/tmp"))
|
||||||
|
file_path = tmp_path / "retry-test.mp4"
|
||||||
|
|
||||||
|
calls = {"count": 0}
|
||||||
|
sleeps = []
|
||||||
|
|
||||||
|
class FakeResponse:
|
||||||
|
headers = {}
|
||||||
|
|
||||||
|
def raise_for_status(self):
|
||||||
|
http_error = requests.exceptions.HTTPError("429")
|
||||||
|
http_error.response = SimpleNamespace(status_code=429)
|
||||||
|
raise http_error
|
||||||
|
|
||||||
|
def fake_get(*args, **kwargs):
|
||||||
|
calls["count"] += 1
|
||||||
|
return FakeResponse()
|
||||||
|
|
||||||
|
monkeypatch.setattr(downloader.session, "get", fake_get)
|
||||||
|
monkeypatch.setattr("direct_downloader.time.sleep", lambda seconds: sleeps.append(seconds))
|
||||||
|
|
||||||
|
success, reason = downloader._download_file(
|
||||||
|
"https://odysee.com/$/download/test/abc",
|
||||||
|
file_path,
|
||||||
|
"retry-test",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert success is False
|
||||||
|
assert "rate-limited" in reason.lower()
|
||||||
|
assert calls["count"] == 3
|
||||||
|
assert sleeps == [0.01, 0.02]
|
||||||
|
|
||||||
|
def test_skip_existing_logs_existing_file_path(self, caplog):
|
||||||
|
downloader = DirectDownloader(Config(), Path("/tmp"))
|
||||||
|
action = make_action(
|
||||||
|
existing_file_relpath="channels/test/claims/item/versions/sd/file.zip",
|
||||||
|
)
|
||||||
|
action.action = "skip_existing"
|
||||||
|
|
||||||
|
with caplog.at_level(logging.INFO):
|
||||||
|
success = downloader.download(action)
|
||||||
|
|
||||||
|
assert success is True
|
||||||
|
assert "Skipping existing file" in caplog.text
|
||||||
|
assert "/tmp/channels/test/claims/item/versions/sd/file.zip" in caplog.text
|
||||||
|
|
||||||
|
def test_download_logs_destination_path(self, monkeypatch, caplog, tmp_path):
|
||||||
|
downloader = DirectDownloader(Config(), Path("/tmp"))
|
||||||
|
action = make_action()
|
||||||
|
action.target_dir = str(tmp_path)
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
downloader,
|
||||||
|
"_download_file",
|
||||||
|
lambda url, file_path, claim_name: (True, None),
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(downloader, "_write_metadata", lambda *args, **kwargs: None)
|
||||||
|
monkeypatch.setattr(downloader, "_write_checksum", lambda *args, **kwargs: None)
|
||||||
|
target_file = tmp_path / "veritasium-trailer.mp4"
|
||||||
|
target_file.write_bytes(b"data")
|
||||||
|
|
||||||
|
with caplog.at_level(logging.INFO):
|
||||||
|
success = downloader.download(action)
|
||||||
|
|
||||||
|
assert success is True
|
||||||
|
assert "Saving to:" in caplog.text
|
||||||
|
assert str(target_file) in caplog.text
|
||||||
@@ -104,6 +104,10 @@ class TestPlanner:
|
|||||||
|
|
||||||
assert len(actions) == 1
|
assert len(actions) == 1
|
||||||
assert actions[0].action == "skip_existing"
|
assert actions[0].action == "skip_existing"
|
||||||
|
assert (
|
||||||
|
actions[0].metadata["existing_file_relpath"]
|
||||||
|
== f"channels/Test__channel123/claims/{claim_folder}/versions/{version_token}/video.mp4"
|
||||||
|
)
|
||||||
|
|
||||||
def test_new_version_detected(self):
|
def test_new_version_detected(self):
|
||||||
channel = Channel(
|
channel = Channel(
|
||||||
|
|||||||
@@ -0,0 +1,15 @@
|
|||||||
|
"""Tests for transport mode selection."""
|
||||||
|
|
||||||
|
from main import resolve_transport_mode
|
||||||
|
|
||||||
|
|
||||||
|
def test_transport_mode_defaults_to_direct():
|
||||||
|
assert resolve_transport_mode() == (True, "direct")
|
||||||
|
|
||||||
|
|
||||||
|
def test_transport_mode_p2p_override_wins():
|
||||||
|
assert resolve_transport_mode(p2p=True) == (False, "p2p")
|
||||||
|
|
||||||
|
|
||||||
|
def test_transport_mode_explicit_direct():
|
||||||
|
assert resolve_transport_mode(direct=True) == (True, "direct")
|
||||||
@@ -256,6 +256,7 @@ def extract_claim_metadata(claim_data: Dict[str, Any]) -> Dict[str, Any]:
|
|||||||
"normalized_name": claim.get("normalized_name"),
|
"normalized_name": claim.get("normalized_name"),
|
||||||
"claim_type": claim.get("claim_type"),
|
"claim_type": claim.get("claim_type"),
|
||||||
"value_type": claim.get("value_type"),
|
"value_type": claim.get("value_type"),
|
||||||
|
"stream_type": value.get("stream_type"),
|
||||||
"permanent_url": claim.get("permanent_url"),
|
"permanent_url": claim.get("permanent_url"),
|
||||||
"short_url": claim.get("short_url"),
|
"short_url": claim.get("short_url"),
|
||||||
"canonical_url": claim.get("canonical_url"),
|
"canonical_url": claim.get("canonical_url"),
|
||||||
|
|||||||
Reference in new issue
Block a user