Complete LBRY Downloader implementation v1.0.0

Features:
- Incremental sync with version tracking (sd_hash)
- Per-channel download paths
- Download limit (default: 10 most recent per channel)
- Interactive setup script with user prompts
- Uninstall script for clean removal
- Comprehensive test suite (42 tests)
- Full documentation suite

Files:
- 9 Python core modules (main, models, utils, config, client, state, planner, downloader, init)
- 3 Setup scripts (setup.py, uninstall.py, install.sh)
- 5 Test files (42 tests, all passing)
- 9 Documentation files
- 4 Build/config files

Total: 31 files, ~4,500 lines

setup.py

@@ -0,0 +1,593 @@
+#!/usr/bin/env python3
+"""
+LBRY Downloader - Interactive Setup Script
+
+This script handles:
+- Dependency checking and installation
+- Virtual environment setup
+- Configuration with user prompts
+- Download location selection
+- Channel configuration
+"""
+
+import os
+import sys
+import subprocess
+import platform
+import shutil
+from pathlib import Path
+from typing import Optional, List, Tuple
+
+
+# Colors for terminal output
+class Colors:
+    GREEN = "\033[92m"
+    YELLOW = "\033[93m"
+    RED = "\033[91m"
+    BLUE = "\033[94m"
+    BOLD = "\033[1m"
+    END = "\033[0m"
+
+
+def print_success(msg: str):
+    print(f"{Colors.GREEN}✓{Colors.END} {msg}")
+
+
+def print_warning(msg: str):
+    print(f"{Colors.YELLOW}⚠{Colors.END} {msg}")
+
+
+def print_error(msg: str):
+    print(f"{Colors.RED}✗{Colors.END} {msg}")
+
+
+def print_info(msg: str):
+    print(f"{Colors.BLUE}ℹ{Colors.END} {msg}")
+
+
+def print_header(msg: str):
+    print(f"\n{Colors.BOLD}{msg}{Colors.END}")
+    print("=" * 60)
+
+
+def run_command(
+    cmd: List[str], check: bool = True, capture: bool = False
+) -> Tuple[int, str, str]:
+    """Run a shell command and return result."""
+    try:
+        if capture:
+            result = subprocess.run(cmd, check=check, capture_output=True, text=True)
+            return result.returncode, result.stdout, result.stderr
+        else:
+            result = subprocess.run(cmd, check=check)
+            return result.returncode, "", ""
+    except subprocess.CalledProcessError as e:
+        if check:
+            raise
+        return e.returncode, "", str(e)
+    except FileNotFoundError:
+        if check:
+            raise
+        return 1, "", f"Command not found: {cmd[0]}"
+
+
+def check_python_version() -> bool:
+    """Check if Python version is 3.8 or higher."""
+    version = sys.version_info
+    if version.major < 3 or (version.major == 3 and version.minor < 8):
+        print_error(f"Python 3.8+ required, found {version.major}.{version.minor}")
+        return False
+    print_success(f"Python {version.major}.{version.minor}.{version.micro} found")
+    return True
+
+
+def check_system_dependencies() -> bool:
+    """Check if required system packages are available."""
+    print_header("Checking System Dependencies")
+
+    missing = []
+
+    # Check for apt (Debian-based)
+    if shutil.which("apt-get"):
+        print_success("Package manager (apt-get) found")
+    elif shutil.which("dnf"):
+        print_success("Package manager (dnf) found")
+    elif shutil.which("pacman"):
+        print_success("Package manager (pacman) found")
+    else:
+        print_warning("No recognized package manager found")
+
+    # Check for git
+    if shutil.which("git"):
+        print_success("Git found")
+    else:
+        print_warning("Git not found (optional)")
+
+    # Check for curl
+    if shutil.which("curl"):
+        print_success("curl found")
+    else:
+        missing.append("curl")
+
+    return len(missing) == 0
+
+
+def install_system_dependencies() -> bool:
+    """Install required system packages."""
+    print_header("Installing System Dependencies")
+
+    packages = ["python3-pip", "python3-venv"]
+
+    if shutil.which("apt-get"):
+        # Debian/Ubuntu/Mint
+        cmd = ["sudo", "apt-get", "update"]
+        print_info("Updating package lists...")
+        try:
+            run_command(cmd)
+        except:
+            print_warning("Failed to update package lists, continuing anyway...")
+
+        cmd = ["sudo", "apt-get", "install", "-y"] + packages
+        print_info("Installing packages: " + ", ".join(packages))
+        try:
+            run_command(cmd)
+            print_success("System dependencies installed")
+            return True
+        except Exception as e:
+            print_error(f"Failed to install packages: {e}")
+            return False
+    else:
+        print_warning("Automatic installation only supported on apt-based systems")
+        print_info("Please manually install: " + ", ".join(packages))
+        return True  # Continue anyway
+
+
+def setup_virtual_environment(install_dir: Path) -> Path:
+    """Create and setup virtual environment."""
+    print_header("Setting up Virtual Environment")
+
+    venv_path = install_dir / "venv"
+
+    if venv_path.exists():
+        print_warning("Virtual environment already exists")
+        response = input("Recreate it? (y/N): ").strip().lower()
+        if response == "y":
+            shutil.rmtree(venv_path)
+        else:
+            print_info("Using existing virtual environment")
+            return venv_path
+
+    print_info("Creating virtual environment...")
+    try:
+        run_command([sys.executable, "-m", "venv", str(venv_path)])
+        print_success("Virtual environment created")
+    except Exception as e:
+        print_error(f"Failed to create virtual environment: {e}")
+        sys.exit(1)
+
+    return venv_path
+
+
+def install_python_dependencies(venv_path: Path, install_dir: Path) -> bool:
+    """Install Python packages in virtual environment."""
+    print_header("Installing Python Dependencies")
+
+    pip_path = venv_path / "bin" / "pip"
+    if not pip_path.exists():
+        pip_path = venv_path / "Scripts" / "pip.exe"  # Windows
+
+    requirements_file = install_dir / "requirements.txt"
+
+    print_info("Upgrading pip...")
+    try:
+        run_command([str(pip_path), "install", "--upgrade", "pip"])
+    except:
+        print_warning("Failed to upgrade pip, continuing...")
+
+    print_info("Installing required packages...")
+    try:
+        run_command([str(pip_path), "install", "-r", str(requirements_file)])
+        print_success("Python dependencies installed")
+        return True
+    except Exception as e:
+        print_error(f"Failed to install Python dependencies: {e}")
+        return False
+
+
+def get_installation_directory() -> Path:
+    """Ask user for installation directory."""
+    print_header("Installation Directory")
+
+    default_dir = Path.home() / "Documents" / "LBRY-Downloader"
+
+    print_info(f"Default: {default_dir}")
+    response = input(f"Enter installation directory [{default_dir}]: ").strip()
+
+    if response:
+        install_dir = Path(response).expanduser().resolve()
+    else:
+        install_dir = default_dir
+
+    # Create directory if it doesn't exist
+    install_dir.mkdir(parents=True, exist_ok=True)
+
+    print_success(f"Installation directory: {install_dir}")
+    return install_dir
+
+
+def get_download_location() -> str:
+    """Ask user for download location."""
+    print_header("Download Location")
+
+    print_info("Where would you like to store downloaded content?")
+    print_info(
+        "(This is the base directory - each channel will have its own subfolder)"
+    )
+    print()
+
+    default_location = "~/Documents/lbry-downloads"
+    print_info(f"Default: {default_location}")
+
+    while True:
+        response = input(f"Enter download location [{default_location}]: ").strip()
+
+        if not response:
+            location = default_location
+        else:
+            location = response
+
+        # Expand and check
+        expanded_path = Path(location).expanduser().resolve()
+
+        try:
+            # Try to create it
+            expanded_path.mkdir(parents=True, exist_ok=True)
+
+            # Check if writable
+            test_file = expanded_path / ".write_test"
+            try:
+                test_file.touch()
+                test_file.unlink()
+                print_success(f"Download location: {expanded_path}")
+                return str(location)
+            except PermissionError:
+                print_error(f"Cannot write to {expanded_path}")
+                print_info("Please choose a different location or fix permissions")
+        except Exception as e:
+            print_error(f"Invalid path: {e}")
+
+
+def get_channels() -> List[dict]:
+    """Ask user for channels to download."""
+    print_header("Channel Configuration")
+
+    print_info("Enter the channels you want to download from.")
+    print_info("You can add more later by editing the config file.")
+    print()
+    print_info("Supported formats:")
+    print_info("  - Odysee URL: https://odysee.com/@ChannelName:1")
+    print_info("  - LBRY URI: lbry://@ChannelName#1")
+    print_info("  - Just press Enter when done")
+    print()
+
+    channels = []
+
+    while True:
+        channel_num = len(channels) + 1
+        url = input(f"Channel {channel_num} URL (or Enter to finish): ").strip()
+
+        if not url:
+            if len(channels) == 0:
+                print_warning(
+                    "No channels added. You can add them later in the config file."
+                )
+            break
+
+        # Basic validation
+        if not ("odysee.com" in url or "lbry://" in url or url.startswith("@")):
+            print_warning("URL format looks unusual. Make sure it's correct.")
+            confirm = input("Use this URL anyway? (Y/n): ").strip().lower()
+            if confirm == "n":
+                continue
+
+        channels.append({"input": url, "enabled": True})
+        print_success(f"Added: {url}")
+
+    return channels
+
+
+def get_download_limit() -> int:
+    """Ask user for download limit."""
+    print_header("Download Limit")
+
+    print_info("How many items should be downloaded per channel per run?")
+    print_info("  - Lower numbers = safer for testing")
+    print_info("  - Higher numbers = faster initial sync")
+    print_info("  - Downloads most recent items first")
+    print()
+
+    print_info("Recommended values:")
+    print_info("  - 5-10: Good for testing or large channels")
+    print_info("  - 20-50: Moderate syncing")
+    print_info("  - 0: Download everything (use with caution!)")
+    print()
+
+    while True:
+        response = input("Download limit per channel [10]: ").strip()
+
+        if not response:
+            return 10
+
+        try:
+            limit = int(response)
+            if limit < 0:
+                print_error("Please enter 0 or a positive number")
+                continue
+
+            if limit == 0:
+                confirm = (
+                    input("Are you sure you want to download ALL items? (y/N): ")
+                    .strip()
+                    .lower()
+                )
+                if confirm != "y":
+                    continue
+                print_warning("Download limit set to ALL items")
+            else:
+                print_success(f"Download limit set to {limit} items per channel")
+
+            return limit
+        except ValueError:
+            print_error("Please enter a number")
+
+
+def create_config(
+    download_location: str, channels: List[dict], download_limit: int
+) -> str:
+    """Generate configuration file content."""
+    config = f"""# LBRY Downloader Configuration
+# Generated by setup script on {platform.platform()}
+
+lbrynet:
+  # URL of the local LBRY daemon (lbrynet)
+  api_url: "http://127.0.0.1:5279"
+  # Timeout for daemon requests in seconds
+  timeout_seconds: 60
+
+general:
+  # Base directory for all downloads and state
+  base_dir: "{download_location}"
+  # Path to the state database
+  state_file: "{download_location}/state/database.json"
+  # Number of concurrent downloads (keep low to avoid daemon issues)
+  max_workers: 2
+  # Logging level: DEBUG, INFO, WARNING, ERROR, CRITICAL
+  log_level: "INFO"
+  # If true, show what would be downloaded without actually downloading
+  dry_run: false
+  # Verify that previously downloaded files still exist
+  verify_existing_files: true
+  # Write SHA256 checksums for downloaded files
+  write_checksums: true
+  # Filename handling: 'original' or 'safe' (aggressively sanitized)
+  filename_mode: "original"
+  # Include reposts in downloads (default: false)
+  include_reposts: false
+  # Number of claims to fetch per page from the daemon
+  channel_page_size: 50
+  # Keep records of claims even if they disappear from the channel
+  keep_missing_claim_records: true
+  # Download limit: Number of most recent downloads per channel (0 = all)
+  download_limit: {download_limit}
+
+channels:
+"""
+
+    for channel in channels:
+        config += f'''  - input: "{channel["input"]}"
+    enabled: true
+'''
+
+    if not channels:
+        config += """  # Add your channels here:
+  # - input: "https://odysee.com/@ChannelName:1"
+  #   enabled: true
+"""
+
+    return config
+
+
+def create_launcher_script(install_dir: Path, venv_path: Path) -> None:
+    """Create launcher scripts for easy execution."""
+    print_header("Creating Launcher Scripts")
+
+    # Create bin directory
+    bin_dir = install_dir / "bin"
+    bin_dir.mkdir(exist_ok=True)
+
+    # Create main launcher
+    launcher = bin_dir / "lbry-downloader"
+    launcher_content = f"""#!/bin/bash
+# LBRY Downloader Launcher
+
+INSTALL_DIR="{install_dir}"
+VENV_DIR="{venv_path}"
+
+cd "$INSTALL_DIR"
+source "$VENV_DIR/bin/activate"
+
+python main.py "$@"
+"""
+
+    launcher.write_text(launcher_content)
+    launcher.chmod(0o755)
+    print_success(f"Created launcher: {launcher}")
+
+    # Create quick test script
+    test_script = bin_dir / "lbry-test"
+    test_content = f"""#!/bin/bash
+# Quick test script
+
+INSTALL_DIR="{install_dir}"
+VENV_DIR="{venv_path}"
+
+cd "$INSTALL_DIR"
+source "$VENV_DIR/bin/activate"
+
+echo "Testing LBRY Downloader..."
+python main.py --dry-run "$@"
+"""
+
+    test_script.write_text(test_content)
+    test_script.chmod(0o755)
+    print_success(f"Created test script: {test_script}")
+
+    # Check if we should add to PATH
+    shell_rc = Path.home() / ".bashrc"
+    if not shell_rc.exists():
+        shell_rc = Path.home() / ".zshrc"
+
+    if shell_rc.exists():
+        print_info(
+            f"To use 'lbry-downloader' from anywhere, add this to your {shell_rc.name}:"
+        )
+        print(f'  export PATH="{bin_dir}:$PATH"')
+
+
+def run_setup() -> None:
+    """Main setup function."""
+    print(f"""
+{Colors.BOLD}╔══════════════════════════════════════════════════════════════╗
+║           LBRY Downloader - Interactive Setup               ║
+║                                                              ║
+║  This script will help you install and configure the        ║
+║  LBRY Downloader tool.                                       ║
+╚══════════════════════════════════════════════════════════════╝{Colors.END}
+""")
+
+    # Check Python version
+    if not check_python_version():
+        print_error("Python 3.8+ is required. Please upgrade Python.")
+        sys.exit(1)
+
+    # Check system dependencies
+    if not check_system_dependencies():
+        print_info("Some system dependencies are missing.")
+        response = input("Install system dependencies? (Y/n): ").strip().lower()
+        if response != "n":
+            if not install_system_dependencies():
+                print_warning("Failed to install some dependencies")
+                response = input("Continue anyway? (y/N): ").strip().lower()
+                if response != "y":
+                    sys.exit(1)
+
+    # Get installation directory
+    install_dir = get_installation_directory()
+
+    # Copy files if not running from install directory
+    current_dir = Path(__file__).parent.resolve()
+    if current_dir != install_dir:
+        print_header("Copying Files")
+        print_info(f"Copying from {current_dir} to {install_dir}")
+
+        # Copy Python files
+        for pattern in ["*.py", "*.md", "*.txt", "*.toml", "*.example", "Makefile"]:
+            for file in current_dir.glob(pattern):
+                if file.name != "setup.py":  # Don't copy setup script itself
+                    shutil.copy2(file, install_dir)
+                    print_success(f"Copied {file.name}")
+
+        # Copy tests directory
+        tests_src = current_dir / "tests"
+        tests_dst = install_dir / "tests"
+        if tests_src.exists():
+            if tests_dst.exists():
+                shutil.rmtree(tests_dst)
+            shutil.copytree(tests_src, tests_dst)
+            print_success("Copied tests/")
+
+    # Setup virtual environment
+    venv_path = setup_virtual_environment(install_dir)
+
+    # Install Python dependencies
+    if not install_python_dependencies(venv_path, install_dir):
+        print_error("Failed to install Python dependencies")
+        sys.exit(1)
+
+    # Get configuration from user
+    download_location = get_download_location()
+    channels = get_channels()
+    download_limit = get_download_limit()
+
+    # Create config file
+    print_header("Creating Configuration")
+    config_content = create_config(download_location, channels, download_limit)
+
+    config_path = Path(download_location).expanduser() / "config.yaml"
+    config_path.parent.mkdir(parents=True, exist_ok=True)
+    config_path.write_text(config_content)
+    print_success(f"Created configuration: {config_path}")
+
+    # Create necessary directories
+    (config_path.parent / "state").mkdir(exist_ok=True)
+    (config_path.parent / "channels").mkdir(exist_ok=True)
+    print_success("Created directory structure")
+
+    # Create launcher scripts
+    create_launcher_script(install_dir, venv_path)
+
+    # Final instructions
+    print(f"""
+{Colors.BOLD}{Colors.GREEN}╔══════════════════════════════════════════════════════════════╗
+║                    Setup Complete!                           ║
+╚══════════════════════════════════════════════════════════════╝{Colors.END}
+
+Next steps:
+
+1. Start the LBRY daemon:
+   lbrynet start
+   
+   (If lbrynet is not installed, see LINUX_SETUP.md)
+
+2. Test the configuration:
+   cd {install_dir}
+   ./bin/lbry-test
+   
+   Or run directly:
+   source venv/bin/activate
+   python main.py --dry-run
+
+3. Run the downloader:
+   ./bin/lbry-downloader
+   
+   Or:
+   python main.py
+
+Configuration file:
+   {config_path}
+
+To add more channels later, edit the config file:
+   nano {config_path}
+
+Documentation:
+   - README.md - Project overview
+   - LINUX_SETUP.md - Detailed Linux setup
+   - QUICKSTART.md - Quick start guide
+   - config.yaml.example - Configuration examples
+
+Happy downloading!
+""")
+
+
+if __name__ == "__main__":
+    try:
+        run_setup()
+    except KeyboardInterrupt:
+        print("\n\nSetup interrupted by user.")
+        sys.exit(1)
+    except Exception as e:
+        print_error(f"Setup failed: {e}")
+        import traceback
+
+        traceback.print_exc()
+        sys.exit(1)