Update dependencies and add new features for improved functionality

- Updated Cargo.lock and Cargo.toml to include new dependencies
- Added new files for backup and watcher functionality in libmarlin
- Introduced integration tests and documentation updates
- Set workspace resolver to version 2 for better dependency resolution

cli-bin/Cargo.toml

@@ -13,6 +13,7 @@ libmarlin          = { path = "../libmarlin" }   # ← core library
 anyhow             = "1"
 clap               = { version = "4", features = ["derive"] }
 clap_complete      = "4.1"
+ctrlc              = "3.4"
 glob               = "0.3"
 rusqlite           = { version = "0.31", features = ["bundled", "backup"] }
 shellexpand        = "3.1"

cli-bin/src/cli.rs

@@ -9,6 +9,7 @@ pub mod remind;
 pub mod annotate;
 pub mod version;
 pub mod event;
+pub mod watch;
 
 use clap::{Parser, Subcommand, ValueEnum};
 use clap_complete::Shell;
@@ -123,6 +124,10 @@ pub enum Commands {
     /// Calendar events & timelines
     #[command(subcommand)]
     Event(event::EventCmd),
+
+    /// Watch directories for changes
+    #[command(subcommand)]
+    Watch(watch::WatchCmd),
 }
 
 #[derive(Subcommand, Debug)]

cli-bin/src/cli/watch.rs

@@ -0,0 +1,102 @@
+// src/cli/watch.rs
+
+use anyhow::Result;
+use clap::Subcommand;
+use libmarlin::watcher::{WatcherConfig, WatcherState};
+use rusqlite::Connection;
+use std::path::PathBuf;
+use std::sync::Arc;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::thread;
+use std::time::{Duration, Instant};
+use tracing::info;
+
+/// Commands related to file watching functionality
+#[derive(Subcommand, Debug)]
+pub enum WatchCmd {
+    /// Start watching a directory for changes
+    Start {
+        /// Directory to watch (defaults to current directory)
+        #[arg(default_value = ".")]
+        path: PathBuf,
+        
+        /// Debounce window in milliseconds (default: 100ms)
+        #[arg(long, default_value = "100")]
+        debounce_ms: u64,
+    },
+    
+    /// Show status of currently active watcher
+    Status,
+    
+    /// Stop the currently running watcher
+    Stop,
+}
+
+/// Run a watch command
+pub fn run(cmd: &WatchCmd, _conn: &mut Connection, _format: super::Format) -> Result<()> {
+    match cmd {
+        WatchCmd::Start { path, debounce_ms } => {
+            let mut marlin = libmarlin::Marlin::open_default()?;
+            let config = WatcherConfig {
+                debounce_ms: *debounce_ms,
+                ..Default::default()
+            };
+            let canon_path = path.canonicalize().unwrap_or_else(|_| path.clone());
+            info!("Starting watcher for directory: {}", canon_path.display());
+
+            let mut watcher = marlin.watch(&canon_path, Some(config))?;
+            
+            let status = watcher.status();
+            info!("Watcher started. Press Ctrl+C to stop watching.");
+            info!("Watching {} paths", status.watched_paths.len());
+            
+            let start_time = Instant::now();
+            let mut last_status_time = Instant::now();
+            let running = Arc::new(AtomicBool::new(true));
+            let r_clone = running.clone();
+
+            ctrlc::set_handler(move || {
+                info!("Ctrl+C received. Signaling watcher to stop...");
+                r_clone.store(false, Ordering::SeqCst);
+            })?;
+
+            info!("Watcher run loop started. Waiting for Ctrl+C or stop signal...");
+            while running.load(Ordering::SeqCst) {
+                let current_status = watcher.status();
+                if current_status.state == WatcherState::Stopped {
+                    info!("Watcher has stopped (detected by state). Exiting loop.");
+                    break;
+                }
+
+                // Corrected line: removed the extra closing parenthesis
+                if last_status_time.elapsed() > Duration::from_secs(10) { 
+                    let uptime = start_time.elapsed();
+                    info!(
+                        "Watcher running for {}s, processed {} events, queue: {}, state: {:?}",
+                        uptime.as_secs(),
+                        current_status.events_processed,
+                        current_status.queue_size,
+                        current_status.state
+                    );
+                    last_status_time = Instant::now();
+                }
+                thread::sleep(Duration::from_millis(200));
+            }
+
+            info!("Watcher run loop ended. Explicitly stopping watcher instance...");
+            watcher.stop()?; 
+            info!("Watcher instance fully stopped.");
+            Ok(())
+        }
+        WatchCmd::Status => {
+            info!("Status command: No active watcher process to query in this CLI invocation model.");
+            info!("To see live status, run 'marlin watch start' which prints periodic updates.");
+            Ok(())
+        }
+        WatchCmd::Stop => {
+            info!("Stop command: No active watcher process to stop in this CLI invocation model.");
+            info!("Please use Ctrl+C in the terminal where 'marlin watch start' is running.");
+            Ok(())
+        }
+    }
+}

cli-bin/src/main.rs

@@ -154,6 +154,7 @@ fn main() -> Result<()> {
         Commands::Annotate(a_cmd)  => cli::annotate::run(&a_cmd,  &mut conn, args.format)?,
         Commands::Version(v_cmd)   => cli::version::run(&v_cmd,   &mut conn, args.format)?,
         Commands::Event(e_cmd)     => cli::event::run(&e_cmd,     &mut conn, args.format)?,
+        Commands::Watch(watch_cmd) => cli::watch::run(&watch_cmd, &mut conn, args.format)?,
     }
 
     Ok(())

cli-bin/tests/integration/watcher/watcher_test.rs

@@ -0,0 +1,364 @@
+//! Integration test for the file watcher functionality
+//! 
+//! Tests various aspects of the file system watcher including:
+//! - Basic event handling (create, modify, delete files)
+//! - Debouncing of events
+//! - Hierarchical event coalescing
+//! - Graceful shutdown and event draining
+
+use marlin::watcher::{FileWatcher, WatcherConfig, WatcherState};
+use std::path::{Path, PathBuf};
+use std::fs::{self, File};
+use std::io::Write;
+use std::thread;
+use std::time::{Duration, Instant};
+use tempfile::tempdir;
+
+// Mock filesystem event simulator inspired by inotify-sim
+struct MockEventSimulator {
+    temp_dir: PathBuf,
+    files_created: Vec<PathBuf>,
+}
+
+impl MockEventSimulator {
+    fn new(temp_dir: PathBuf) -> Self {
+        Self {
+            temp_dir,
+            files_created: Vec::new(),
+        }
+    }
+
+    fn create_file(&mut self, relative_path: &str, content: &str) -> PathBuf {
+        let path = self.temp_dir.join(relative_path);
+        if let Some(parent) = path.parent() {
+            fs::create_dir_all(parent).expect("Failed to create parent directory");
+        }
+        
+        let mut file = File::create(&path).expect("Failed to create file");
+        file.write_all(content.as_bytes()).expect("Failed to write content");
+        
+        self.files_created.push(path.clone());
+        path
+    }
+    
+    fn modify_file(&self, relative_path: &str, new_content: &str) -> PathBuf {
+        let path = self.temp_dir.join(relative_path);
+        let mut file = File::create(&path).expect("Failed to update file");
+        file.write_all(new_content.as_bytes()).expect("Failed to write content");
+        path
+    }
+    
+    fn delete_file(&mut self, relative_path: &str) {
+        let path = self.temp_dir.join(relative_path);
+        fs::remove_file(&path).expect("Failed to delete file");
+        
+        self.files_created.retain(|p| p != &path);
+    }
+    
+    fn create_burst(&mut self, count: usize, prefix: &str) -> Vec<PathBuf> {
+        let mut paths = Vec::with_capacity(count);
+        
+        for i in 0..count {
+            let file_path = format!("{}/burst_file_{}.txt", prefix, i);
+            let path = self.create_file(&file_path, &format!("Content {}", i));
+            paths.push(path);
+            
+            // Small delay to simulate rapid but not instantaneous file creation
+            thread::sleep(Duration::from_micros(10));
+        }
+        
+        paths
+    }
+    
+    fn cleanup(&self) {
+        // No need to do anything as tempdir will clean itself
+    }
+}
+
+#[test]
+fn test_basic_watch_functionality() {
+    let temp_dir = tempdir().expect("Failed to create temp directory");
+    let temp_path = temp_dir.path().to_path_buf();
+    
+    let mut simulator = MockEventSimulator::new(temp_path.clone());
+    
+    // Create a test file before starting the watcher
+    let initial_file = simulator.create_file("initial.txt", "Initial content");
+    
+    // Configure and start the watcher
+    let config = WatcherConfig {
+        debounce_ms: 100,
+        batch_size: 100,
+        max_queue_size: 1000,
+        drain_timeout_ms: 1000,
+    };
+    
+    let mut watcher = FileWatcher::new(vec![temp_path.clone()], config)
+        .expect("Failed to create file watcher");
+    
+    // Start the watcher in a separate thread
+    let watcher_thread = thread::spawn(move || {
+        watcher.start().expect("Failed to start watcher");
+        
+        // Let it run for a short time
+        thread::sleep(Duration::from_secs(5));
+        
+        // Stop the watcher
+        watcher.stop().expect("Failed to stop watcher");
+        
+        // Return the watcher for inspection
+        watcher
+    });
+    
+    // Wait for watcher to initialize
+    thread::sleep(Duration::from_millis(500));
+    
+    // Generate events
+    let file1 = simulator.create_file("test1.txt", "Hello, world!");
+    thread::sleep(Duration::from_millis(200));
+    
+    let file2 = simulator.create_file("dir1/test2.txt", "Hello from subdirectory!");
+    thread::sleep(Duration::from_millis(200));
+    
+    simulator.modify_file("test1.txt", "Updated content");
+    thread::sleep(Duration::from_millis(200));
+    
+    simulator.delete_file("test1.txt");
+    
+    // Wait for watcher thread to complete
+    let finished_watcher = watcher_thread.join().expect("Watcher thread panicked");
+    
+    // Check status after processing events
+    let status = finished_watcher.status();
+    
+    // Assertions
+    assert_eq!(status.state, WatcherState::Stopped);
+    assert!(status.events_processed > 0, "Expected events to be processed");
+    assert_eq!(status.queue_size, 0, "Expected empty queue after stopping");
+    
+    // Clean up
+    simulator.cleanup();
+}
+
+#[test]
+fn test_debouncing() {
+    let temp_dir = tempdir().expect("Failed to create temp directory");
+    let temp_path = temp_dir.path().to_path_buf();
+    
+    let mut simulator = MockEventSimulator::new(temp_path.clone());
+    
+    // Configure watcher with larger debounce window for this test
+    let config = WatcherConfig {
+        debounce_ms: 200,  // 200ms debounce window
+        batch_size: 100,
+        max_queue_size: 1000,
+        drain_timeout_ms: 1000,
+    };
+    
+    let mut watcher = FileWatcher::new(vec![temp_path.clone()], config)
+        .expect("Failed to create file watcher");
+    
+    // Start the watcher in a separate thread
+    let watcher_thread = thread::spawn(move || {
+        watcher.start().expect("Failed to start watcher");
+        
+        // Let it run for enough time to observe debouncing
+        thread::sleep(Duration::from_secs(3));
+        
+        // Stop the watcher
+        watcher.stop().expect("Failed to stop watcher");
+        
+        // Return the watcher for inspection
+        watcher
+    });
+    
+    // Wait for watcher to initialize
+    thread::sleep(Duration::from_millis(500));
+    
+    // Rapidly update the same file multiple times within the debounce window
+    let test_file = "test_debounce.txt";
+    simulator.create_file(test_file, "Initial content");
+    
+    // Update the same file multiple times within debounce window
+    for i in 1..10 {
+        simulator.modify_file(test_file, &format!("Update {}", i));
+        thread::sleep(Duration::from_millis(10)); // Short delay between updates
+    }
+    
+    // Wait for debounce window and processing
+    thread::sleep(Duration::from_millis(500));
+    
+    // Complete the test
+    let finished_watcher = watcher_thread.join().expect("Watcher thread panicked");
+    let status = finished_watcher.status();
+    
+    // We should have processed fewer events than modifications made
+    // due to debouncing (exact count depends on implementation details)
+    assert!(status.events_processed < 10, 
+            "Expected fewer events processed than modifications due to debouncing");
+    
+    // Clean up
+    simulator.cleanup();
+}
+
+#[test]
+fn test_event_flood() {
+    let temp_dir = tempdir().expect("Failed to create temp directory");
+    let temp_path = temp_dir.path().to_path_buf();
+    
+    let mut simulator = MockEventSimulator::new(temp_path.clone());
+    
+    // Configure with settings tuned for burst handling
+    let config = WatcherConfig {
+        debounce_ms: 100,
+        batch_size: 500,  // Handle larger batches
+        max_queue_size: 10000,  // Large queue for burst
+        drain_timeout_ms: 5000, // Longer drain time for cleanup
+    };
+    
+    let mut watcher = FileWatcher::new(vec![temp_path.clone()], config)
+        .expect("Failed to create file watcher");
+    
+    // Start the watcher
+    let watcher_thread = thread::spawn(move || {
+        watcher.start().expect("Failed to start watcher");
+        
+        // Let it run for enough time to process a large burst
+        thread::sleep(Duration::from_secs(10));
+        
+        // Stop the watcher
+        watcher.stop().expect("Failed to stop watcher");
+        
+        // Return the watcher for inspection
+        watcher
+    });
+    
+    // Wait for watcher to initialize
+    thread::sleep(Duration::from_millis(500));
+    
+    // Create 1000 files in rapid succession (smaller scale for test)
+    let start_time = Instant::now();
+    let created_files = simulator.create_burst(1000, "flood");
+    let creation_time = start_time.elapsed();
+    
+    println!("Created 1000 files in {:?}", creation_time);
+    
+    // Wait for processing to complete
+    thread::sleep(Duration::from_secs(5));
+    
+    // Complete the test
+    let finished_watcher = watcher_thread.join().expect("Watcher thread panicked");
+    let status = finished_watcher.status();
+    
+    // Verify processing occurred
+    assert!(status.events_processed > 0, "Expected events to be processed");
+    assert_eq!(status.queue_size, 0, "Expected empty queue after stopping");
+    
+    // Clean up
+    simulator.cleanup();
+}
+
+#[test]
+fn test_hierarchical_debouncing() {
+    let temp_dir = tempdir().expect("Failed to create temp directory");
+    let temp_path = temp_dir.path().to_path_buf();
+    
+    let mut simulator = MockEventSimulator::new(temp_path.clone());
+    
+    // Configure watcher
+    let config = WatcherConfig {
+        debounce_ms: 200,
+        batch_size: 100,
+        max_queue_size: 1000,
+        drain_timeout_ms: 1000,
+    };
+    
+    let mut watcher = FileWatcher::new(vec![temp_path.clone()], config)
+        .expect("Failed to create file watcher");
+    
+    // Start the watcher
+    let watcher_thread = thread::spawn(move || {
+        watcher.start().expect("Failed to start watcher");
+        
+        // Let it run
+        thread::sleep(Duration::from_secs(5));
+        
+        // Stop the watcher
+        watcher.stop().expect("Failed to stop watcher");
+        
+        // Return the watcher
+        watcher
+    });
+    
+    // Wait for watcher to initialize
+    thread::sleep(Duration::from_millis(500));
+    
+    // Create directory structure
+    let nested_dir = "parent/child/grandchild";
+    fs::create_dir_all(temp_path.join(nested_dir)).expect("Failed to create nested directories");
+    
+    // Create files in the hierarchy
+    simulator.create_file("parent/file1.txt", "Content 1");
+    simulator.create_file("parent/child/file2.txt", "Content 2");
+    simulator.create_file("parent/child/grandchild/file3.txt", "Content 3");
+    
+    // Wait a bit
+    thread::sleep(Duration::from_millis(300));
+    
+    // Complete the test
+    let finished_watcher = watcher_thread.join().expect("Watcher thread panicked");
+    
+    // Clean up
+    simulator.cleanup();
+}
+
+#[test]
+fn test_graceful_shutdown() {
+    let temp_dir = tempdir().expect("Failed to create temp directory");
+    let temp_path = temp_dir.path().to_path_buf();
+    
+    let mut simulator = MockEventSimulator::new(temp_path.clone());
+    
+    // Configure watcher with specific drain timeout
+    let config = WatcherConfig {
+        debounce_ms: 100,
+        batch_size: 100,
+        max_queue_size: 1000,
+        drain_timeout_ms: 2000, // 2 second drain timeout
+    };
+    
+    let mut watcher = FileWatcher::new(vec![temp_path.clone()], config)
+        .expect("Failed to create file watcher");
+    
+    // Start the watcher
+    watcher.start().expect("Failed to start watcher");
+    
+    // Wait for initialization
+    thread::sleep(Duration::from_millis(500));
+    
+    // Create files
+    for i in 0..10 {
+        simulator.create_file(&format!("shutdown_test_{}.txt", i), "Shutdown test");
+        thread::sleep(Duration::from_millis(10));
+    }
+    
+    // Immediately request shutdown while events are being processed
+    let shutdown_start = Instant::now();
+    watcher.stop().expect("Failed to stop watcher");
+    let shutdown_duration = shutdown_start.elapsed();
+    
+    // Shutdown should take close to the drain timeout but not excessively longer
+    println!("Shutdown took {:?}", shutdown_duration);
+    assert!(shutdown_duration >= Duration::from_millis(100), 
+            "Shutdown was too quick, may not have drained properly");
+    assert!(shutdown_duration <= Duration::from_millis(3000), 
+            "Shutdown took too long");
+    
+    // Verify final state
+    let status = watcher.status();
+    assert_eq!(status.state, WatcherState::Stopped);
+    assert_eq!(status.queue_size, 0, "Queue should be empty after shutdown");
+    
+    // Clean up
+    simulator.cleanup();
+}

cli-bin/tests/test.md

@@ -65,4 +65,10 @@ sudo install -Dm755 target/release/marlin /usr/local/bin/marlin &&
 cargo test --all -- --nocapture
 ```
 
+or
+
+```bash
+./run_all_tests.sh
+```
+
 Stick that in a shell alias (`alias marlin-ci='…'`) and you’ve got a 5-second upgrade-and-verify loop.