Add refresh_data script and startup script for offline data setup

- Introduced `scripts/refresh_data.py` to download UNRATE and WTI data from FRED and store it in a SQLite database.
- Added `startup.sh` script to set up a Python virtual environment and install dependencies for offline use.
- Updated `repo-context.txt` to reflect the new scripts and their purposes.

repo-context.txt

@@ -9,8 +9,10 @@ Generated on: 2025-05-30
     ├── scripts/
         ├── lagged_oil_unrate_chart_styled.py
         ├── pyproject.toml
+        ├── refresh_data.py
     ├── README.md
     ├── requirements.txt
+    ├── startup.sh
 ```
 
 ## Important Files
@@ -143,13 +145,69 @@ This project is released under the MIT License.
 ## python_charts/requirements.txt
 ```
 pandas>=1.5.0
-pandas_datareader>=0.10.0    # good
+pandas_datareader>=0.10.0
 matplotlib>=3.5.0
 lxml>=4.6.0
 python-dateutil>=2.8.1
 
 ```
 
+## python_charts/startup.sh
+```
+#!/usr/bin/env bash
+set -euo pipefail
+
+python3 -m venv venv
+source venv/bin/activate
+
+pip install --no-index --find-links=wheelhouse -r requirements.txt
+
+echo "✅  Virtual-env ready (offline install)."
+
+```
+
+## scripts/refresh_data.py
+```python
+#!/usr/bin/env python3
+"""
+refresh_data.py
+---------------
+Download UNRATE and DCOILWTICO from FRED and store them in SQLite
+(data/fred.db).  Run on a connected machine; commit the DB afterwards.
+"""
+
+from datetime import date
+import pathlib
+import sqlite3
+
+import pandas as pd
+from pandas_datareader.data import DataReader
+
+HERE = pathlib.Path(__file__).resolve().parent
+DATA_PATH = HERE.parent / "data"
+DB_FILE = DATA_PATH / "fred.db"
+
+START = "1948-01-01"          # earliest UNRATE observation
+END = date.today().isoformat()
+SERIES = ("UNRATE", "DCOILWTICO")
+
+
+def main() -> None:
+    DATA_PATH.mkdir(exist_ok=True)
+    with sqlite3.connect(DB_FILE) as conn:
+        for s in SERIES:
+            print(f"↯ downloading {s} …")
+            df = DataReader(s, "fred", START, END)
+            df.to_sql(s, conn, if_exists="replace", index_label="date")
+            print(f"✓ wrote {s}: {len(df):,} rows")
+    print(f"\nDone.  SQLite DB at {DB_FILE} ({DB_FILE.stat().st_size/1024:.0f} KB)")
+
+
+if __name__ == "__main__":
+    main()
+
+```
+
 ## scripts/pyproject.toml
 ```
 # pyproject.toml
@@ -175,122 +233,205 @@ dependencies = [
 """
 lagged_oil_unrate_chart_styled.py
 ---------------------------------
-Fetch and plot UNRATE vs. WTI oil price with a configurable lag,
-date range, and polished dual-axis styling.
+Offline version: reads UNRATE & WTI series from data/fred.db (populated
+by scripts/refresh_data.py) and plots them with a configurable lag.
 
-Usage:
-    python lagged_oil_unrate_chart_styled.py \
-        --offset 18 \
-        --start 1973-01-01 \
-        --end 2025-05-31 \
-        --extend-years 3
-
-Dependencies:
-    pip install pandas pandas_datareader matplotlib python-dateutil
+Usage
+-----
+  python scripts/lagged_oil_unrate_chart_styled.py \
+      --offset 18 \
+      --start 1973-01-01 \
+      --end   2025-05-31 \
+      --extend-years 3
 """
+
+from __future__ import annotations
+
 import argparse
 from datetime import datetime
-from dateutil.relativedelta import relativedelta
+from pathlib import Path
 
 import matplotlib.dates as mdates
 import matplotlib.pyplot as plt
 import pandas as pd
-import pandas_datareader.data as web
+from dateutil.relativedelta import relativedelta
+import sqlite3
 
 
-def fetch_series(start, end):
-    # FRED series
-    unrate = web.DataReader("UNRATE", "fred", start, end)
-    oil   = web.DataReader("DCOILWTICO", "fred", start, end)
+# ──────────────────────────────────────────────────────────────────────────
+# Data helpers
+# ──────────────────────────────────────────────────────────────────────────
+def fetch_series(
+    start: datetime,
+    end: datetime,
+    db_path: str | Path = Path("data/fred.db"),
+) -> tuple[pd.DataFrame, pd.DataFrame]:
+    """
+    Load UNRATE and DCOILWTICO from a local SQLite DB, then:
+
+    * convert UNRATE to month-end timestamps
+    * average WTI daily prices to month-end
+    """
+    db_path = Path(db_path)
+    if not db_path.exists():
+        raise FileNotFoundError(
+            f"{db_path} not found.  Run scripts/refresh_data.py on a "
+            "machine with internet, commit the new DB, then retry."
+        )
+
+    with sqlite3.connect(db_path) as conn:
+        unrate = (
+            pd.read_sql(
+                "SELECT date, UNRATE AS value FROM UNRATE", conn,
+                parse_dates=["date"], index_col="date"
+            )
+            .loc[start:end]
+        )
+        oil = (
+            pd.read_sql(
+                "SELECT date, DCOILWTICO AS value FROM DCOILWTICO", conn,
+                parse_dates=["date"], index_col="date"
+            )
+            .loc[start:end]
+        )
 
     # Resample WTI to month-end average
     oil_monthly = oil.resample("M").mean()
-    # Ensure UNRATE is at month-end timestamps, too
+    # Ensure UNRATE is at month-end, too
     unrate.index = unrate.index.to_period("M").to_timestamp("M")
     return unrate, oil_monthly
 
 
+# ──────────────────────────────────────────────────────────────────────────
+# Plotting
+# ──────────────────────────────────────────────────────────────────────────
 def plot_lagged(
     unrate: pd.DataFrame,
     oil: pd.DataFrame,
     offset_months: int,
     start_date: datetime,
     end_date: datetime,
-    extend_years: int
-):
-    # Shift oil by offset: positive = lag; negative = lead
+    extend_years: int,
+) -> None:
+    """Render the dual-axis chart."""
     oil_shifted = oil.shift(offset_months)
 
-    # Build figure
     fig, ax1 = plt.subplots(figsize=(14, 7))
     plt.title("Unemployment Rate and US Oil Price", fontsize=20, weight="bold")
     plt.suptitle(
         "Civilian Unemployment Rate and WTI Crude Oil Price",
         fontsize=14,
-        y=0.93
+        y=0.93,
     )
 
-    # Plot UNRATE on left axis
-    ax1.plot(unrate.index, unrate["UNRATE"], 
-             label="UNRATE (LHS)", linewidth=2.5, color="#1f77b4")
+    # Left axis – UNRATE
+    ax1.plot(
+        unrate.index,
+        unrate["value"],
+        label="UNRATE (LHS)",
+        linewidth=2.5,
+        color="#1f77b4",
+    )
     ax1.set_ylabel("Unemployment rate (%)", fontsize=12)
-    ax1.set_ylim(0, unrate["UNRATE"].max() * 1.1)
+    ax1.set_ylim(0, unrate["value"].max() * 1.1)
     ax1.yaxis.set_major_formatter("{x:.0f}%")
     ax1.grid(axis="y", linestyle="--", alpha=0.4)
 
-    # Plot shifted oil on right axis
+    # Right axis – shifted oil
     ax2 = ax1.twinx()
-    ax2.plot(oil_shifted.index, oil_shifted["DCOILWTICO"],
-             label="USOIL (RHS)", linewidth=2, linestyle="-", alpha=0.8, color="#FF9900")
+    ax2.plot(
+        oil_shifted.index,
+        oil_shifted["value"],
+        label="USOIL (RHS)",
+        linewidth=2,
+        linestyle="-",
+        alpha=0.8,
+        color="#FF9900",
+    )
     ax2.set_ylabel("Oil price (USD)", fontsize=12)
-    ax2.set_ylim(0, oil_shifted["DCOILWTICO"].max() * 1.1)
+    ax2.set_ylim(0, oil_shifted["value"].max() * 1.1)
     ax2.yaxis.set_major_formatter("${x:,.0f}")
 
-    # X-axis formatting
+    # X-axis
     ax1.set_xlim(start_date, end_date + relativedelta(years=extend_years))
     ax1.xaxis.set_major_locator(mdates.YearLocator(base=5))
     ax1.xaxis.set_minor_locator(mdates.YearLocator(1))
     ax1.xaxis.set_major_formatter(mdates.DateFormatter("%Y"))
     plt.setp(ax1.get_xticklabels(), rotation=0, fontsize=11)
 
-    # Combine legends
+    # Legends
     lines, labels = ax1.get_legend_handles_labels()
     l2, l2l = ax2.get_legend_handles_labels()
     ax1.legend(lines + l2, labels + l2l, loc="upper left", frameon=False, fontsize=12)
 
-    # Footnotes
+    # Footnote
     foot = (
-        f"Dates: {start_date.strftime('%Y')} through {unrate.index[-1].strftime('%d %b %Y')}.\n"
-        "Source: FRED, pandas_datareader."
+        f"Dates: {start_date.strftime('%Y')} through "
+        f"{unrate.index[-1].strftime('%d %b %Y')}.\n"
+        "Source: Local FRED snapshot (offline)."
     )
     plt.annotate(
         foot,
-        (0,0), (0, -40), 
-        xycoords="axes fraction", textcoords="offset points",
-        va="top", fontsize=10, color="gray"
+        (0, 0),
+        (0, -40),
+        xycoords="axes fraction",
+        textcoords="offset points",
+        va="top",
+        fontsize=10,
+        color="gray",
     )
 
     fig.tight_layout(rect=[0, 0.03, 1, 0.95])
     plt.show()
 
 
-if __name__ == "__main__":
+# ──────────────────────────────────────────────────────────────────────────
+# CLI
+# ──────────────────────────────────────────────────────────────────────────
+def main() -> None:
     p = argparse.ArgumentParser()
-    p.add_argument("-o", "--offset", type=int, default=18,
-                   help="lag in months (positive=lag, negative=lead)")
-    p.add_argument("--start", type=str, default="1973-01-01",
-                   help="series start date (YYYY-MM-DD)")
-    p.add_argument("--end", type=str, default=datetime.today().strftime("%Y-%m-%d"),
-                   help="series end date (YYYY-MM-DD)")
-    p.add_argument("--extend-years", type=int, default=3,
-                   help="how many years beyond end_date to show on the x-axis")
+    p.add_argument(
+        "-o",
+        "--offset",
+        type=int,
+        default=18,
+        help="lag in months (positive=lag, negative=lead)",
+    )
+    p.add_argument(
+        "--start",
+        type=str,
+        default="1973-01-01",
+        help="series start date (YYYY-MM-DD)",
+    )
+    p.add_argument(
+        "--end",
+        type=str,
+        default=datetime.today().strftime("%Y-%m-%d"),
+        help="series end date (YYYY-MM-DD)",
+    )
+    p.add_argument(
+        "--extend-years",
+        type=int,
+        default=3,
+        help="how many years beyond end_date to show on the x-axis",
+    )
+    p.add_argument(
+        "--db",
+        type=str,
+        default="data/fred.db",
+        help="path to local FRED SQLite file",
+    )
     args = p.parse_args()
 
     start_dt = datetime.fromisoformat(args.start)
-    end_dt   = datetime.fromisoformat(args.end)
+    end_dt = datetime.fromisoformat(args.end)
 
-    unrate, oil = fetch_series(start_dt, end_dt)
+    unrate, oil = fetch_series(start_dt, end_dt, args.db)
     plot_lagged(unrate, oil, args.offset, start_dt, end_dt, args.extend_years)
 
+
+if __name__ == "__main__":
+    main()
+
 ```
 

scripts/refresh_data.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+"""
+refresh_data.py
+---------------
+Download UNRATE and DCOILWTICO from FRED and store them in SQLite
+(data/fred.db).  Run on a connected machine; commit the DB afterwards.
+"""
+
+from datetime import date
+import pathlib
+import sqlite3
+
+import pandas as pd
+from pandas_datareader.data import DataReader
+
+HERE = pathlib.Path(__file__).resolve().parent
+DATA_PATH = HERE.parent / "data"
+DB_FILE = DATA_PATH / "fred.db"
+
+START = "1948-01-01"          # earliest UNRATE observation
+END = date.today().isoformat()
+SERIES = ("UNRATE", "DCOILWTICO")
+
+
+def main() -> None:
+    DATA_PATH.mkdir(exist_ok=True)
+    with sqlite3.connect(DB_FILE) as conn:
+        for s in SERIES:
+            print(f"↯ downloading {s} …")
+            df = DataReader(s, "fred", START, END)
+            df.to_sql(s, conn, if_exists="replace", index_label="date")
+            print(f"✓ wrote {s}: {len(df):,} rows")
+    print(f"\nDone.  SQLite DB at {DB_FILE} ({DB_FILE.stat().st_size/1024:.0f} KB)")
+
+
+if __name__ == "__main__":
+    main()

startup.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+python3 -m venv venv
+source venv/bin/activate
+
+pip install --no-index --find-links=wheelhouse -r requirements.txt
+
+echo "✅  Virtual-env ready (offline install)."