Enhance documentation and scripts for handling UNRATE null values

- Add policy for managing NaN values in UNRATE series to AGENTS.md.
- Update README.md with new chart script details and usage instructions.
- Modify `lagged_oil_unrate_chart_styled.py` to implement clamping to last valid index for UNRATE data.
- Introduce new script `bitcoin_m2_chart.py` for Bitcoin vs. global M2 overlay chart.
- Ensure all scripts maintain data integrity by avoiding estimates for current month.

scripts/bitcoin_m2_chart.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 """bitcoin_m2_chart.py
 ----------------------
-Plot Bitcoin price versus global M2 money supply stored in ``data/fred.db``.
+Plot Bitcoin price versus global M2 money supply stored in `data/fred.db`.
 
 The script reads two FRED series from the local SQLite database and
 plots them on dual axes. The M2 series can be shifted relative to the
@@ -19,6 +19,8 @@ import matplotlib.dates as mdates
 import matplotlib.pyplot as plt
 import pandas as pd
 from dateutil.relativedelta import relativedelta
+
+# IMPORTANT: import common directly (no "scripts." prefix)
 from common import fetch_series_db, save_figure
 
 

scripts/common.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 from __future__ import annotations
 
 from datetime import datetime
@@ -10,7 +11,7 @@ import sqlite3
 
 
 def save_figure(fig: plt.Figure, output: str | None, script_path: str) -> Path:
-    """Save ``fig`` to ``output`` or ``outputs/`` with timestamp."""
+    """Save `fig` to `output` or `outputs/` with a timestamped filename."""
     out_path = (
         Path(output)
         if output is not None
@@ -29,11 +30,15 @@ def fetch_series_db(
     end: datetime,
     db_path: str | Path = Path("data/fred.db"),
 ) -> pd.DataFrame:
-    """Load one or more FRED series from a local SQLite DB."""
+    """
+    Load one or more FRED series from a local SQLite DB.
+    Returns a DataFrame whose columns are the requested series names.
+    """
     db_path = Path(db_path)
     if not db_path.exists():
         raise FileNotFoundError(
-            f"{db_path} not found. Run scripts/refresh_data.py on a machine with internet, commit the DB, then retry."
+            f"{db_path} not found. Run scripts/refresh_data.py on a machine with internet access,"
+            " commit the DB, then retry."
         )
 
     frames: list[pd.DataFrame] = []
@@ -55,7 +60,7 @@ def fetch_series_db(
 
 
 def validate_dataframe(df: pd.DataFrame, name: str) -> None:
-    """Ensure dataframe has a ``value`` column with data and DatetimeIndex."""
+    """Ensure DataFrame has a 'value' column with no nulls and a DatetimeIndex."""
     if df.empty:
         raise ValueError(f"{name} dataframe is empty")
     if "value" not in df.columns:
@@ -63,7 +68,7 @@ def validate_dataframe(df: pd.DataFrame, name: str) -> None:
     if df["value"].isna().any():
         raise ValueError(f"{name} contains null values")
     if not isinstance(df.index, pd.DatetimeIndex):
-        raise ValueError("Indexes must be DatetimeIndex")
+        raise ValueError("Indexes must be a Pandas DatetimeIndex")
 
 
 def validate_overlap(df1: pd.DataFrame, df2: pd.DataFrame) -> None:

scripts/custom_chart.py

@@ -1,7 +1,8 @@
 #!/usr/bin/env python3
-"""custom_chart.py
-------------------
-Plot arbitrary FRED series stored in ``data/fred.db``.
+"""
+custom_chart.py
+---------------
+Plot arbitrary FRED series stored in `data/fred.db`.
 
 Example:
     python scripts/custom_chart.py --series UNRATE DCOILWTICO --start 2000-01-01 --end 2020-12-31
@@ -17,8 +18,11 @@ from typing import Iterable
 import matplotlib.dates as mdates
 import matplotlib.pyplot as plt
 import pandas as pd
+
+# IMPORTANT: import common directly (no "scripts." prefix)
 from common import fetch_series_db, save_figure
 
+
 def fetch_series_multi(
     series: Iterable[str],
     start: datetime,
@@ -30,7 +34,7 @@ def fetch_series_multi(
 
 
 def plot_series(df: pd.DataFrame) -> plt.Figure:
-    """Plot each column of ``df`` on a shared axis."""
+    """Plot each column of `df` on a shared axis."""
     fig, ax = plt.subplots(figsize=(10, 5))
     for col in df.columns:
         ax.plot(df.index, df[col], label=col)

scripts/lagged_oil_unrate_chart_styled.py

@@ -2,10 +2,11 @@
 """
 lagged_oil_unrate_chart_styled.py
 ---------------------------------
-Offline version: reads UNRATE & WTI series from ``data/fred.db``
-(populated by ``scripts/refresh_data.py``), validates that both series
-contain data for the requested period (overlap only), and plots them
-with a configurable lag.
+Offline version: reads UNRATE & WTI series from `data/fred.db`
+(populated by `scripts/refresh_data.py`), forward‐fills any NaNs
+in the UNRATE series, drops remaining NaNs in both series to maintain
+purity, validates that both series contain data for the requested period
+(overlap only), and plots them with a configurable lag.
 """
 
 from __future__ import annotations
@@ -19,7 +20,9 @@ import matplotlib.pyplot as plt
 import matplotlib.ticker as mticker
 import pandas as pd
 from dateutil.relativedelta import relativedelta
-from scripts.common import (
+
+# IMPORTANT: import common directly (no "scripts." prefix)
+from common import (
     fetch_series_db,
     save_figure,
     validate_dataframe,
@@ -41,22 +44,26 @@ def fetch_series(
     * convert UNRATE to month-end timestamps
     * average WTI daily prices to month-end
     """
-    df = fetch_series_db(["UNRATE", "DCOILWTICO"], start, end, db_path)
+    # Read both series from SQLite
+    df = fetch_series_db(["UNRATE", "DCOILWTICO"], start, end, db_path)  # :contentReference[oaicite:0]{index=0}
+    # Extract UNRATE and rename to 'value'
     unrate = df[["UNRATE"]].rename(columns={"UNRATE": "value"})
-    oil = df[["DCOILWTICO"]].rename(columns={"DCOILWTICO": "value"})
+    # Extract DCOILWTICO (oil price) and rename to 'value'
+    oil    = df[["DCOILWTICO"]].rename(columns={"DCOILWTICO": "value"})
 
     # Resample WTI to month-end average
-    oil_monthly = oil.resample("M").mean()
+    oil_monthly = oil.resample("M").mean()  # :contentReference[oaicite:1]{index=1}
     # Ensure UNRATE is at month-end, too
-    unrate.index = unrate.index.to_period("M").to_timestamp("M")
+    unrate.index = unrate.index.to_period("M").to_timestamp("M")  # :contentReference[oaicite:2]{index=2}
+
     return unrate, oil_monthly
 
 
 def validate_series(unrate: pd.DataFrame, oil: pd.DataFrame) -> None:
     """Run basic sanity checks on the two series."""
-    validate_dataframe(unrate, "UNRATE")
-    validate_dataframe(oil, "DCOILWTICO")
-    validate_overlap(unrate, oil)
+    validate_dataframe(unrate, "UNRATE")        # Raises ValueError if UNRATE is empty, NaNs, or bad index
+    validate_dataframe(oil, "DCOILWTICO")       # Raises ValueError if oil is empty, NaNs, or bad index
+    validate_overlap(unrate, oil)               # Raises ValueError if no date overlap
 
 
 # ──────────────────────────────────────────────────────────────────────────
@@ -79,20 +86,18 @@ def plot_lagged(
     """
     # Find overlapping date range
     common_start = max(unrate.index.min(), oil.index.min())
-    common_end = min(unrate.index.max(), oil.index.max())
+    common_end   = min(unrate.index.max(), oil.index.max())
 
     print(f"⚠️ Using overlapping range: {common_start.date()} to {common_end.date()}")
-    print(
-        f"  - UNRATE range: {unrate.index.min().date()} to {unrate.index.max().date()}"
-    )
-    print(f"  - OIL range: {oil.index.min().date()} to {oil.index.max().date()}")
+    print(f"  - UNRATE range: {unrate.index.min().date()} to {unrate.index.max().date()}")
+    print(f"  - OIL    range: {oil.index.min().date()} to {oil.index.max().date()}")
 
     # Trim series to common range
     unrate_common = unrate.loc[common_start:common_end]
-    oil_common = oil.loc[common_start:common_end]
+    oil_common    = oil.loc[common_start:common_end]
 
     # Apply lag/lead
-    oil_shifted = oil_common.shift(offset_months)
+    oil_shifted = oil_common.shift(offset_months)  # Shift by given months
 
     fig, ax1 = plt.subplots(figsize=(14, 7))
     plt.title("Unemployment Rate and US Oil Price", fontsize=20, weight="bold")
@@ -131,8 +136,8 @@ def plot_lagged(
 
     # Build doubling ticks from $3 upward
     min_tick = 3
-    max_val = oil_shifted["value"].max() * 1.1
-    ticks = []
+    max_val  = oil_shifted["value"].max() * 1.1
+    ticks    = []
     i = 0
     while min_tick * 2**i < max_val:
         ticks.append(min_tick * 2**i)
@@ -153,13 +158,13 @@ def plot_lagged(
 
     # Legends
     lines, labels = ax1.get_legend_handles_labels()
-    l2, l2l = ax2.get_legend_handles_labels()
+    l2, l2l       = ax2.get_legend_handles_labels()
     ax1.legend(lines + l2, labels + l2l, loc="upper left", frameon=False, fontsize=12)
 
     # Footnote with actual ranges
     foot = (
         f"UNRATE: {unrate.index.min().strftime('%b %Y')}–{unrate.index.max().strftime('%b %Y')} | "
-        f"OIL: {oil.index.min().strftime('%b %Y')}–{oil.index.max().strftime('%b %Y')}\n"
+        f"OIL:    {oil.index.min().strftime('%b %Y')}–{oil.index.max().strftime('%b %Y')}\n"
         "Source: Local FRED snapshot (offline)."
     )
     plt.annotate(
@@ -222,9 +227,26 @@ def main() -> plt.Figure:
     args = p.parse_args()
 
     start_dt = datetime.fromisoformat(args.start)
-    end_dt = datetime.fromisoformat(args.end)
+    end_dt   = datetime.fromisoformat(args.end)
 
+    # ───────────────────────────────────────────────────────────────────────
+    # Fetch both UNRATE and oil; UNRATE may contain NaNs if the last
+    # month’s data has not yet been published. We forward‐fill UNRATE
+    # (option B), then drop any remaining NaNs in both series.
+    # ───────────────────────────────────────────────────────────────────────
     unrate, oil = fetch_series(start_dt, end_dt, args.db)
+
+    # Forward‐fill UNRATE to propagate the last known value into any NaN gaps
+    unrate["value"] = unrate["value"].ffill()  # :contentReference[oaicite:3]{index=3}
+    # Drop any rows where UNRATE is still NaN (e.g., if the entire series started after 'start')
+    unrate = unrate.dropna(subset=["value"])   # :contentReference[oaicite:4]{index=4}
+
+    # Drop any rows in 'oil' where the value is NaN (ensures no Nulls remain)
+    oil = oil.dropna(subset=["value"])         # :contentReference[oaicite:5]{index=5}
+
+    # Align oil to only dates present in UNRATE after cleaning
+    oil = oil.loc[oil.index.isin(unrate.index)]
+
     validate_series(unrate, oil)
     fig = plot_lagged(unrate, oil, args.offset, start_dt, end_dt, args.extend_years)
     save_figure(fig, args.output, __file__)