From 14e1f4faa5bc68ecd24784e220aa9ea36d26c8c0 Mon Sep 17 00:00:00 2001
From: thePR0M3TH3AN <53631862+PR0M3TH3AN@users.noreply.github.com>
Date: Sun, 1 Jun 2025 11:08:11 -0400
Subject: [PATCH] Clamp series to last UNRATE date

---
 README.md                                 | 15 ++++++------
 scripts/lagged_oil_unrate_chart_styled.py | 28 +++++++++--------------
 2 files changed, 19 insertions(+), 24 deletions(-)

diff --git a/README.md b/README.md
index 3295f74..518abd9 100644
--- a/README.md
+++ b/README.md
@@ -138,10 +138,11 @@ What happens under the hood:
    ```
 
    (\[Real Python]\[1], \[Software Engineering Stack Exchange]\[2])
-3. Converts both series to month‐end indices, resamples oil to a monthly average, and aligns their date ranges.
-4. Validates that neither series is empty, contains `NaN`, or has no overlap (raising a `ValueError` if any issue).
-5. Applies the specified lag (e.g., 18 months), then plots UNRATE on the left y‐axis and log‐scaled WTI on the right y‐axis.
-6. Saves the figure to `outputs/lagged_oil_unrate_chart_styled_<timestamp>.png` (unless `--output` is specified). (\[martinheinz.dev]\[4], \[Real Python]\[1])
+3. Converts both series to month-end indices, resamples oil to a monthly average, and aligns their date ranges.
+4. Clamps both series to the last month where `UNRATE` has a value so any trailing NaN is removed.
+5. Validates that neither series is empty, contains `NaN`, or has no overlap (raising a `ValueError` if any issue).
+6. Applies the specified lag (e.g., 18 months), then plots UNRATE on the left y-axis and log-scaled WTI on the right y-axis.
+7. Saves the figure to `outputs/lagged_oil_unrate_chart_styled_<timestamp>.png` (unless `--output` is specified). (\[martinheinz.dev]\[4], \[Real Python]\[1])
 
 #### Customizing the UNRATE vs. Oil Chart
 
@@ -320,9 +321,9 @@ python-dateutil>=2.8.1
 
   * This indicates that the UNRATE DataFrame has missing (NaN) entries. Possible fixes:
 
-    1. Drop nulls within SQL—e.g., run `refresh_data.py --end` using the last fully published month rather than “today” (\[devgem.io]\[7], \[pyOpenSci]\[8])
-    2. Forward‐fill or backward‐fill missing UNRATE values in `common.py` before validation (if appropriate) (\[pyOpenSci]\[8], \[Real Python]\[1])
-    3. Comment out the strict null check in `validate_dataframe(...)` (not recommended for publication‐quality charts) (\[devgem.io]\[7], \[Real Python]\[1])
+    1. Drop nulls within SQL—e.g., run `refresh_data.py --end` using the last fully published month rather than "today" ([devgem.io][7], [pyOpenSci][8])
+    2. Clamp both series to `UNRATE.last_valid_index()` before validation (the approach used in `lagged_oil_unrate_chart_styled.py`).
+    3. Comment out the strict null check in `validate_dataframe(...)` (not recommended for publication-quality charts) ([devgem.io][7], [Real Python][1])
 
 * **Database not found**
 
diff --git a/scripts/lagged_oil_unrate_chart_styled.py b/scripts/lagged_oil_unrate_chart_styled.py
index e9d05c5..36b94dd 100644
--- a/scripts/lagged_oil_unrate_chart_styled.py
+++ b/scripts/lagged_oil_unrate_chart_styled.py
@@ -3,10 +3,10 @@
 lagged_oil_unrate_chart_styled.py
 ---------------------------------
 Offline version: reads UNRATE & WTI series from `data/fred.db`
-(populated by `scripts/refresh_data.py`), forward‐fills any NaNs
-in the UNRATE series, drops remaining NaNs in both series to maintain
-purity, validates that both series contain data for the requested period
-(overlap only), and plots them with a configurable lag.
+(populated by `scripts/refresh_data.py`), clamps both series to the
+last month where UNRATE has a valid value, validates that they contain
+data for the requested period (overlap only), and plots them with a
+configurable lag.
 """
 
 from __future__ import annotations
@@ -269,22 +269,16 @@ def main(argv: list[str] | None = None) -> plt.Figure:
         raise SystemExit(1)
 
     # ───────────────────────────────────────────────────────────────────────
-    # Fetch both UNRATE and oil; UNRATE may contain NaNs if the last
-    # month’s data has not yet been published. We forward‐fill UNRATE
-    # (option B), then drop any remaining NaNs in both series.
+    # Fetch both UNRATE and OIL. If UNRATE has a trailing NaN for the most
+    # recent month, clamp both series to its last valid index so that
+    # validation works on officially published data only.
     # ───────────────────────────────────────────────────────────────────────
     unrate, oil = fetch_series(start_dt, end_dt, args.db)
 
-    # Forward‐fill UNRATE to propagate the last known value into any NaN gaps
-    unrate["value"] = unrate["value"].ffill()
-    # Drop any rows where UNRATE is still NaN (e.g., if the entire series started after 'start')
-    unrate = unrate.dropna(subset=["value"])
-
-    # Drop any rows in 'oil' where the value is NaN (ensures no Nulls remain)
-    oil = oil.dropna(subset=["value"])
-
-    # Align oil to only dates present in UNRATE after cleaning
-    oil = oil.loc[oil.index.isin(unrate.index)]
+    last_date = unrate.last_valid_index()
+    if last_date is not None:
+        unrate = unrate.loc[:last_date]
+        oil = oil.loc[:last_date]
 
     validate_series(unrate, oil)
     fig = plot_lagged(