Merge pull request #884 from Lumiwealth/feature/thetadata-cache-layout

Refactor ThetaData cache directory layout

Author: grzesir

docs/remote_cache.md

@@ -48,18 +48,19 @@ Notes:
 Example ThetaData quote cache on macOS:
 
 ```
-/Users/<user>/Library/Caches/lumibot/1.0/thetadata/stock_SPY_minute_ohlc.parquet
+/Users/<user>/Library/Caches/lumibot/1.0/thetadata/stock/minute/ohlc/stock_SPY_minute_ohlc.parquet
 ```
 
 With the example configuration above the remote key becomes:
 
 ```
-prod/cache/v1/thetadata/stock_SPY_minute_ohlc.parquet
+prod/cache/v1/thetadata/stock/minute/ohlc/stock_SPY_minute_ohlc.parquet
 ```
 
-This format aligns with the intended IAM policy layout (provider/asset class
-segments) and keeps migration straightforward for other data sources such as
-Polygon or DataBento.
+This format aligns with the intended IAM policy layout (provider → asset class →
+timespan → datastyle) and keeps migration straightforward for other data sources
+such as Polygon or DataBento. Option-chain caches now live at
+`thetadata/option/option_chains/<symbol>_<date>.parquet`.
 
 ## Implementation Overview
 

lumibot/backtesting/thetadata_backtesting_pandas.py

@@ -288,7 +288,8 @@ def _finalize_day_frame(
         )
 
         expected_last_dt = self.to_default_timezone(current_dt).replace(hour=0, minute=0, second=0, microsecond=0) - timedelta(days=1)
-        target_index = pd.date_range(end=expected_last_dt, periods=requested_length, freq="D", tz=self.tzinfo)
+        expected_last_dt_utc = expected_last_dt.astimezone(pytz.UTC)
+        target_index = pd.date_range(end=expected_last_dt_utc, periods=requested_length, freq="D", tz=pytz.UTC).tz_convert(self.tzinfo)
 
         # DEBUG-LOG: Target index details
         logger.debug(

lumibot/tools/thetadata_helper.py

@@ -26,6 +26,17 @@
 BOOT_GRACE_PERIOD = 5.0
 MAX_RESTART_ATTEMPTS = 3
 
+
+def _resolve_asset_folder(asset_obj: Asset) -> str:
+    asset_type = getattr(asset_obj, "asset_type", None) or "stock"
+    asset_key = str(asset_type).strip().lower()
+    return asset_key
+
+
+def _normalize_folder_component(value: str, fallback: str) -> str:
+    normalized = str(value or "").strip().lower().replace(" ", "_")
+    return normalized or fallback
+
 # Global process tracking for ThetaTerminal
 THETA_DATA_PROCESS = None
 THETA_DATA_PID = None
@@ -785,7 +796,11 @@ def get_trading_dates(asset: Asset, start: datetime, end: datetime):
 def build_cache_filename(asset: Asset, timespan: str, datastyle: str = "ohlc"):
     """Helper function to create the cache filename for a given asset and timespan"""
 
-    lumibot_cache_folder = Path(LUMIBOT_CACHE_FOLDER) / CACHE_SUBFOLDER
+    provider_root = Path(LUMIBOT_CACHE_FOLDER) / CACHE_SUBFOLDER
+    asset_folder = _resolve_asset_folder(asset)
+    timespan_folder = _normalize_folder_component(timespan, "unknown")
+    datastyle_folder = _normalize_folder_component(datastyle, "default")
+    base_folder = provider_root / asset_folder / timespan_folder / datastyle_folder
 
     # If It's an option then also add the expiration date, strike price and right to the filename
     if asset.asset_type == "option":
@@ -799,7 +814,7 @@ def build_cache_filename(asset: Asset, timespan: str, datastyle: str = "ohlc"):
         uniq_str = asset.symbol
 
     cache_filename = f"{asset.asset_type}_{uniq_str}_{timespan}_{datastyle}.parquet"
-    cache_file = lumibot_cache_folder / cache_filename
+    cache_file = base_folder / cache_filename
     return cache_file
 
 
@@ -1969,7 +1984,7 @@ def get_chains_cached(
     Retrieve option chain with caching (MATCHES POLYGON PATTERN).
 
     This function follows the EXACT same caching strategy as Polygon:
-    1. Check cache: LUMIBOT_CACHE_FOLDER/thetadata/option_chains/{symbol}_{date}.parquet
+    1. Check cache: LUMIBOT_CACHE_FOLDER/thetadata/<asset-type>/option_chains/{symbol}_{date}.parquet
     2. Reuse files within RECENT_FILE_TOLERANCE_DAYS (default 7 days)
     3. If not found, fetch from ThetaData and save to cache
     4. Use pyarrow engine with snappy compression
@@ -2006,7 +2021,7 @@ def get_chains_cached(
         return None
 
     # 2) Build cache folder path
-    chain_folder = Path(LUMIBOT_CACHE_FOLDER) / "thetadata" / "option_chains"
+    chain_folder = Path(LUMIBOT_CACHE_FOLDER) / "thetadata" / _resolve_asset_folder(asset) / "option_chains"
     chain_folder.mkdir(parents=True, exist_ok=True)
 
     # 3) Check for recent cached file (within RECENT_FILE_TOLERANCE_DAYS)

setup.py

@@ -5,7 +5,7 @@
 
 setuptools.setup(
     name="lumibot",
-    version="4.2.1",
+    version="4.2.3",
     author="Robert Grzesik",
     author_email="rob@lumiwealth.com",
     description="Backtesting and Trading Library, Made by Lumiwealth",

tests/test_backtest_cache_manager.py

@@ -67,7 +67,7 @@ def _build_settings(prefix: str = "prod/cache") -> BacktestCacheSettings:
 def test_remote_key_uses_relative_cache_path(tmp_path, monkeypatch):
     cache_root = tmp_path / "cache"
     cache_root.mkdir()
-    local_file = cache_root / "thetadata" / "bars" / "spy.parquet"
+    local_file = cache_root / "thetadata" / "stock" / "minute" / "ohlc" / "stock_SPY_minute_ohlc.parquet"
     local_file.parent.mkdir(parents=True, exist_ok=True)
 
     monkeypatch.setattr(backtest_cache, "LUMIBOT_CACHE_FOLDER", cache_root)
@@ -76,17 +76,17 @@ def test_remote_key_uses_relative_cache_path(tmp_path, monkeypatch):
     manager = BacktestCacheManager(settings, client_factory=lambda settings: StubS3Client())
 
     remote_key = manager.remote_key_for(local_file)
-    assert remote_key == "stage/cache/v3/thetadata/bars/spy.parquet"
+    assert remote_key == "stage/cache/v3/thetadata/stock/minute/ohlc/stock_SPY_minute_ohlc.parquet"
 
 
 def test_ensure_local_file_downloads_from_s3(tmp_path, monkeypatch):
     cache_root = tmp_path / "cache"
     cache_root.mkdir()
-    local_file = cache_root / "thetadata" / "bars" / "spy.parquet"
+    local_file = cache_root / "thetadata" / "stock" / "minute" / "ohlc" / "stock_SPY_minute_ohlc.parquet"
 
     monkeypatch.setattr(backtest_cache, "LUMIBOT_CACHE_FOLDER", cache_root)
 
-    remote_key = "stage/cache/v3/thetadata/bars/spy.parquet"
+    remote_key = "stage/cache/v3/thetadata/stock/minute/ohlc/stock_SPY_minute_ohlc.parquet"
     objects = {("test-bucket", remote_key): b"cached-data"}
 
     stub = StubS3Client(objects)
@@ -101,7 +101,7 @@ def test_ensure_local_file_downloads_from_s3(tmp_path, monkeypatch):
 def test_ensure_local_file_handles_missing_remote(tmp_path, monkeypatch):
     cache_root = tmp_path / "cache"
     cache_root.mkdir()
-    local_file = cache_root / "thetadata" / "bars" / "spy.parquet"
+    local_file = cache_root / "thetadata" / "stock" / "minute" / "ohlc" / "stock_SPY_minute_ohlc.parquet"
 
     monkeypatch.setattr(backtest_cache, "LUMIBOT_CACHE_FOLDER", cache_root)
 
@@ -116,13 +116,13 @@ def test_ensure_local_file_handles_missing_remote(tmp_path, monkeypatch):
 def test_on_local_update_uploads_file(tmp_path, monkeypatch):
     cache_root = tmp_path / "cache"
     cache_root.mkdir()
-    local_file = cache_root / "thetadata" / "bars" / "spy.parquet"
+    local_file = cache_root / "thetadata" / "stock" / "minute" / "ohlc" / "stock_SPY_minute_ohlc.parquet"
     local_file.parent.mkdir(parents=True, exist_ok=True)
     local_file.write_bytes(b"new-data")
 
     monkeypatch.setattr(backtest_cache, "LUMIBOT_CACHE_FOLDER", cache_root)
 
-    remote_key = "stage/cache/v3/thetadata/bars/spy.parquet"
+    remote_key = "stage/cache/v3/thetadata/stock/minute/ohlc/stock_SPY_minute_ohlc.parquet"
     stub = StubS3Client({("test-bucket", remote_key): b"old"})
     manager = BacktestCacheManager(_build_settings(prefix="stage/cache"), client_factory=lambda s: stub)
 

tests/test_backtesting_data_source_env.py

@@ -75,6 +75,9 @@ def test_auto_select_polygon_case_insensitive(self, clean_environment, restore_t
         # Configure caplog to capture INFO level logs from lumibot.strategies._strategy
         import logging
         caplog.set_level(logging.INFO, logger='lumibot.strategies._strategy')
+        polygon_key = os.environ.get("POLYGON_API_KEY")
+        if not polygon_key:
+            pytest.skip("Polygon API key not configured")
 
         with patch.dict(os.environ, {'BACKTESTING_DATA_SOURCE': 'polygon'}):
             # Re-import credentials to pick up env change

tests/test_thetadata_helper.py

@@ -324,13 +324,13 @@ def test_get_trading_dates():
 def test_build_cache_filename(mocker, tmpdir, datastyle):
     asset = Asset("SPY")
     timespan = "1D"
-    mocker.patch.object(thetadata_helper, "LUMIBOT_CACHE_FOLDER", tmpdir)
-    expected = tmpdir / "thetadata" / f"stock_SPY_1D_{datastyle}.parquet"
+    mocker.patch.object(thetadata_helper, "LUMIBOT_CACHE_FOLDER", str(tmpdir))
+    expected = tmpdir / "thetadata" / "stock" / "1d" / datastyle / f"stock_SPY_1D_{datastyle}.parquet"
     assert thetadata_helper.build_cache_filename(asset, timespan, datastyle) == expected
 
     expire_date = datetime.date(2023, 8, 1)
     option_asset = Asset("SPY", asset_type="option", expiration=expire_date, strike=100, right="CALL")
-    expected = tmpdir / "thetadata" / f"option_SPY_230801_100_CALL_1D_{datastyle}.parquet"
+    expected = tmpdir / "thetadata" / "option" / "1d" / datastyle / f"option_SPY_230801_100_CALL_1D_{datastyle}.parquet"
     assert thetadata_helper.build_cache_filename(option_asset, timespan, datastyle) == expected
 
     # Bad option asset with no expiration
@@ -427,8 +427,8 @@ def test_missing_dates():
     ],
 )
 def test_update_cache(mocker, tmpdir, df_all, df_cached, datastyle):
-    mocker.patch.object(thetadata_helper, "LUMIBOT_CACHE_FOLDER", tmpdir)
-    cache_file = Path(tmpdir / "thetadata" / f"stock_SPY_1D_{datastyle}.parquet")
+    mocker.patch.object(thetadata_helper, "LUMIBOT_CACHE_FOLDER", str(tmpdir))
+    cache_file = thetadata_helper.build_cache_filename(Asset("SPY"), "1D", datastyle)
 
     # Empty DataFrame of df_all, don't write cache file
     thetadata_helper.update_cache(cache_file, df_all, df_cached)
@@ -550,8 +550,9 @@ def on_local_update(self, local_path, payload=None):
 )
 def test_load_data_from_cache(mocker, tmpdir, df_cached, datastyle):
     # Setup some basics
-    mocker.patch.object(thetadata_helper, "LUMIBOT_CACHE_FOLDER", tmpdir)
-    cache_file = Path(tmpdir / "thetadata" / f"stock_SPY_1D_{datastyle}.parquet")
+    mocker.patch.object(thetadata_helper, "LUMIBOT_CACHE_FOLDER", str(tmpdir))
+    asset = Asset("SPY")
+    cache_file = thetadata_helper.build_cache_filename(asset, "1D", datastyle)
 
     # No cache file should return None (not raise)
     assert thetadata_helper.load_cache(cache_file) is None
@@ -1371,8 +1372,8 @@ def test_chains_cache_reuse(self):
 
         # CLEAR CACHE to ensure first call downloads fresh data
         # This prevents cache pollution from previous tests in the suite
-        # Chains are stored in: LUMIBOT_CACHE_FOLDER / "thetadata" / "option_chains"
-        chain_folder = Path(LUMIBOT_CACHE_FOLDER) / "thetadata" / "option_chains"
+        # Chains are stored in: LUMIBOT_CACHE_FOLDER / "thetadata" / "option" / "option_chains"
+        chain_folder = Path(LUMIBOT_CACHE_FOLDER) / "thetadata" / "option" / "option_chains"
         if chain_folder.exists():
             # Delete all AAPL chain cache files
             for cache_file in chain_folder.glob("AAPL_*.parquet"):
@@ -1407,6 +1408,46 @@ def test_chains_cache_reuse(self):
         assert time2 < time1 * 0.1, f"Cache not working: time1={time1:.2f}s, time2={time2:.2f}s (should be 10x faster)"
         print(f"✓ Cache speedup: {time1/time2:.1f}x faster ({time1:.2f}s -> {time2:.4f}s)")
 
+
+def test_finalize_day_frame_handles_dst_fallback():
+    tz = pytz.timezone("America/New_York")
+    utc = pytz.UTC
+    frame_index = pd.date_range(
+        end=tz.localize(datetime.datetime(2024, 10, 31, 16, 0)),
+        periods=5,
+        freq="D",
+    )
+    frame = pd.DataFrame(
+        {
+            "open": [100 + i for i in range(len(frame_index))],
+            "high": [101 + i for i in range(len(frame_index))],
+            "low": [99 + i for i in range(len(frame_index))],
+            "close": [100.5 + i for i in range(len(frame_index))],
+            "volume": [1000 + i for i in range(len(frame_index))],
+        },
+        index=frame_index,
+    )
+
+    data_source = ThetaDataBacktestingPandas(
+        datetime_start=utc.localize(datetime.datetime(2024, 10, 1)),
+        datetime_end=utc.localize(datetime.datetime(2024, 11, 5)),
+        username="user",
+        password="pass",
+        use_quote_data=False,
+    )
+
+    current_dt = utc.localize(datetime.datetime(2024, 11, 4, 13, 30))
+    result = data_source._finalize_day_frame(
+        frame,
+        current_dt,
+        requested_length=len(frame_index),
+        timeshift=None,
+        asset=Asset("TSLA"),
+    )
+
+    assert result is not None
+    assert len(result) == len(frame_index)
+
     def test_chains_strike_format(self):
         """Test strikes are floats (not integers) and properly converted."""
         username = os.environ.get("THETADATA_USERNAME")

tests/test_thetadata_pandas_verification.py

@@ -44,7 +44,7 @@ def count_cache_files():
     cache_dir = get_cache_dir()
     if not cache_dir.exists():
         return 0
-    return len(list(cache_dir.glob("*.parquet")))
+    return sum(1 for _ in cache_dir.rglob("*.parquet"))
 
 
 class WeeklyMomentumOptionsStrategy(Strategy):