Merge pull request #890 from Lumiwealth/feature/timezone-and-sanitization-fixes

Align ThetaData fill and bump boto3 floor

Author: grzesir

lumibot/backtesting/databento_backtesting_pandas.py

@@ -410,6 +410,7 @@ def get_last_price(self, asset, quote=None, exchange=None):
             # OPTIMIZATION: Check cache first
             self._check_and_clear_cache()
             current_dt = self.get_datetime()
+            current_dt_aware = to_datetime_aware(current_dt)
 
             # Try to get data from our cached pandas_data first
             search_asset = asset
@@ -435,8 +436,6 @@ def get_last_price(self, asset, quote=None, exchange=None):
 
                 if not df.empty and 'close' in df.columns:
                         # Ensure current_dt is timezone-aware for comparison
-                        current_dt_aware = to_datetime_aware(current_dt)
-
                         # Step back one bar so only fully closed bars are visible
                         bar_delta = timedelta(minutes=1)
                         if asset_data.timestep == "hour":
@@ -454,19 +453,45 @@ def get_last_price(self, asset, quote=None, exchange=None):
                             filtered_df = df[df.index <= current_dt_aware]
 
                         if not filtered_df.empty:
-                            last_price = filtered_df['close'].iloc[-1]
-                            if not pd.isna(last_price):
-                                price = float(last_price)
+                            valid_closes = filtered_df['close'].dropna()
+                            if not valid_closes.empty:
+                                price = float(valid_closes.iloc[-1])
                                 # OPTIMIZATION: Cache the result
                                 self._last_price_cache[cache_key] = price
                                 return price
 
-            # If no cached data, try to get recent data
+            # If no cached data, try to load it for the backtest window
+            try:
+                fetched_bars = self.get_historical_prices(
+                    asset_separated,
+                    length=1,
+                    quote=quote_asset,
+                    timestep="minute",
+                )
+                if fetched_bars is not None:
+                    asset_data = self.pandas_data.get(search_asset)
+                    if asset_data is not None:
+                        df = asset_data.df
+                        if not df.empty and 'close' in df.columns:
+                            valid_closes = df[df.index <= current_dt_aware]['close'].dropna()
+                            if not valid_closes.empty:
+                                price = float(valid_closes.iloc[-1])
+                                self._last_price_cache[cache_key] = price
+                                return price
+            except Exception as exc:
+                logger.debug(
+                    "Attempted to hydrate Databento cache for %s but hit error: %s",
+                    asset.symbol,
+                    exc,
+                )
+
+            # If still no data, fall back to direct fetch (live-style)
             logger.warning(f"No cached data for {asset.symbol}, attempting direct fetch")
             return databento_helper.get_last_price_from_databento(
                 api_key=self._api_key,
                 asset=asset_separated,
-                venue=exchange
+                venue=exchange,
+                reference_date=current_dt_aware
             )
 
         except DataBentoAuthenticationError as e:

lumibot/backtesting/thetadata_backtesting_pandas.py

@@ -771,7 +771,7 @@ def _update_pandas_data(self, asset, quote, length, timestep, start_dt=None):
                 quote_columns = ['bid', 'ask', 'bid_size', 'ask_size', 'bid_condition', 'ask_condition', 'bid_exchange', 'ask_exchange']
                 existing_quote_cols = [col for col in quote_columns if col in df.columns]
                 if existing_quote_cols:
-                    df[existing_quote_cols] = df[existing_quote_cols].fillna(method='ffill')
+                    df[existing_quote_cols] = df[existing_quote_cols].ffill()
 
                     # Log how much forward filling occurred
                     if 'bid' in df.columns and 'ask' in df.columns:

lumibot/components/options_helper.py

@@ -791,18 +791,11 @@ def get_expiration_on_or_after_date(self, dt: Union[date, datetime], chains: Uni
                     self.strategy.log_message(f"Cannot validate data without underlying symbol, returning {exp_date}", color="yellow")
                     return exp_date
 
-        # No future expirations with valid data; log and check last available
-        if expiration_dates:
-            # Check the last available expiry for data
-            for exp_str, exp_date in reversed(expiration_dates):
-                strikes = specific_chain.get(exp_str)
-                if strikes and len(strikes) > 0:
-                    self.strategy.log_message(
-                        f"No valid expirations on or after {dt}; using latest available {exp_date} for {call_or_put_caps}.",
-                        color="yellow",
-                    )
-                    return exp_date
-
+        # No future expirations with tradeable data; let the caller skip entries gracefully.
+        self.strategy.log_message(
+            f"No valid expirations on or after {dt} with tradeable data for {call_or_put_caps}; skipping.",
+            color="yellow",
+        )
         return None
 
     # ============================================================

lumibot/strategies/_strategy.py

@@ -133,10 +133,12 @@ def _normalize_backtest_datetime(value):
         """
         if value is None:
             return None
-        if isinstance(value, datetime.datetime) and (
-            value.tzinfo is None or value.tzinfo.utcoffset(value) is None
-        ):
-            return to_datetime_aware(value)
+        if isinstance(value, datetime.datetime):
+            tzinfo = value.tzinfo
+            if tzinfo is None or tzinfo.utcoffset(value) is None:
+                return to_datetime_aware(value)
+            if not hasattr(tzinfo, "zone"):
+                return value.astimezone(LUMIBOT_DEFAULT_PYTZ)
         return value
 
     @property

lumibot/tools/ccxt_data_store.py

@@ -445,7 +445,7 @@ def _fill_missing_data(self, df:DataFrame, freq:str)->DataFrame:
         if freq == "1d":
             dt_range = pd.date_range(start=df.index.min(), end=df.index.max(), freq="D")
         else:
-            dt_range = pd.date_range(start=df.index.min(), end=df.index.max(), freq="T")
+            dt_range = pd.date_range(start=df.index.min(), end=df.index.max(), freq="min")
 
         df_complete = df.reindex(dt_range).ffill()
         df_complete['missing'] = np.where(df_complete.index.isin(df.index), 0, 1)

lumibot/tools/databento_helper.py

@@ -947,6 +947,7 @@ def get_last_price_from_databento(
     api_key: str,
     asset: Asset,
     venue: Optional[str] = None,
+    reference_date: Optional[datetime] = None,
     **kwargs
 ) -> Optional[Union[float, Decimal]]:
     """
@@ -978,12 +979,14 @@ def get_last_price_from_databento(
 
         # For continuous futures, resolve to the current active contract
         if asset.asset_type == Asset.AssetType.CONT_FUTURE:
-            # Use Asset class method to resolve continuous futures to actual contract (returns string)
-            resolved_symbol = asset.resolve_continuous_futures_contract(year_digits=1)
+            # Resolve based on reference date when backtesting so we match the contract in use
+            resolved_symbol = _format_futures_symbol_for_databento(
+                asset,
+                reference_date=reference_date,
+            )
             if resolved_symbol is None:
                 logger.error(f"Could not resolve continuous futures contract for {asset.symbol}")
                 return None
-            # Generate the correct DataBento symbol format (should be single result)
             symbols_to_try = _generate_databento_symbol_alternatives(asset.symbol, resolved_symbol)
             logger.info(f"Resolved continuous future {asset.symbol} to specific contract: {resolved_symbol}")
             logger.info(f"DataBento symbol format for last price: {symbols_to_try[0]}")
@@ -1000,12 +1003,17 @@ def get_last_price_from_databento(
             if hasattr(range_result, 'end') and range_result.end:
                 if hasattr(range_result.end, 'tz_localize'):
                     # Already a pandas Timestamp
-                    available_end = range_result.end if range_result.end.tz else range_result.end.tz_localize('UTC')
+                    if range_result.end.tz is not None:
+                        available_end = range_result.end.tz_convert('UTC')
+                    else:
+                        available_end = range_result.end.tz_localize('UTC')
                 else:
                     # Convert to pandas Timestamp
-                    available_end = pd.to_datetime(range_result.end).tz_localize('UTC')
+                    ts = pd.to_datetime(range_result.end)
+                    available_end = ts if ts.tz is not None else ts.tz_localize('UTC')
             elif isinstance(range_result, dict) and 'end' in range_result:
-                available_end = pd.to_datetime(range_result['end']).tz_localize('UTC')
+                ts = pd.to_datetime(range_result['end'])
+                available_end = ts if ts.tz is not None else ts.tz_localize('UTC')
             else:
                 logger.warning(f"Could not parse dataset range for {dataset}: {range_result}")
                 # Fallback: use a recent date that's likely to have data
@@ -1047,10 +1055,10 @@ def get_last_price_from_databento(
                         df = pd.DataFrame(data)
 
                     if not df.empty:
-                        # Get the last available price (close price of most recent bar)
                         if 'close' in df.columns:
-                            price = df['close'].iloc[-1]
-                            if pd.notna(price):
+                            closes = df['close'].dropna()
+                            if not closes.empty:
+                                price = closes.iloc[-1]
                                 logger.info(f"✓ SUCCESS: Got last price for {symbol_to_use}: {price}")
                                 return float(price)