Merge pull request #891 from Lumiwealth/feature/databento-gc-roll

Align DataBento futures rolling with COMEX schedule

Author: grzesir

lumibot/data_sources/databento_data_polars.py

@@ -24,8 +24,12 @@
 from .data_source import DataSource
 from .polars_mixin import PolarsMixin
 from lumibot.entities import Asset, Bars, Quote
-from lumibot.tools import databento_helper_polars
-from lumibot.tools.databento_helper_polars import _ensure_polars_datetime_timezone as _ensure_polars_tz
+from lumibot.tools import databento_helper_polars, futures_roll
+from lumibot.tools.databento_helper_polars import (
+    _ensure_polars_datetime_timezone as _ensure_polars_tz,
+    _format_futures_symbol_for_databento,
+    _generate_databento_symbol_alternatives,
+)
 from lumibot.tools.lumibot_logger import get_logger
 
 logger = get_logger(__name__)
@@ -512,38 +516,30 @@ def _warn_stale(self, symbol: str, context: str):
 
     def _resolve_futures_symbol(self, asset: Asset, reference_date: datetime = None) -> str:
         """Resolve asset to specific futures contract symbol"""
-        if asset.asset_type in [Asset.AssetType.FUTURE, Asset.AssetType.CONT_FUTURE]:
-            # For continuous futures, resolve to specific contract
-            if asset.asset_type == Asset.AssetType.CONT_FUTURE:
-                if hasattr(asset, 'resolve_continuous_futures_contract'):
-                    return asset.resolve_continuous_futures_contract(
-                        reference_date=reference_date,
-                        year_digits=1,
-                    )
-            
-            # Manual resolution for common futures
-            symbol = asset.symbol.upper()
-            month = reference_date.month if reference_date else datetime.now().month
-            year = reference_date.year if reference_date else datetime.now().year
-            
-            # Quarterly contracts
-            if month <= 3:
-                month_code = 'H'
-            elif month <= 6:
-                month_code = 'M'
-            elif month <= 9:
-                month_code = 'U'
-            else:
-                month_code = 'Z'
-            
-            year_digit = year % 10
-            
-            if symbol in ["ES", "NQ", "RTY", "YM", "MES", "MNQ", "MYM", "M2K", "CL", "GC", "SI"]:
-                return f"{symbol}{month_code}{year_digit}"
-            
+        if asset.asset_type not in [Asset.AssetType.FUTURE, Asset.AssetType.CONT_FUTURE]:
             return asset.symbol
-        
-        return asset.symbol
+
+        ref_dt = reference_date or datetime.now(timezone.utc)
+
+        if asset.asset_type == Asset.AssetType.FUTURE and asset.expiration:
+            return _format_futures_symbol_for_databento(asset, reference_date=reference_date)
+
+        if asset.asset_type == Asset.AssetType.CONT_FUTURE:
+            resolved_contract = futures_roll.resolve_symbol_for_datetime(
+                asset,
+                ref_dt,
+                year_digits=2,
+            )
+        else:
+            temp_asset = Asset(asset.symbol, Asset.AssetType.CONT_FUTURE)
+            resolved_contract = futures_roll.resolve_symbol_for_datetime(
+                temp_asset,
+                ref_dt,
+                year_digits=2,
+            )
+
+        databento_symbol = _generate_databento_symbol_alternatives(asset.symbol, resolved_contract)
+        return databento_symbol[0] if databento_symbol else resolved_contract
 
     def get_historical_prices(
         self,

lumibot/tools/databento_helper.py

@@ -784,6 +784,22 @@ def _fetch_and_update_futures_multiplier(
             logger.info(f"[MULTIPLIER] AFTER update: asset.multiplier = {asset.multiplier}")
         else:
             logger.error(f"[MULTIPLIER] ✗ Definition missing unit_of_measure_qty field! Fields: {list(definition.keys())}")
+
+        if (
+            asset.asset_type == Asset.AssetType.FUTURE
+            and getattr(asset, "expiration", None) in (None, "")
+        ):
+            expiration_value = definition.get('expiration')
+            if expiration_value:
+                try:
+                    expiration_ts = pd.to_datetime(expiration_value, utc=True, errors='coerce')
+                except Exception as exc:
+                    logger.debug(f"[MULTIPLIER] Unable to parse expiration '{expiration_value}' for {asset.symbol}: {exc}")
+                    expiration_ts = None
+
+                if expiration_ts is not None and not pd.isna(expiration_ts):
+                    asset.expiration = expiration_ts.date()
+                    logger.debug(f"[MULTIPLIER] ✓ Captured expiration for {asset.symbol}: {asset.expiration}")
     else:
         logger.error(f"[MULTIPLIER] ✗ Failed to get definition from DataBento for {resolved_symbol}")
 

lumibot/tools/databento_helper_polars.py

@@ -863,6 +863,22 @@ def _fetch_and_update_futures_multiplier(
             logger.debug(f"[MULTIPLIER] AFTER update: asset.multiplier = {asset.multiplier}")
         else:
             logger.error(f"[MULTIPLIER] ✗ Definition missing unit_of_measure_qty field! Fields: {list(definition.keys())}")
+
+        if (
+            asset.asset_type == Asset.AssetType.FUTURE
+            and getattr(asset, "expiration", None) in (None, "")
+        ):
+            expiration_value = definition.get('expiration')
+            if expiration_value:
+                try:
+                    expiration_ts = pd.to_datetime(expiration_value, utc=True, errors='coerce')
+                except Exception as exc:
+                    logger.debug(f"[MULTIPLIER] Unable to parse expiration '{expiration_value}' for {asset.symbol}: {exc}")
+                    expiration_ts = None
+
+                if expiration_ts is not None and not pd.isna(expiration_ts):
+                    asset.expiration = expiration_ts.date()
+                    logger.debug(f"[MULTIPLIER] ✓ Captured expiration for {asset.symbol}: {asset.expiration}")
     else:
         logger.error(f"[MULTIPLIER] ✗ Failed to get definition from DataBento for {resolved_symbol}")
 

lumibot/tools/futures_roll.py

@@ -30,13 +30,32 @@
 class RollRule:
     offset_business_days: int
     anchor: str
+    contract_months: Optional[Tuple[int, ...]] = None
+
+
+_DEFAULT_CONTRACT_MONTHS: Tuple[int, ...] = (3, 6, 9, 12)
 
 
 ROLL_RULES: Dict[str, RollRule] = {
-    symbol: RollRule(offset_business_days=8, anchor="third_friday")
+    symbol: RollRule(offset_business_days=8, anchor="third_friday", contract_months=_DEFAULT_CONTRACT_MONTHS)
     for symbol in {"ES", "MES", "NQ", "MNQ", "YM", "MYM"}
 }
 
+ROLL_RULES.update(
+    {
+        "GC": RollRule(
+            offset_business_days=7,
+            anchor="third_last_business_day",
+            contract_months=(2, 4, 6, 8, 10, 12),
+        ),
+        "SI": RollRule(
+            offset_business_days=7,
+            anchor="third_last_business_day",
+            contract_months=(1, 3, 5, 7, 9, 12),
+        ),
+    }
+)
+
 YearMonth = Tuple[int, int]
 
 
@@ -72,25 +91,61 @@ def _subtract_business_days(dt: datetime, days: int) -> datetime:
     return result
 
 
+def _third_last_business_day(year: int, month: int) -> datetime:
+    if month == 12:
+        next_month = 1
+        next_year = year + 1
+    else:
+        next_month = month + 1
+        next_year = year
+
+    last_day = _to_timezone(datetime(next_year, next_month, 1)) - timedelta(days=1)
+
+    remaining = 3
+    cursor = last_day
+    while remaining > 0:
+        if cursor.weekday() < 5:
+            remaining -= 1
+            if remaining == 0:
+                break
+        cursor -= timedelta(days=1)
+    return cursor.replace(hour=0, minute=0, second=0, microsecond=0)
+
+
 def _calculate_roll_trigger(year: int, month: int, rule: RollRule) -> datetime:
     if rule.anchor == "third_friday":
         anchor = _third_friday(year, month)
+    elif rule.anchor == "third_last_business_day":
+        anchor = _third_last_business_day(year, month)
     else:
         anchor = _to_timezone(datetime(year, month, 15))
     if rule.offset_business_days <= 0:
         return anchor
     return _subtract_business_days(anchor, rule.offset_business_days)
 
 
-def _advance_quarter(current_month: int, current_year: int) -> YearMonth:
-    quarter_months = [3, 6, 9, 12]
-    idx = quarter_months.index(current_month)
-    next_idx = (idx + 1) % len(quarter_months)
-    next_month = quarter_months[next_idx]
-    next_year = current_year + (1 if next_idx == 0 else 0)
+def _get_contract_months(rule: Optional[RollRule]) -> Tuple[int, ...]:
+    if rule and rule.contract_months:
+        return tuple(sorted(rule.contract_months))
+    return _DEFAULT_CONTRACT_MONTHS
+
+
+def _advance_contract(current_month: int, current_year: int, months: Tuple[int, ...]) -> YearMonth:
+    months_sorted = tuple(sorted(months))
+    idx = months_sorted.index(current_month)
+    next_idx = (idx + 1) % len(months_sorted)
+    next_month = months_sorted[next_idx]
+    next_year = current_year + (1 if next_idx <= idx else 0)
     return next_year, next_month
 
 
+def _select_contract(year: int, month: int, months: Tuple[int, ...]) -> YearMonth:
+    for candidate in sorted(months):
+        if month <= candidate:
+            return year, candidate
+    return year + 1, sorted(months)[0]
+
+
 def _legacy_mid_month(reference_date: datetime) -> YearMonth:
     quarter_months = [3, 6, 9, 12]
     year = reference_date.year
@@ -118,27 +173,22 @@ def determine_contract_year_month(symbol: str, reference_date: Optional[datetime
     ref = _normalize_reference_date(reference_date)
     symbol_upper = symbol.upper()
     rule = ROLL_RULES.get(symbol_upper)
-
-    quarter_months = [3, 6, 9, 12]
     year = ref.year
     month = ref.month
 
     if rule is None:
         return _legacy_mid_month(ref)
 
-    if month in quarter_months:
+    contract_months = _get_contract_months(rule)
+
+    if month in contract_months:
         target_year, target_month = year, month
-        roll_point = _calculate_roll_trigger(target_year, target_month, rule)
-        if ref >= roll_point:
-            target_year, target_month = _advance_quarter(target_month, target_year)
     else:
-        candidates = [m for m in quarter_months if m > month]
-        if candidates:
-            target_month = candidates[0]
-            target_year = year
-        else:
-            target_month = quarter_months[0]
-            target_year = year + 1
+        target_year, target_month = _select_contract(year, month, contract_months)
+
+    roll_point = _calculate_roll_trigger(target_year, target_month, rule)
+    if ref >= roll_point:
+        target_year, target_month = _advance_contract(target_month, target_year, contract_months)
 
     return target_year, target_month
 
@@ -201,6 +251,7 @@ def build_roll_schedule(asset, start: datetime, end: datetime, year_digits: int
 
     symbol_upper = asset.symbol.upper()
     rule = ROLL_RULES.get(symbol_upper)
+    contract_months = _get_contract_months(rule)
 
     schedule = []
     cursor = start

tests/test_futures_roll.py

@@ -36,3 +36,23 @@ def test_resolve_symbols_for_range_produces_sequential_contracts():
 
     symbols = futures_roll.resolve_symbols_for_range(asset, start, end, year_digits=1)
     assert symbols == ["MESU5", "MESZ5", "MESH6"], symbols
+
+
+def test_comex_gold_rolls_on_third_last_business_day_offset():
+    asset_symbol = "GC"
+
+    year, month = futures_roll.determine_contract_year_month(asset_symbol, _dt(2025, 2, 14))
+    assert (year, month) == (2025, 2)
+
+    # Seven business days before the third last business day of February 2025 is Feb 17
+    year, month = futures_roll.determine_contract_year_month(asset_symbol, _dt(2025, 2, 17))
+    assert (year, month) == (2025, 4)
+
+
+def test_comex_gold_symbol_sequence_uses_even_month_cycle():
+    asset = Asset("GC", asset_type=Asset.AssetType.CONT_FUTURE)
+    start = _dt(2025, 1, 1)
+    end = _dt(2025, 8, 1)
+
+    symbols = futures_roll.resolve_symbols_for_range(asset, start, end, year_digits=1)
+    assert symbols == ["GCG5", "GCJ5", "GCM5", "GCQ5"], symbols