Merge pull request #745 from betcode-org/task/python-312

liampauling · web-flow · commit bfaa5def6dc6 · 2024-03-12T10:07:19.000Z
upgrade for python 3.12
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ['3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
       fail-fast: false
 
     steps:
@@ -22,14 +22,9 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
 
-    - name: Check setup.py
-      run: |
-        python setup.py check
-
     - name: Install dependencies
       run: |
         pip install -r requirements.txt
-        pip install -r requirements-test.txt
 
     - name: Black fmt
       run: |
@@ -55,15 +50,10 @@ jobs:
       with:
         python-version: 3.11
 
-    - name: Check setup.py
-      run: |
-        python setup.py check
-
     - name: Install dependencies
       run: |
         pip install -r requirements.txt
         pip install -r requirements-speed.txt
-        pip install -r requirements-test.txt
 
     - name: Unittest
       run: |
@@ -87,11 +77,10 @@ jobs:
     - name: Install dependencies
       run: |
         pip install -r requirements.txt
-        pip install -r requirements-test.txt
 
     - name: Build package and docs
       run: |
-        python setup.py sdist bdist_wheel
+        python -m build
         twine check dist/*
         mkdocs build
 
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -2,7 +2,7 @@
 # See https://pre-commit.com/hooks.html for more hooks
 repos:
   - repo: https://github.com/psf/black
-    rev: 23.9.1
+    rev: 24.2.0
     hooks:
       - id: black
         # It is recommended to specify the latest version of Python
diff --git a/.travis.yml b/.travis.yml
diff --git a/HISTORY.rst b/HISTORY.rst
@@ -3,6 +3,23 @@
 Release History
 ---------------
 
+2.6.0 (2024-03-11)
++++++++++++++++++++
+
+**Improvements**
+
+- toml file added / repo cleanup
+
+**Bug Fixes**
+
+- Correctly handle sports data files that contain no valid data
+
+**Libraries**
+
+- python 3.12 added
+- betfairlightweight upgraded to 2.20.1
+- black upgraded to 24.2.0
+
 2.5.10 (2024-02-15)
 +++++++++++++++++++
 
diff --git a/README.md b/README.md
@@ -24,7 +24,7 @@ Support for market, order and custom streaming data.
 
 [join betcode slack group](https://join.slack.com/t/betcode-org/shared_invite/zt-25yz6dt1y-LHya5VzHLOzN3RZEQrSnrA)
 
-Tested on Python 3.8, 3.9, 3.10 and 3.11.
+Tested on Python 3.8, 3.9, 3.10, 3.11 and 3.12.
 
 ## installation
 
diff --git a/docs/index.md b/docs/index.md
@@ -30,7 +30,7 @@ Support for market, order and custom streaming data.
 
 [join slack group](https://join.slack.com/t/betcode-org/shared_invite/zt-25yz6dt1y-LHya5VzHLOzN3RZEQrSnrA)
 
-Tested on Python 3.8, 3.9, 3.10 and 3.11.
+Tested on Python 3.8, 3.9, 3.10, 3.11 and 3.12.
 
 ## installation
 
diff --git a/examples/example-betconnect.py b/examples/example-betconnect.py
@@ -61,9 +61,9 @@ def start(self, flumine) -> None:
         )
         for fixture in active_fixtures:
             # get markets
-            context["active_markets"][
-                fixture.fixture_id
-            ] = client.betting.active_markets(fixture_id=fixture.fixture_id)
+            context["active_markets"][fixture.fixture_id] = (
+                client.betting.active_markets(fixture_id=fixture.fixture_id)
+            )
             # get the selections and prices for WIN market
             fixture_selection_prices = client.betting.selections_for_market(
                 fixture_id=fixture.fixture_id,
diff --git a/examples/strategies/marketrecorder.py b/examples/strategies/marketrecorder.py
@@ -16,7 +16,6 @@
 
 
 class MarketRecorder(BaseStrategy):
-
     """
     Simple raw streaming market recorder, context:
 
@@ -230,9 +229,11 @@ def _load(self, market, compress_file_dir: str, market_definition: dict) -> None
                 bucket=self._bucket,
                 key=self._make_prices_file_s3_key(compress_file_dir, market_definition),
                 extra_args={
-                    "Metadata": self._create_metadata(market_definition)
-                    if market_definition
-                    else {}
+                    "Metadata": (
+                        self._create_metadata(market_definition)
+                        if market_definition
+                        else {}
+                    )
                 },
             )
             logger.info("%s successfully loaded to s3" % compress_file_dir)
diff --git a/flumine/__version__.py b/flumine/__version__.py
@@ -1,6 +1,6 @@
 __title__ = "flumine"
 __description__ = "Betting trading framework"
 __url__ = "https://github.com/betcode-org/flumine"
-__version__ = "2.5.10"
+__version__ = "2.6.0"
 __author__ = "Liam Pauling"
 __license__ = "MIT"
diff --git a/flumine/controls/clientcontrols.py b/flumine/controls/clientcontrols.py
@@ -11,7 +11,6 @@
 
 
 class MaxTransactionCount(BaseControl):
-
     """
     Counts and limits orders based on max
     transaction count as per:
diff --git a/flumine/controls/tradingcontrols.py b/flumine/controls/tradingcontrols.py
@@ -11,7 +11,6 @@
 
 
 class OrderValidation(BaseControl):
-
     """
     Validates order price and size is valid for
     exchange.
@@ -186,7 +185,6 @@ def _validate(self, order: BaseOrder, package_type: OrderPackageType) -> None:
 
 
 class StrategyExposure(BaseControl):
-
     """
     Validates:
         - `strategy.validate_order` function
diff --git a/flumine/markets/blotter.py b/flumine/markets/blotter.py
@@ -24,7 +24,6 @@
 
 
 class Blotter:
-
     """
     Simple and fast class to hold all orders for
     a particular market.
diff --git a/flumine/markets/middleware.py b/flumine/markets/middleware.py
@@ -306,6 +306,8 @@ def __init__(self, operation: str, directory: str):
     def __call__(self, market) -> None:
         pt = market.market_book.publish_time_epoch
         while True:
+            if self._next is None:
+                break
             for update in self._next:
                 if update.market_id != market.market_id:
                     continue
@@ -332,7 +334,13 @@ def add_market(self, market) -> None:
         # create sports data generator
         file_path = os.path.join(self.directory, market.market_id)
         self._gen = self._create_generator(file_path, self.operation, 123)()
-        self._next = next(self._gen)
+        try:
+            self._next = next(self._gen)
+        except StopIteration:
+            logger.error(
+                f"File {market.market_id} cannot be processed (data is not valid)"
+            )
+            self._next = None
 
     def remove_market(self, market) -> None:
         # clear gens
diff --git a/flumine/order/order.py b/flumine/order/order.py
@@ -298,9 +298,11 @@ def info(self) -> dict:
                 "average_price_matched": self.average_price_matched,
             },
             "responses": {
-                "date_time_placed": str(self.responses.date_time_placed)
-                if self.responses.date_time_placed
-                else None,
+                "date_time_placed": (
+                    str(self.responses.date_time_placed)
+                    if self.responses.date_time_placed
+                    else None
+                ),
                 "elapsed_seconds_executable": self.elapsed_seconds_executable,
             },
             "runner_status": self.runner_status,
diff --git a/flumine/strategy/strategy.py b/flumine/strategy/strategy.py
@@ -24,7 +24,6 @@
 
 
 class BaseStrategy:
-
     """
     Strategy object to process MarketBook data
     from streams, order placement and handling
@@ -215,9 +214,9 @@ def get_runner_context(
         try:
             return self._invested[(market_id, selection_id, handicap)]
         except KeyError:
-            self._invested[
-                (market_id, selection_id, handicap)
-            ] = runner_context = RunnerContext(selection_id)
+            self._invested[(market_id, selection_id, handicap)] = runner_context = (
+                RunnerContext(selection_id)
+            )
             return runner_context
 
     def market_cached(self, market_id: str) -> bool:
diff --git a/flumine/streams/datastream.py b/flumine/streams/datastream.py
@@ -198,9 +198,11 @@ def run(self) -> None:
         try:
             self.stream_id = self._stream.subscribe_to_orders(
                 order_filter=filters.streaming_order_filter(
-                    customer_strategy_refs=[config.customer_strategy_ref]
-                    if config.customer_strategy_ref
-                    else None,
+                    customer_strategy_refs=(
+                        [config.customer_strategy_ref]
+                        if config.customer_strategy_ref
+                        else None
+                    ),
                     partition_matched_by_strategy_ref=True,
                     include_overall_position=False,
                 ),
diff --git a/flumine/streams/orderstream.py b/flumine/streams/orderstream.py
@@ -39,9 +39,11 @@ def run(self) -> None:
         try:
             self.stream_id = self._stream.subscribe_to_orders(
                 order_filter=filters.streaming_order_filter(
-                    customer_strategy_refs=[config.customer_strategy_ref]
-                    if config.customer_strategy_ref
-                    else None,
+                    customer_strategy_refs=(
+                        [config.customer_strategy_ref]
+                        if config.customer_strategy_ref
+                        else None
+                    ),
                     partition_matched_by_strategy_ref=True,
                     include_overall_position=False,
                 ),
diff --git a/flumine/streams/streams.py b/flumine/streams/streams.py
@@ -164,7 +164,7 @@ def add_historical_stream(
         market: str,
         market_definition: Optional[MarketDefinition],
         event_processing: bool,
-        **listener_kwargs
+        **listener_kwargs,
     ) -> HistoricalStream:
         for stream in self:
             if (
diff --git a/flumine/worker.py b/flumine/worker.py
@@ -23,7 +23,7 @@ def __init__(
         start_delay: int = 0,
         context: dict = None,
         name: str = None,
-        **kwargs
+        **kwargs,
     ):
         name = name or function.__name__
         threading.Thread.__init__(self, daemon=True, name=name, **kwargs)
diff --git a/pyproject.toml b/pyproject.toml
@@ -0,0 +1,63 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "flumine"
+description = "Betting trading framework"
+readme = "README.md"
+license = {file = "LICENSE"}
+requires-python = ">=3.8"
+authors = [
+    { name = "Liam Pauling", email = "a@unknown.com" },
+]
+classifiers = [
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+]
+dependencies = [
+    "betfairlightweight==2.20.1",
+    "tenacity>=7.0.0,<8.2.4",
+    "python-json-logger==2.0.7",
+    "requests",
+    "betconnect==0.1.7",
+    "smart-open>=6,<7",
+]
+dynamic = ["version"]
+
+[project.optional-dependencies]
+speed = [
+    "betfairlightweight[speed]==2.20.1"
+]
+test = [
+    "black==24.2.0",
+    "coverage",
+    "pre-commit",
+    "mkdocs",
+    "mkdocs-material",
+    "build",
+    "twine",
+]
+
+[project.urls]
+Homepage = "https://github.com/betcode-org"
+Documentation = "https://betcode-org.github.io/flumine/"
+Repository = "https://github.com/betcode-org/flumine.git"
+"Bug Tracker" = "https://github.com/betcode-org/flumine/issues"
+Changelog = "https://github.com/betcode-org/flumine/blob/master/HISTORY.rst"
+
+[tool.hatch.version]
+path = "flumine/__version__.py"
+
+[tool.hatch.build.targets.sdist]
+include = [
+    "/flumine",
+    "/HISTORY.rst",
+    "/README.md",
+    "/tests",
+]
diff --git a/requirements-speed.txt b/requirements-speed.txt
@@ -1 +1 @@
-betfairlightweight[speed]==2.19.1
+betfairlightweight[speed]==2.20.1
diff --git a/requirements-test.txt b/requirements-test.txt
diff --git a/requirements.txt b/requirements.txt
diff --git a/setup.py b/setup.py
diff --git a/tests/test_middleware.py b/tests/test_middleware.py
diff --git a/tests/test_streams.py b/tests/test_streams.py
