From 3f09b5701ed4eef7d4efe90b2da39be9f1a9c7b3 Mon Sep 17 00:00:00 2001 From: grillazz Date: Tue, 17 Jun 2025 20:02:04 +0200 Subject: [PATCH 1/6] add structlog --- app/main.py | 29 ++++++++++++++++++++++++++--- pyproject.toml | 2 ++ uv.lock | 38 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 66 insertions(+), 3 deletions(-) diff --git a/app/main.py b/app/main.py index 3dbfdac..3fe3231 100644 --- a/app/main.py +++ b/app/main.py @@ -1,7 +1,11 @@ +import logging +import os from contextlib import asynccontextmanager +from logging.handlers import RotatingFileHandler from pathlib import Path import asyncpg +import orjson # from apscheduler import AsyncScheduler # from apscheduler.datastores.sqlalchemy import SQLAlchemyDataStore @@ -9,6 +13,7 @@ from fastapi import Depends, FastAPI, Request from fastapi.responses import HTMLResponse from fastapi.templating import Jinja2Templates +from whenever._whenever import Instant from app.api.health import router as health_router from app.api.ml import router as ml_router @@ -23,9 +28,27 @@ from app.services.auth import AuthBearer # from app.services.scheduler import SchedulerMiddleware -from app.utils.logging import AppLogger +import structlog + +log_date = Instant.now().py_datetime().strftime("%Y%m%d") + +structlog.configure( + cache_logger_on_first_use=True, + wrapper_class=structlog.make_filtering_bound_logger(logging.INFO), + processors=[ + structlog.contextvars.merge_contextvars, + structlog.processors.add_log_level, + structlog.processors.format_exc_info, + structlog.processors.TimeStamper(fmt="iso", utc=True), + structlog.processors.JSONRenderer(serializer=orjson.dumps), + ], + # log per day and per process? + logger_factory=structlog.BytesLoggerFactory( + file=Path(f"cuul_{log_date}_{str(os.getpid())}").with_suffix(".log").open("wb") + ) +) -logger = AppLogger().get_logger() +logger = structlog.get_logger() templates = Jinja2Templates(directory=Path(__file__).parent.parent / "templates") @@ -45,7 +68,7 @@ async def lifespan(_app: FastAPI): min_size=5, max_size=20, ) - logger.info(f"Postgres pool created: {_app.postgres_pool.get_idle_size()=}") + logger.info("Postgres pool created", _app.postgres_pool.get_idle_size()) yield finally: # close redis connection and release the resources diff --git a/pyproject.toml b/pyproject.toml index 2e69052..2331d5e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,8 @@ dependencies = [ "polyfactory>=2.21.0", "granian>=2.3.2", "apscheduler[redis,sqlalchemy]>=4.0.0a6", + "structlog>=25.4.0", + "whenever>=0.8.5", ] [tool.uv] diff --git a/uv.lock b/uv.lock index 786fcac..0779de4 100644 --- a/uv.lock +++ b/uv.lock @@ -453,8 +453,10 @@ dependencies = [ { name = "redis" }, { name = "rich" }, { name = "sqlalchemy" }, + { name = "structlog" }, { name = "uvicorn", extra = ["standard"] }, { name = "uvloop" }, + { name = "whenever" }, ] [package.dev-dependencies] @@ -492,8 +494,10 @@ requires-dist = [ { name = "redis", specifier = ">=6.2.0" }, { name = "rich", specifier = ">=14.0.0" }, { name = "sqlalchemy", specifier = ">=2.0.41" }, + { name = "structlog", specifier = ">=25.4.0" }, { name = "uvicorn", extras = ["standard"], specifier = ">=0.34.3" }, { name = "uvloop", specifier = ">=0.21.0" }, + { name = "whenever", specifier = ">=0.8.5" }, ] [package.metadata.requires-dev] @@ -1575,6 +1579,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/41/94/8af675a62e3c91c2dee47cf92e602cfac86e8767b1a1ac3caf1b327c2ab0/starlette-0.46.0-py3-none-any.whl", hash = "sha256:913f0798bd90ba90a9156383bcf1350a17d6259451d0d8ee27fc0cf2db609038", size = 71991 }, ] +[[package]] +name = "structlog" +version = "25.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/79/b9/6e672db4fec07349e7a8a8172c1a6ae235c58679ca29c3f86a61b5e59ff3/structlog-25.4.0.tar.gz", hash = "sha256:186cd1b0a8ae762e29417095664adf1d6a31702160a46dacb7796ea82f7409e4", size = 1369138 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/4a/97ee6973e3a73c74c8120d59829c3861ea52210667ec3e7a16045c62b64d/structlog-25.4.0-py3-none-any.whl", hash = "sha256:fe809ff5c27e557d14e613f45ca441aabda051d119ee5a0102aaba6ce40eed2c", size = 68720 }, +] + [[package]] name = "tenacity" version = "8.5.0" @@ -1822,6 +1835,31 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/52/24/ab44c871b0f07f491e5d2ad12c9bd7358e527510618cb1b803a88e986db1/werkzeug-3.1.3-py3-none-any.whl", hash = "sha256:54b78bf3716d19a65be4fceccc0d1d7b89e608834989dfae50ea87564639213e", size = 224498 }, ] +[[package]] +name = "whenever" +version = "0.8.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tzdata", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/83/6e/d8081fe842dc829662bdb318bdad1256f4f3875cb0441294a6fa39eb9199/whenever-0.8.5.tar.gz", hash = "sha256:23c7e0119103ef71aab080caf332e17b2b8ee4cb5e0ab61b393263755c377e19", size = 234290 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/23/4a/70bc93422f129aa9ea9397cc731e3ec9f0332d81642ae19ff2ec7b758abc/whenever-0.8.5-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:08c5c77c0387e3fda2726d312a68e374d441cd21ebe98802d07921ff6c6a7ecf", size = 401659 }, + { url = "https://files.pythonhosted.org/packages/a3/51/b1fba840313edf65083c7afc68f796c9b5b5064db03dc5ccc6029600afaa/whenever-0.8.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:98c58420616b6c5ab824471e89c2fa0aff939ef28163e5bbfb7dfbea3d3f8098", size = 389978 }, + { url = "https://files.pythonhosted.org/packages/ef/0b/c964f4842d8dc1f0372199ad9e9ed4276e8a419e2fbdf1f17e417e285cc9/whenever-0.8.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c4a3dbed6168f68f5bad63c7a543446c530eff102bb24504bc4d87bea29ef62c", size = 417973 }, + { url = "https://files.pythonhosted.org/packages/3a/70/37b98f97a9364999e98ea9074775ca4c1df09fcded85e9bce3738373ad7d/whenever-0.8.5-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ce9b1b4ef98ee92b80d1d2cf518f0767d631cbcd2df3ad4fd1d863fdad2c030c", size = 453330 }, + { url = "https://files.pythonhosted.org/packages/5d/2a/a8c5cc159da33860e709720034bd39b39f94b8c191da2a0eeaf5d75659c9/whenever-0.8.5-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:739d608ee0586d0972d6f7789c705016e416b3ef8ae5337c1f73605feb23a784", size = 550280 }, + { url = "https://files.pythonhosted.org/packages/1b/f8/edc24161c6cd3ee93363bd14b4e5ff679239f1d185a1f588d93b60b90221/whenever-0.8.5-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4487c7684f7292e66b9a4c2bd6b5efbd7271c48c4410648d90e967dc5e0144ca", size = 463600 }, + { url = "https://files.pythonhosted.org/packages/35/db/2f873f0854f577f16f20b651762acd4df279a3fa8b5f855113b609a15465/whenever-0.8.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ef58e82d825a763860bf1de61c28bba7ffa0a7da3a6456381d441caf6693f845", size = 433784 }, + { url = "https://files.pythonhosted.org/packages/50/df/555db022bdab1fa39e7b5d3756da1841f499e569cb7fda74a356dba6b89b/whenever-0.8.5-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f3992f8f3424ff52c5c21a95792b2dfbe537df08ab951ef7a0654ba517b7a28e", size = 481548 }, + { url = "https://files.pythonhosted.org/packages/95/fe/d427a3e3d6ae9c69690f4c2a1b40e301d7abf45a2c6c21aa3040f25fe642/whenever-0.8.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a1988f40fe0d28902aa3f132e4fb23a18599cf9ad49a54311265425631d8ec96", size = 595909 }, + { url = "https://files.pythonhosted.org/packages/26/0c/c90389b0f52473cdfb30046ad9369b379ab924d9d2d3be590e09890d49a6/whenever-0.8.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:17d47930a318563180c06dcf3332db3956658751d2d29b367c8e550fdda34b2c", size = 716887 }, + { url = "https://files.pythonhosted.org/packages/20/45/5e1f15b51a7311579d707fac7250e3c5312e7c2491972b85c1e5859b7fa0/whenever-0.8.5-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:0a28c299195fdf9082d7fa7d1aa985d709aef8c206160bd50b52d1913261af3d", size = 653268 }, + { url = "https://files.pythonhosted.org/packages/c9/73/935c542a6ec07699773c231871f4fd1a727a2fba995599275bbe5ad0b500/whenever-0.8.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4920995d03e26e225d2a733d9f58e3ed3cce7a33994b0bf2f6d94c9f85059dd4", size = 605528 }, + { url = "https://files.pythonhosted.org/packages/82/15/bf5332b353239af8120db4934fa6e0b1a4f5772a56185070d85f9523c462/whenever-0.8.5-cp313-cp313-win32.whl", hash = "sha256:f67d1054de92486baf8d48a28c0e2ff5fc78ab2e772b054f69520953727862f5", size = 348552 }, + { url = "https://files.pythonhosted.org/packages/a9/34/e539f26dff602085c001f15264ca508e3df3cd9a7fd50d0a51c3f9759976/whenever-0.8.5-cp313-cp313-win_amd64.whl", hash = "sha256:f7b7f1814fd3d216c8ff5d62076a46f21b38d03af71a59887efa3fc3e8d1c5bb", size = 343086 }, +] + [[package]] name = "wsproto" version = "1.2.0" From c09c338b37ecc6f6a95fa9ea443722accb8c1ca2 Mon Sep 17 00:00:00 2001 From: grillazz Date: Wed, 18 Jun 2025 08:49:32 +0200 Subject: [PATCH 2/6] wip: add structlog --- app/database.py | 5 +- app/main.py | 122 ++++++++++++++++--------------------------- app/utils/logging.py | 25 +++++++++ compose.yml | 1 - granian-compose.yml | 1 - 5 files changed, 73 insertions(+), 81 deletions(-) diff --git a/app/database.py b/app/database.py index 9d8285b..f3b301d 100644 --- a/app/database.py +++ b/app/database.py @@ -3,9 +3,8 @@ from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine from app.config import settings as global_settings -from app.utils.logging import AppLogger - -logger = AppLogger().get_logger() +from app.utils.logging import setup_structlog +logger = setup_structlog() engine = create_async_engine( global_settings.asyncpg_url.unicode_string(), diff --git a/app/main.py b/app/main.py index 3fe3231..29c8847 100644 --- a/app/main.py +++ b/app/main.py @@ -1,19 +1,14 @@ import logging import os from contextlib import asynccontextmanager -from logging.handlers import RotatingFileHandler from pathlib import Path import asyncpg import orjson - -# from apscheduler import AsyncScheduler -# from apscheduler.datastores.sqlalchemy import SQLAlchemyDataStore -# from apscheduler.eventbrokers.redis import RedisEventBroker +import structlog from fastapi import Depends, FastAPI, Request from fastapi.responses import HTMLResponse from fastapi.templating import Jinja2Templates -from whenever._whenever import Instant from app.api.health import router as health_router from app.api.ml import router as ml_router @@ -22,93 +17,68 @@ from app.api.stuff import router as stuff_router from app.api.user import router as user_router from app.config import settings as global_settings - -# from app.database import engine from app.redis import get_redis from app.services.auth import AuthBearer +from whenever._whenever import Instant +from app.utils.logging import setup_structlog -# from app.services.scheduler import SchedulerMiddleware -import structlog - -log_date = Instant.now().py_datetime().strftime("%Y%m%d") - -structlog.configure( - cache_logger_on_first_use=True, - wrapper_class=structlog.make_filtering_bound_logger(logging.INFO), - processors=[ - structlog.contextvars.merge_contextvars, - structlog.processors.add_log_level, - structlog.processors.format_exc_info, - structlog.processors.TimeStamper(fmt="iso", utc=True), - structlog.processors.JSONRenderer(serializer=orjson.dumps), - ], - # log per day and per process? - logger_factory=structlog.BytesLoggerFactory( - file=Path(f"cuul_{log_date}_{str(os.getpid())}").with_suffix(".log").open("wb") - ) -) - -logger = structlog.get_logger() +logger = setup_structlog() templates = Jinja2Templates(directory=Path(__file__).parent.parent / "templates") - @asynccontextmanager -async def lifespan(_app: FastAPI): - # Load the redis connection - _app.redis = await get_redis() - - _postgres_dsn = global_settings.postgres_url.unicode_string() - +async def lifespan(app: FastAPI): + app.redis = await get_redis() + postgres_dsn = global_settings.postgres_url.unicode_string() try: - # TODO: cache with the redis connection - # Initialize the postgres connection pool - _app.postgres_pool = await asyncpg.create_pool( - dsn=_postgres_dsn, + app.postgres_pool = await asyncpg.create_pool( + dsn=postgres_dsn, min_size=5, max_size=20, ) - logger.info("Postgres pool created", _app.postgres_pool.get_idle_size()) + logger.info("Postgres pool created", idle_size=app.postgres_pool.get_idle_size()) yield finally: - # close redis connection and release the resources - await _app.redis.close() - # close postgres connection pool and release the resources - await _app.postgres_pool.close() - - -app = FastAPI(title="Stuff And Nonsense API", version="0.19.0", lifespan=lifespan) - -app.include_router(stuff_router) -app.include_router(nonsense_router) -app.include_router(shakespeare_router) -app.include_router(user_router) -app.include_router(ml_router, prefix="/v1/ml", tags=["ML"]) - - -app.include_router(health_router, prefix="/v1/public/health", tags=["Health, Public"]) -app.include_router( - health_router, - prefix="/v1/health", - tags=["Health, Bearer"], - dependencies=[Depends(AuthBearer())], -) + await app.redis.close() + await app.postgres_pool.close() + +def create_app() -> FastAPI: + app = FastAPI( + title="Stuff And Nonsense API", + version="0.19.0", + lifespan=lifespan, + ) + app.include_router(stuff_router) + app.include_router(nonsense_router) + app.include_router(shakespeare_router) + app.include_router(user_router) + app.include_router(ml_router, prefix="/v1/ml", tags=["ML"]) + app.include_router(health_router, prefix="/v1/public/health", tags=["Health, Public"]) + app.include_router( + health_router, + prefix="/v1/health", + tags=["Health, Bearer"], + dependencies=[Depends(AuthBearer())], + ) + @app.get("/index", response_class=HTMLResponse) + def get_index(request: Request): + return templates.TemplateResponse("index.html", {"request": request}) -@app.get("/index", response_class=HTMLResponse) -def get_index(request: Request): - return templates.TemplateResponse("index.html", {"request": request}) + return app +app = create_app() +# --- Unused/experimental code and TODOs --- +# from apscheduler import AsyncScheduler +# from apscheduler.datastores.sqlalchemy import SQLAlchemyDataStore +# from apscheduler.eventbrokers.redis import RedisEventBroker +# from app.database import engine +# from app.services.scheduler import SchedulerMiddleware # _scheduler_data_store = SQLAlchemyDataStore(engine, schema="scheduler") -# _scheduler_event_broker = RedisEventBroker( -# client_or_url=global_settings.redis_url.unicode_string() -# ) +# _scheduler_event_broker = RedisEventBroker(client_or_url=global_settings.redis_url.unicode_string()) # _scheduler_himself = AsyncScheduler(_scheduler_data_store, _scheduler_event_broker) -# # app.add_middleware(SchedulerMiddleware, scheduler=_scheduler_himself) - - -# TODO: every not GET meth should reset cache -# TODO: every scheduler task which needs to act on database should have access to connection pool via request - maybe ? -# TODO: https://stackoverflow.com/questions/16053364/make-sure-only-one-worker-launches-the-apscheduler-event-in-a-pyramid-web-app-ru +# TODO: every non-GET method should reset cache +# TODO: scheduler tasks needing DB should access connection pool via request +# TODO: https://stackoverflow.com/questions/16053364/make-sure-only-one-worker-launches-the-apscheduler-event-in-a-pyramid-web-app-ru \ No newline at end of file diff --git a/app/utils/logging.py b/app/utils/logging.py index f74f509..6a0833f 100644 --- a/app/utils/logging.py +++ b/app/utils/logging.py @@ -5,6 +5,12 @@ from app.utils.singleton import SingletonMeta +import logging +import os +import orjson +import structlog +from whenever._whenever import Instant +from pathlib import Path class AppLogger(metaclass=SingletonMeta): _logger = None @@ -22,3 +28,22 @@ def __init__(self, width=200, style=None, **kwargs): console=Console(color_system="256", width=width, style=style, stderr=True), **kwargs, ) + +def setup_structlog() -> structlog.BoundLogger: + log_date = Instant.now().py_datetime().strftime("%Y%m%d") + log_path = Path(f"cuul_{log_date}_{os.getpid()}.log") + structlog.configure( + cache_logger_on_first_use=True, + wrapper_class=structlog.make_filtering_bound_logger(logging.INFO), + processors=[ + structlog.contextvars.merge_contextvars, + structlog.processors.add_log_level, + structlog.processors.format_exc_info, + structlog.processors.TimeStamper(fmt="iso", utc=True), + structlog.processors.JSONRenderer(serializer=orjson.dumps), + ], + logger_factory=structlog.BytesLoggerFactory( + file=log_path.open("wb") + ) + ) + return structlog.get_logger() diff --git a/compose.yml b/compose.yml index 9f74f4c..293db32 100644 --- a/compose.yml +++ b/compose.yml @@ -10,7 +10,6 @@ services: - .secrets command: bash -c " uvicorn app.main:app - --log-config ./logging-uvicorn.json --host 0.0.0.0 --port 8080 --lifespan=on --use-colors --loop uvloop --http httptools --reload --log-level debug diff --git a/granian-compose.yml b/granian-compose.yml index 40e285c..d786653 100644 --- a/granian-compose.yml +++ b/granian-compose.yml @@ -12,7 +12,6 @@ services: granian --interface asgi --host 0.0.0.0 --port 8080 app.main:app --access-log --log-level debug - --log-config ./logging-granian.json " volumes: - ./app:/panettone/app From 7e0024876cd957132e61a9d88d0df87907c02795 Mon Sep 17 00:00:00 2001 From: grillazz Date: Sat, 28 Jun 2025 22:05:45 +0200 Subject: [PATCH 3/6] wip: BytesToTextIOWrapper wraps a text handler and encodes bytes to text. --- app/utils/logging.py | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/app/utils/logging.py b/app/utils/logging.py index 6a0833f..2c5ec53 100644 --- a/app/utils/logging.py +++ b/app/utils/logging.py @@ -1,4 +1,4 @@ -import logging +from logging.handlers import RotatingFileHandler from rich.console import Console from rich.logging import RichHandler @@ -29,9 +29,37 @@ def __init__(self, width=200, style=None, **kwargs): **kwargs, ) + +class BytesToTextIOWrapper: + def __init__(self, handler, encoding="utf-8"): + self.handler = handler + self.encoding = encoding + + def write(self, b): + if isinstance(b, bytes): + self.handler.stream.write(b.decode(self.encoding)) + else: + self.handler.stream.write(b) + self.handler.flush() + + def flush(self): + self.handler.flush() + + def close(self): + self.handler.close() + + def setup_structlog() -> structlog.BoundLogger: log_date = Instant.now().py_datetime().strftime("%Y%m%d") - log_path = Path(f"cuul_{log_date}_{os.getpid()}.log") + log_path = Path(f"{log_date}_{os.getpid()}.log") + handler = RotatingFileHandler( + filename=log_path, + mode="a", # text mode + maxBytes=10 * 1024 * 1024, + backupCount=5, + encoding="utf-8" + ) + file_like = BytesToTextIOWrapper(handler) structlog.configure( cache_logger_on_first_use=True, wrapper_class=structlog.make_filtering_bound_logger(logging.INFO), @@ -43,7 +71,7 @@ def setup_structlog() -> structlog.BoundLogger: structlog.processors.JSONRenderer(serializer=orjson.dumps), ], logger_factory=structlog.BytesLoggerFactory( - file=log_path.open("wb") + file=file_like ) ) return structlog.get_logger() From d0d26687df9f981f6e52867e6cf339de2139be4f Mon Sep 17 00:00:00 2001 From: grillazz Date: Sun, 29 Jun 2025 08:59:06 +0200 Subject: [PATCH 4/6] wip: lint --- app/database.py | 1 + app/main.py | 8 +------- app/utils/logging.py | 14 ++++++++------ 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/app/database.py b/app/database.py index f3b301d..d830b89 100644 --- a/app/database.py +++ b/app/database.py @@ -4,6 +4,7 @@ from app.config import settings as global_settings from app.utils.logging import setup_structlog + logger = setup_structlog() engine = create_async_engine( diff --git a/app/main.py b/app/main.py index 29c8847..7dd3403 100644 --- a/app/main.py +++ b/app/main.py @@ -1,11 +1,7 @@ -import logging -import os from contextlib import asynccontextmanager from pathlib import Path import asyncpg -import orjson -import structlog from fastapi import Depends, FastAPI, Request from fastapi.responses import HTMLResponse from fastapi.templating import Jinja2Templates @@ -19,10 +15,8 @@ from app.config import settings as global_settings from app.redis import get_redis from app.services.auth import AuthBearer -from whenever._whenever import Instant from app.utils.logging import setup_structlog - logger = setup_structlog() templates = Jinja2Templates(directory=Path(__file__).parent.parent / "templates") @@ -81,4 +75,4 @@ def get_index(request: Request): # app.add_middleware(SchedulerMiddleware, scheduler=_scheduler_himself) # TODO: every non-GET method should reset cache # TODO: scheduler tasks needing DB should access connection pool via request -# TODO: https://stackoverflow.com/questions/16053364/make-sure-only-one-worker-launches-the-apscheduler-event-in-a-pyramid-web-app-ru \ No newline at end of file +# TODO: https://stackoverflow.com/questions/16053364/make-sure-only-one-worker-launches-the-apscheduler-event-in-a-pyramid-web-app-ru diff --git a/app/utils/logging.py b/app/utils/logging.py index 2c5ec53..e889639 100644 --- a/app/utils/logging.py +++ b/app/utils/logging.py @@ -1,16 +1,16 @@ +import logging +import os from logging.handlers import RotatingFileHandler +from pathlib import Path +import orjson +import structlog from rich.console import Console from rich.logging import RichHandler +from whenever._whenever import Instant from app.utils.singleton import SingletonMeta -import logging -import os -import orjson -import structlog -from whenever._whenever import Instant -from pathlib import Path class AppLogger(metaclass=SingletonMeta): _logger = None @@ -30,6 +30,8 @@ def __init__(self, width=200, style=None, **kwargs): ) + +# TODO: merge this wrapper with the one in structlog under one hood of AppLogger class BytesToTextIOWrapper: def __init__(self, handler, encoding="utf-8"): self.handler = handler From 1098e39f7129e49c8beb858d0d3dcd95b3c36adc Mon Sep 17 00:00:00 2001 From: grillazz Date: Sun, 29 Jun 2025 21:19:30 +0200 Subject: [PATCH 5/6] wip: replace AppLogger with AppStructLogger --- app/api/health.py | 4 +-- app/api/ml.py | 4 +-- app/api/stuff.py | 4 +-- app/api/user.py | 4 +-- app/database.py | 4 +-- app/main.py | 4 +-- app/models/base.py | 4 +-- app/services/auth.py | 4 +-- app/services/smtp.py | 4 +-- app/utils/logging.py | 82 +++++++++++++++++++------------------------- 10 files changed, 53 insertions(+), 65 deletions(-) diff --git a/app/api/health.py b/app/api/health.py index 761c143..3a59010 100644 --- a/app/api/health.py +++ b/app/api/health.py @@ -6,9 +6,9 @@ from starlette.concurrency import run_in_threadpool from app.services.smtp import SMTPEmailService -from app.utils.logging import AppLogger +from app.utils.logging import AppStructLogger -logger = AppLogger().get_logger() +logger = AppStructLogger().get_logger() router = APIRouter() diff --git a/app/api/ml.py b/app/api/ml.py index ca600a9..98d94f4 100644 --- a/app/api/ml.py +++ b/app/api/ml.py @@ -4,9 +4,9 @@ from fastapi.responses import StreamingResponse from app.services.llm import get_llm_service -from app.utils.logging import AppLogger +from app.utils.logging import AppStructLogger -logger = AppLogger().get_logger() +logger = AppStructLogger().get_logger() router = APIRouter() diff --git a/app/api/stuff.py b/app/api/stuff.py index 552bd20..07b39d8 100644 --- a/app/api/stuff.py +++ b/app/api/stuff.py @@ -5,9 +5,9 @@ from app.database import get_db from app.models.stuff import Stuff from app.schemas.stuff import StuffResponse, StuffSchema -from app.utils.logging import AppLogger +from app.utils.logging import AppStructLogger -logger = AppLogger().get_logger() +logger = AppStructLogger().get_logger() router = APIRouter(prefix="/v1/stuff") diff --git a/app/api/user.py b/app/api/user.py index 4b2d468..0d5b3bb 100644 --- a/app/api/user.py +++ b/app/api/user.py @@ -7,9 +7,9 @@ from app.models.user import User from app.schemas.user import TokenResponse, UserLogin, UserResponse, UserSchema from app.services.auth import create_access_token -from app.utils.logging import AppLogger +from app.utils.logging import AppStructLogger -logger = AppLogger().get_logger() +logger = AppStructLogger().get_logger() router = APIRouter(prefix="/v1/user") diff --git a/app/database.py b/app/database.py index d830b89..c900087 100644 --- a/app/database.py +++ b/app/database.py @@ -3,9 +3,9 @@ from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine from app.config import settings as global_settings -from app.utils.logging import setup_structlog +from app.utils.logging import AppStructLogger -logger = setup_structlog() +logger = AppStructLogger().get_logger() engine = create_async_engine( global_settings.asyncpg_url.unicode_string(), diff --git a/app/main.py b/app/main.py index 7dd3403..70ca866 100644 --- a/app/main.py +++ b/app/main.py @@ -15,9 +15,9 @@ from app.config import settings as global_settings from app.redis import get_redis from app.services.auth import AuthBearer -from app.utils.logging import setup_structlog +from app.utils.logging import AppStructLogger -logger = setup_structlog() +logger = AppStructLogger().get_logger() templates = Jinja2Templates(directory=Path(__file__).parent.parent / "templates") @asynccontextmanager diff --git a/app/models/base.py b/app/models/base.py index bb03011..6f114b4 100644 --- a/app/models/base.py +++ b/app/models/base.py @@ -6,9 +6,9 @@ from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import DeclarativeBase, declared_attr -from app.utils.logging import AppLogger +from app.utils.logging import AppStructLogger -logger = AppLogger().get_logger() +logger = AppStructLogger().get_logger() class Base(DeclarativeBase): diff --git a/app/services/auth.py b/app/services/auth.py index 51fff89..3509f8a 100644 --- a/app/services/auth.py +++ b/app/services/auth.py @@ -6,9 +6,9 @@ from app.config import settings as global_settings from app.models.user import User -from app.utils.logging import AppLogger +from app.utils.logging import AppStructLogger -logger = AppLogger().get_logger() +logger = AppStructLogger().get_logger() async def get_from_redis(request: Request, key: str): diff --git a/app/services/smtp.py b/app/services/smtp.py index 098a261..4f1df89 100644 --- a/app/services/smtp.py +++ b/app/services/smtp.py @@ -7,10 +7,10 @@ from pydantic import EmailStr from app.config import settings as global_settings -from app.utils.logging import AppLogger +from app.utils.logging import AppStructLogger from app.utils.singleton import SingletonMetaNoArgs -logger = AppLogger().get_logger() +logger = AppStructLogger().get_logger() @define diff --git a/app/utils/logging.py b/app/utils/logging.py index e889639..49736f8 100644 --- a/app/utils/logging.py +++ b/app/utils/logging.py @@ -5,30 +5,9 @@ import orjson import structlog -from rich.console import Console -from rich.logging import RichHandler from whenever._whenever import Instant -from app.utils.singleton import SingletonMeta - - -class AppLogger(metaclass=SingletonMeta): - _logger = None - - def __init__(self): - self._logger = logging.getLogger(__name__) - - def get_logger(self): - return self._logger - - -class RichConsoleHandler(RichHandler): - def __init__(self, width=200, style=None, **kwargs): - super().__init__( - console=Console(color_system="256", width=width, style=style, stderr=True), - **kwargs, - ) - +from app.utils.singleton import SingletonMetaNoArgs # TODO: merge this wrapper with the one in structlog under one hood of AppLogger @@ -50,30 +29,39 @@ def flush(self): def close(self): self.handler.close() +# @define +class AppStructLogger(metaclass=SingletonMetaNoArgs): + _logger = None -def setup_structlog() -> structlog.BoundLogger: - log_date = Instant.now().py_datetime().strftime("%Y%m%d") - log_path = Path(f"{log_date}_{os.getpid()}.log") - handler = RotatingFileHandler( - filename=log_path, - mode="a", # text mode - maxBytes=10 * 1024 * 1024, - backupCount=5, - encoding="utf-8" - ) - file_like = BytesToTextIOWrapper(handler) - structlog.configure( - cache_logger_on_first_use=True, - wrapper_class=structlog.make_filtering_bound_logger(logging.INFO), - processors=[ - structlog.contextvars.merge_contextvars, - structlog.processors.add_log_level, - structlog.processors.format_exc_info, - structlog.processors.TimeStamper(fmt="iso", utc=True), - structlog.processors.JSONRenderer(serializer=orjson.dumps), - ], - logger_factory=structlog.BytesLoggerFactory( - file=file_like + def __init__(self): + _log_date = Instant.now().py_datetime().strftime("%Y%m%d") + _log_path = Path(f"{_log_date}_{os.getpid()}.log") + _handler = RotatingFileHandler( + filename=_log_path, + mode="a", # text mode + maxBytes=10 * 1024 * 1024, + backupCount=5, + encoding="utf-8" + ) + structlog.configure( + cache_logger_on_first_use=True, + wrapper_class=structlog.make_filtering_bound_logger(logging.INFO), + processors=[ + structlog.contextvars.merge_contextvars, + structlog.processors.add_log_level, + structlog.processors.format_exc_info, + structlog.processors.TimeStamper(fmt="iso", utc=True), + structlog.processors.JSONRenderer(serializer=orjson.dumps), + ], + logger_factory=structlog.BytesLoggerFactory( + file=BytesToTextIOWrapper(_handler) + ) ) - ) - return structlog.get_logger() + self._logger = structlog.get_logger() + + def get_logger(self) -> structlog.BoundLogger: + """ + Returns: + structlog.BoundLogger: The configured logger instance. + """ + return self._logger From 8e7692bd32e28251032871bc603c3c3bd9894a8a Mon Sep 17 00:00:00 2001 From: grillazz Date: Sun, 29 Jun 2025 21:22:46 +0200 Subject: [PATCH 6/6] wip: reimplement AppStructLogger with attrs lib --- app/utils/logging.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/app/utils/logging.py b/app/utils/logging.py index 49736f8..7ba940a 100644 --- a/app/utils/logging.py +++ b/app/utils/logging.py @@ -5,6 +5,7 @@ import orjson import structlog +from attrs import define, field from whenever._whenever import Instant from app.utils.singleton import SingletonMetaNoArgs @@ -29,16 +30,17 @@ def flush(self): def close(self): self.handler.close() -# @define + +@define(slots=True) class AppStructLogger(metaclass=SingletonMetaNoArgs): - _logger = None + _logger: structlog.BoundLogger = field(init=False) - def __init__(self): + def __attrs_post_init__(self): _log_date = Instant.now().py_datetime().strftime("%Y%m%d") _log_path = Path(f"{_log_date}_{os.getpid()}.log") _handler = RotatingFileHandler( filename=_log_path, - mode="a", # text mode + mode="a", maxBytes=10 * 1024 * 1024, backupCount=5, encoding="utf-8" @@ -60,8 +62,4 @@ def __init__(self): self._logger = structlog.get_logger() def get_logger(self) -> structlog.BoundLogger: - """ - Returns: - structlog.BoundLogger: The configured logger instance. - """ return self._logger