Merge branch 'main' into dependabot/pip/certifi-2025.8.3

Author: alphatownsman

.dockerignore

@@ -1,4 +1,5 @@
 .DS_Store
+.data
 .env
 .venv
 .vscode
@@ -14,8 +15,10 @@ __pycache__
 /media
 /static
 /test_data
+/tests
 /neodb
 /neodb-takahe/docs
 /neodb-takahe/docker
 /neodb-takahe/static-collected
 /neodb-takahe/takahe/local_settings.py
+/neodb-takahe/tests

.github/workflows/check.yml

@@ -33,9 +33,10 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-          cache: pip
       - name: Install uv
         uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
       - name: Install Dependencies
         run: |
           uv sync

.github/workflows/tests.yml

@@ -48,23 +48,23 @@ jobs:
       uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
-        cache: pip
     - name: Install uv
       uses: astral-sh/setup-uv@v6
+      with:
+        enable-cache: true
     - name: Install Dependencies
       run: |
         uv sync
         sudo apt install -y gettext
     - name: Run Tests
       env:
+        NEODB_SECRET_KEY: test
+        NEODB_SITE_NAME: test
+        NEODB_SITE_DOMAIN: example.org
         NEODB_DB_URL: postgres://testuser:testpass@127.0.0.1/test_neodb
         TAKAHE_DB_URL: postgres://testuser:testpass@127.0.0.1/test_neodb_takahe
         NEODB_REDIS_URL: redis://127.0.0.1:6379/0
         NEODB_SEARCH_URL: typesense://testuser:testpass@127.0.0.1:8108/cat
-        NEODB_SITE_NAME: test
-        NEODB_SITE_DOMAIN: test.domain
-        NEODB_SECRET_KEY: test
-        SPOTIFY_API_KEY: TEST
       run: |
         uv run manage.py compilemessages -i .venv -l zh_Hans
         uv run pytest

README.md

@@ -37,6 +37,7 @@ Follow us on [Fediverse](https://mastodon.online/@neodb), [Bluesky](https://bsky
     * Bangumi
     * Board Game Geek
     * Archive of Our Own
+    * WikiData
     * any RSS link to a podcast
 - Logged in users can manage their collections:
   + mark an item as wishlist/in progress/complete
@@ -49,6 +50,7 @@ Follow us on [Fediverse](https://mastodon.online/@neodb), [Bluesky](https://bsky
     * Goodreads reading list
     * Letterboxd watch list
     * Douban archive (via [Doufen](https://doufen.org/))
+    * Steam Library
 - Social features:
   + view home feed with friends' activities
     * every activity can be set as viewable to self/follower-only/public

catalog/apps.py

@@ -12,6 +12,6 @@ def ready(self):
         from journal import models as journal_models  # noqa
 
         # register cron jobs
-        from catalog.jobs import DiscoverGenerator, PodcastUpdater  # noqa
+        from catalog.jobs import DiscoverGenerator, PodcastUpdater, CatalogStats  # noqa
 
         init_catalog_audit_log()

catalog/book/models.py

@@ -294,7 +294,7 @@ def set_work(self, work: "Work | None"):
         if work:
             work.editions.add(self)
 
-    def merge_to(self, to_item: "Edition | None"):  # type: ignore[reportIncompatibleMethodOverride]
+    def merge_to(self, to_item):
         super().merge_to(to_item)
         if to_item:
             if self.merge_title():
@@ -314,45 +314,24 @@ def delete(self, *args, **kwargs):
             self.set_work(None)
         return super().delete(*args, **kwargs)
 
-    def update_linked_items_from_external_resource(self, resource):
-        """add Work from resource.metadata['work'] if not yet"""
-        links = resource.required_resources + resource.related_resources
-        for w in links:
-            if w.get("model") == "Work":
-                work_res = ExternalResource.objects.filter(
-                    id_type=w["id_type"], id_value=w["id_value"]
-                ).first()
-                if work_res:
-                    work = work_res.item
-                    if not work:
-                        logger.warning(f"Unable to find work for {work_res}")
-                else:
-                    logger.warning(
-                        f"Unable to find resource for {w['id_type']}:{w['id_value']}"
-                    )
-                    work = Work.objects.filter(
-                        primary_lookup_id_type=w["id_type"],
-                        primary_lookup_id_value=w["id_value"],
-                    ).first()
-                if work:
-                    w = self.get_work()
-                    if w:
-                        if w != work:
-                            w.log_action(
-                                {"!link_and_merge": [str(self), str(resource)]}
-                            )
-                            logger.info(
-                                f"Merging {work} to {w} when fetching from {resource}"
-                            )
-                            work.merge_to(w)
-                    else:
-                        self.set_work(work)
-
-    def merge_data_from_external_resource(
-        self, p: "ExternalResource", ignore_existing_content: bool = False
-    ):
-        super().merge_data_from_external_resource(p, ignore_existing_content)
-        self.merge_title()
+    def process_fetched_item(self, fetched, link_type):
+        if link_type == ExternalResource.LinkType.PARENT and isinstance(fetched, Work):
+            w = self.get_work()
+            if w:
+                if w == fetched:
+                    return False
+                w.log_action({"!merge_on_fetch": [str(self), str(fetched)]})
+                logger.info(f"Merging {fetched} to {w} when fetched {self}.")
+                fetched.merge_to(w)
+            else:
+                self.set_work(fetched)
+            return True
+        return False
+
+    def normalize_metadata(self, override_resources=[]):
+        r = super().normalize_metadata(override_resources)
+        r |= self.merge_title()
+        return r
 
     def merge_title(self) -> bool:
         # Edition should have only one title, so extra titles will be merged to other_title, return True if updated
@@ -491,13 +470,13 @@ def lookup_id_type_choices(cls):
         ]
         return [(i.value, i.label) for i in id_types]
 
-    def merge_to(self, to_item: "Work | None"):  # type: ignore[reportIncompatibleMethodOverride]
+    def merge_to(self, to_item):
         super().merge_to(to_item)
         if not to_item:
             return
         for edition in self.editions.all():
             edition.set_work(to_item)
-        to_item.language = uniq(to_item.language + self.language)  # type: ignore
+        to_item.language = uniq(to_item.language + self.language)
         to_item.localized_title = uniq(to_item.localized_title + self.localized_title)
         to_item.save()
 
@@ -515,29 +494,6 @@ def cover_image_url(self):
         e = next(filter(lambda e: e.cover_image_url, self.editions.all()), None)
         return e.cover_image_url if e else None
 
-    def update_linked_items_from_external_resource(self, resource):
-        """add Edition from resource.metadata['required_resources'] if not yet"""
-        links = resource.required_resources + resource.related_resources
-        for e in links:
-            if e.get("model") == "Edition":
-                edition_res = ExternalResource.objects.filter(
-                    id_type=e["id_type"], id_value=e["id_value"]
-                ).first()
-                if edition_res:
-                    edition = edition_res.item
-                    if not edition:
-                        logger.warning(f"Unable to find edition for {edition_res}")
-                else:
-                    logger.warning(
-                        f"Unable to find resource for {e['id_type']}:{e['id_value']}"
-                    )
-                    edition = Edition.objects.filter(
-                        primary_lookup_id_type=e["id_type"],
-                        primary_lookup_id_value=e["id_value"],
-                    ).first()
-                if edition:
-                    edition.set_work(self)
-
     def to_indexable_doc(self):
         return {}  # no index for Work, for now
 

catalog/common/downloaders.py

@@ -47,8 +47,8 @@ def get_mock_mode():
 
 def get_mock_file(url):
     fn = url.replace("***REMOVED***", "1234")  # Thank you, Github Action -_-!
+    fn = re.sub(r"key=[*A-Za-z0-9_\-]+", "key_8964", fn)
     fn = re.sub(r"[^\w]", "_", fn)
-    fn = re.sub(r"_key_[*A-Za-z0-9]+", "_key_8964", fn)
     if len(fn) > 255:
         fn = fn[:255]
     return fn
@@ -70,7 +70,8 @@ def __init__(self, url):
         except Exception:
             self.content = b"Error: response file not found"
             self.status_code = 404
-            logger.debug(f"local response not found for {url} at {fn}")
+            if ".jpg" not in self.url:
+                logger.warning(f"local response not found for {url} at {fn}")
 
     @property
     def text(self):
@@ -89,9 +90,7 @@ def xml(self):
 
     @property
     def headers(self):
-        return {
-            "Content-Type": "image/jpeg" if self.url.endswith("jpg") else "text/html"
-        }
+        return {"Content-Type": "image/jpeg" if ".jpg" in self.url else "text/html"}
 
 
 class DownloaderResponse(Response):

catalog/common/migrations.py

@@ -1,10 +1,9 @@
-import logging
+from time import sleep
 
 from django.db import connection, models
+from loguru import logger
 from tqdm import tqdm
 
-logger = logging.getLogger(__name__)
-
 
 def fix_20250208():
     logger.warning("Fixing soft-deleted editions...")
@@ -139,3 +138,68 @@ def normalize_language_20250524():
                 i.save(update_fields=["metadata"])
                 u += 1
     logger.warning(f"normalize_language finished. {u} of {c} items updated.")
+
+
+def link_tmdb_wikidata_20250815(limit=None):
+    """
+    Scan all TMDB Movie and TVShow resources, refetch them, and link to WikiData resources if available.
+
+    This function:
+    1. Finds all ExternalResources with TMDB Movie and TVShow ID types
+    2. Refetches each TMDB resource to ensure we have the latest data
+    3. If the TMDB resource has a WikiData ID, fetches the corresponding WikiData resource
+    4. Links both resources to the same Item
+    """
+    from catalog.common import IdType, SiteManager
+    from catalog.common.models import ExternalResource
+    from catalog.sites.wikidata import WikiData
+
+    logger.warning("Starting TMDB-WikiData linking process")
+    tmdb_resources = ExternalResource.objects.filter(
+        id_type__in=[IdType.TMDB_Movie, IdType.TMDB_TV]
+    )
+    if limit:
+        tmdb_resources = tmdb_resources[:limit]
+    count_total = tmdb_resources.count()
+    count_with_wikidata = 0
+    count_errors = 0
+    logger.warning(f"Found {count_total} TMDB resources to process")
+    for resource in tqdm(tmdb_resources, total=count_total):
+        try:
+            site_cls = SiteManager.get_site_cls_by_id_type(resource.id_type)
+            if not site_cls:
+                logger.error(f"Could not find site class for {resource.id_type}")
+                count_errors += 1
+                continue
+            site = site_cls(resource.url)
+            try:
+                resource_content = site.scrape()
+            except Exception as e:
+                logger.error(f"Failed to scrape {resource.url}: {e}")
+                count_errors += 1
+                continue
+            wikidata_id = resource_content.lookup_ids.get(IdType.WikiData)
+            if not wikidata_id:
+                continue
+            resource.update_content(resource_content)
+            count_with_wikidata += 1
+            wiki_site = WikiData(id_value=wikidata_id)
+            try:
+                wiki_site.get_resource_ready()
+                logger.success(f"Linked WikiData {wiki_site} to {site}")
+            except Exception as e:
+                logger.error(f"Failed to process WikiData {wikidata_id}: {e}")
+                count_errors += 1
+            sleep(0.5)
+        except Exception as e:
+            logger.error(f"Error processing resource {resource}: {e}")
+            count_errors += 1
+    logger.warning("TMDB-WikiData linking process completed:")
+    logger.warning(f"  Total TMDB resources processed: {count_total}")
+    logger.warning(f"  TMDB resources with WikiData IDs: {count_with_wikidata}")
+    logger.warning(f"  Errors encountered: {count_errors}")
+    return {
+        "total": count_total,
+        "with_wikidata": count_with_wikidata,
+        "errors": count_errors,
+    }