chore: prepare 1.4.0 (#604)

Author: matiwinnetou

.env.IntegrationTest

@@ -64,6 +64,7 @@ REMOVE_SPENT_UTXOS=false
 #The number of safe blocks to keep in the store. 2160 blocks *(20 seconds/block in average)=4320 seconds=12 hours.
 REMOVE_SPENT_UTXOS_LAST_BLOCKS_GRACE_COUNT=2160
 BLOCK_TRANSACTION_API_TIMEOUT_SECS=5
+REMOVE_SPENT_UTXOS_BATCH_SIZE=3000
 
 YACI_SPRING_PROFILES=postgres,n2c-socat
 # database profiles: h2, h2-testdata, postgres
@@ -79,6 +80,8 @@ HOST_CLUSTER_API_PORT=10000
 HOST_OGMIOS_PORT=1337
 HOST_KUPO_PORT=1442
 HOST_VIEWER_PORT=5173
+PROMETHEUS_PORT=9090
+GRAFANA_PORT=3000
 
 ## Devkit env
 DEVKIT_ENABLED=true
@@ -149,4 +152,14 @@ CONTINUE_PARSING_ON_ERROR=false
 SYNC=false
 
 ## Peer Discovery
-PEER_DISCOVERY=false
+PEER_DISCOVERY=false
+
+## Token Registry
+TOKEN_REGISTRY_ENABLED=false
+TOKEN_REGISTRY_BASE_URL=https://tokens.cardano.org/api
+TOKEN_REGISTRY_CACHE_TTL_HOURS=12
+TOKEN_REGISTRY_LOGO_FETCH=false
+TOKEN_REGISTRY_REQUEST_TIMEOUT_SECONDS=2
+
+## Mithril version for Docker build
+MITHRIL_VERSION=2524.0

.env.docker-compose

@@ -59,9 +59,10 @@ SEARCH_LIMIT=100
 
 ## Yaci Indexer env
 INDEXER_DOCKER_IMAGE_TAG=main
-REMOVE_SPENT_UTXOS=false
-#The number of safe blocks to keep in the store. 2160 blocks *(20 seconds/block in average)=4320 seconds=12 hours.
-REMOVE_SPENT_UTXOS_LAST_BLOCKS_GRACE_COUNT=2160
+REMOVE_SPENT_UTXOS=true
+#The number of safe blocks to keep in the store. 129600 blocks *(20 seconds/block in average)=30 days.
+REMOVE_SPENT_UTXOS_LAST_BLOCKS_GRACE_COUNT=129600
+REMOVE_SPENT_UTXOS_BATCH_SIZE=3000
 BLOCK_TRANSACTION_API_TIMEOUT_SECS=5
 
 YACI_SPRING_PROFILES=postgres,n2c-socket
@@ -109,3 +110,14 @@ SYNC=true
 
 ## Peer Discovery
 PEER_DISCOVERY=false
+
+## Token Registry
+TOKEN_REGISTRY_ENABLED=false
+# your local org token registry, e.g. http://myexchange.org/cardano-token-registry/api
+# https://tokens.cardano.org/api cannot be used as it is not possible to request hundreds of tokens due to security / DDOS attacks
+# if you use it, it will work for some requests but not others
+TOKEN_REGISTRY_BASE_URL=
+TOKEN_REGISTRY_CACHE_TTL_HOURS=12
+# by default fetching of logo is disabled as it can add up to a lot of bytes over the wire
+TOKEN_REGISTRY_LOGO_FETCH=false
+TOKEN_REGISTRY_REQUEST_TIMEOUT_SECONDS=2

.env.docker-compose-preprod

@@ -59,9 +59,10 @@ SEARCH_LIMIT=100
 
 ## Yaci Indexer env
 INDEXER_DOCKER_IMAGE_TAG=main
-REMOVE_SPENT_UTXOS=false
-#The number of safe blocks to keep in the store. 2160 blocks *(20 seconds/block in average)=4320 seconds=12 hours.
-REMOVE_SPENT_UTXOS_LAST_BLOCKS_GRACE_COUNT=2160
+REMOVE_SPENT_UTXOS=true
+#The number of safe blocks to keep in the store. 129600 blocks *(20 seconds/block in average)=30 days
+REMOVE_SPENT_UTXOS_LAST_BLOCKS_GRACE_COUNT=129600
+REMOVE_SPENT_UTXOS_BATCH_SIZE=3000
 BLOCK_TRANSACTION_API_TIMEOUT_SECS=5
 
 YACI_SPRING_PROFILES=postgres,n2c-socket
@@ -108,4 +109,12 @@ GRAFANA_PORT=3000
 POSTGRESQL_EXPORTER_PORT=9187
 
 ## Peer Discovery
-PEER_DISCOVERY=true
+PEER_DISCOVERY=true
+
+## Token Registry
+# Externally hosted token registry is currently only available for mainnet
+TOKEN_REGISTRY_ENABLED=true
+TOKEN_REGISTRY_BASE_URL=http://preview.integrations.cf-systems.internal:8080/api
+TOKEN_REGISTRY_CACHE_TTL_HOURS=1
+TOKEN_REGISTRY_LOGO_FETCH=true
+TOKEN_REGISTRY_REQUEST_TIMEOUT_SECONDS=2

.env.docker-compose-profile-advanced-level

@@ -15,7 +15,8 @@ DB_POSTGRES_MAX_PARALLEL_WORKERS_PER_GATHER=8
 DB_POSTGRES_MAX_PARALLEL_WORKERS=16
 DB_POSTGRES_SEQ_PAGE_COST=0.5
 DB_POSTGRES_JIT=off
-DB_POSTGRES_BGWRITER_LRU_MAXPAGES=100
+DB_POSTGRES_BGWRITER_LRU_MAXPAGES=300
 DB_POSTGRES_BGWRITER_DELAY=200ms
 DB_POSTGRES_WAL_BUFFERS=512MB
 DB_POSTGRES_CHECKPOINT_COMPLETION_TARGET=0.9
+DB_POSTGRES_AUTOVACUUM_MAX_WORKERS=5

.env.docker-compose-profile-mid-level

@@ -17,5 +17,6 @@ DB_POSTGRES_MAX_PARALLEL_WORKERS_PER_GATHER=4
 DB_POSTGRES_MAX_PARALLEL_WORKERS=8
 DB_POSTGRES_SEQ_PAGE_COST=1.0
 DB_POSTGRES_JIT=off
-DB_POSTGRES_BGWRITER_LRU_MAXPAGES=100
+DB_POSTGRES_BGWRITER_LRU_MAXPAGES=200
 DB_POSTGRES_BGWRITER_DELAY=200ms
+DB_POSTGRES_AUTOVACUUM_MAX_WORKERS=5

.env.h2

@@ -40,8 +40,10 @@ SEARCH_LIMIT=100
 
 ## Yaci Indexer env
 INDEXER_DOCKER_IMAGE_TAG=main
-REMOVE_SPENT_UTXOS=false
-REMOVE_SPENT_UTXOS_LAST_BLOCKS_GRACE_COUNT=2160
+REMOVE_SPENT_UTXOS=true
+REMOVE_SPENT_UTXOS_LAST_BLOCKS_GRACE_COUNT=129600
+REMOVE_SPENT_UTXOS_BATCH_SIZE=3000
+
 BLOCK_TRANSACTION_API_TIMEOUT_SECS=5
 
 YACI_SPRING_PROFILES=h2,n2c-socket
@@ -102,3 +104,10 @@ CONTINUE_PARSING_ON_ERROR=false
 
 ## Indexer sync starts after node is at tip. Set false for offline mode.
 SYNC=false
+
+## Token Registry
+TOKEN_REGISTRY_ENABLED=false
+TOKEN_REGISTRY_BASE_URL=https://tokens.cardano.org/api
+TOKEN_REGISTRY_CACHE_TTL_HOURS=12
+TOKEN_REGISTRY_LOGO_FETCH=false
+TOKEN_REGISTRY_REQUEST_TIMEOUT_SECONDS=2

.env.h2-testdata

@@ -40,9 +40,11 @@ SEARCH_LIMIT=100
 
 ## Yaci Indexer env
 INDEXER_DOCKER_IMAGE_TAG=main
-REMOVE_SPENT_UTXOS=false
-#The number of safe blocks to keep in the store. 2160 blocks *(20 seconds/block in average)=4320 seconds=12 hours.
-REMOVE_SPENT_UTXOS_LAST_BLOCKS_GRACE_COUNT=2160
+REMOVE_SPENT_UTXOS=true
+#The number of safe blocks to keep in the store. 129600 blocks *(20 seconds/block in average)=30 days.
+REMOVE_SPENT_UTXOS_LAST_BLOCKS_GRACE_COUNT=129600
+REMOVE_SPENT_UTXOS_BATCH_SIZE=3000
+
 BLOCK_TRANSACTION_API_TIMEOUT_SECS=5
 
 YACI_SPRING_PROFILES=h2-testdata,n2c-socket
@@ -103,3 +105,10 @@ CONTINUE_PARSING_ON_ERROR=false
 
 ## Indexer sync starts after node is at tip. Set false for offline mode.
 SYNC=false
+
+## Token Registry
+TOKEN_REGISTRY_ENABLED=false
+TOKEN_REGISTRY_BASE_URL=https://tokens.cardano.org/api
+TOKEN_REGISTRY_CACHE_TTL_HOURS=12
+TOKEN_REGISTRY_LOGO_FETCH=false
+TOKEN_REGISTRY_REQUEST_TIMEOUT_SECONDS=2

.github/workflows/pr-preprod-tests.yaml

@@ -18,7 +18,7 @@ jobs:
     timeout-minutes: 30
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Update local repository
         run: |
@@ -39,10 +39,35 @@ jobs:
           # Stop all services
           docker compose \
             --env-file .env.docker-compose-preprod \
-            --env-file .env.docker-compose-profile-entry-level \
+            --env-file .env.docker-compose-profile-mid-level \
             -f docker-compose.yaml \
             down
 
+      - name: Configure environment for full-history tests
+        run: |
+          cd /home/integration/git/cardano-rosetta-java
+
+          ensure_var() {
+            local key="$1"
+            local value="$2"
+            local file=".env.docker-compose-preprod"
+            if grep -q "^${key}=" "$file"; then
+              sed -i "s#^${key}=.*#${key}=${value}#" "$file"
+            else
+              echo "${key}=${value}" >> "$file"
+            fi
+          }
+
+          ensure_var REMOVE_SPENT_UTXOS false
+          ensure_var REMOVE_SPENT_UTXOS_LAST_BLOCKS_GRACE_COUNT 129600
+          ensure_var DB_PORT 5433
+          ensure_var TOKEN_REGISTRY_ENABLED true
+          ensure_var TOKEN_REGISTRY_BASE_URL http://preview.integrations.cf-systems.internal:8080/api
+          ensure_var TOKEN_REGISTRY_CACHE_TTL_HOURS 1
+          ensure_var TOKEN_REGISTRY_LOGO_FETCH true
+          ensure_var TOKEN_REGISTRY_REQUEST_TIMEOUT_SECONDS 2
+          ensure_var PEER_DISCOVERY true
+
 
       - name: Build and start services with PR code
         run: |
@@ -55,7 +80,7 @@ jobs:
           # Build and start all services
           docker compose \
             --env-file .env.docker-compose-preprod \
-            --env-file .env.docker-compose-profile-entry-level \
+            --env-file .env.docker-compose-profile-mid-level \
             -f docker-compose.yaml \
             up --build -d --wait
 
@@ -110,6 +135,17 @@ jobs:
           # Sync Python dependencies
           uv sync
 
+      - name: Create test environment file
+        run: |
+          cd /home/integration/git/cardano-rosetta-java
+
+          # Merge env files so tests can read actual configuration
+          cat .env.docker-compose-preprod > .env.test
+          cat .env.docker-compose-profile-mid-level >> .env.test
+
+          # Copy to tests directory
+          cp .env.test tests/data-endpoints/.env
+
       - name: Run smoke tests (validate test data)
         id: smoke_tests
         run: |
@@ -142,6 +178,7 @@ jobs:
 
           # Run behavioral tests (skip smoke tests)
           uv run pytest -m "not smoke" \
+            -n auto \
             --alluredir=./allure-results \
             --tb=short \
             -v || TEST_RESULT=$?
@@ -155,6 +192,27 @@ jobs:
           ROSETTA_URL: http://localhost:8082
           CARDANO_NETWORK: preprod
 
+      - name: Run construction API tests
+        id: construction_test
+        run: |
+          export PATH="$HOME/.local/bin:$PATH"
+
+          cd /home/integration/git/cardano-rosetta-java/tests/integration
+
+          # Run construction API snapshot tests
+          uv run test_construction_api.py \
+            -v || CONSTRUCTION_RESULT=$?
+
+          # Output test result
+          echo "construction_result=${CONSTRUCTION_RESULT:-0}" >> $GITHUB_OUTPUT
+
+          # Don't fail the whole job if construction tests fail
+          # These are informational for now
+          exit 0
+        env:
+          ROSETTA_URL: http://localhost:8082
+          CARDANO_NETWORK: preprod
+
       - name: Generate Allure report
         if: always()
         run: |
@@ -217,23 +275,26 @@ jobs:
         if: failure() && (steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled')
         run: |
           cd /home/integration/git/cardano-rosetta-java
+          sed -i "s#^DB_PORT=.*#DB_PORT=5433#" .env.docker-compose-preprod
 
           echo "⚠️ Test failed - stopping services"
 
           # Stop all services cleanly
           docker compose \
             --env-file .env.docker-compose-preprod \
-            --env-file .env.docker-compose-profile-entry-level \
+            --env-file .env.docker-compose-profile-mid-level \
             -f docker-compose.yaml \
             down
 
           # Check if migrations changed in this PR
           LAST_TAG=$(git tag --sort=-version:refname | head -1)
-          git checkout $LAST_TAG
+          git checkout -f $LAST_TAG
+          sed -i "s#^DB_PORT=.*#DB_PORT=5433#" .env.docker-compose-preprod
 
           STABLE_MIGRATION_HASH=$(find yaci-indexer/src/main/resources/db/store -type f -name 'V*.sql' -exec md5sum {} \; | sort | md5sum | cut -d' ' -f1)
 
-          git checkout -
+          git checkout -f -
+          sed -i "s#^DB_PORT=.*#DB_PORT=5433#" .env.docker-compose-preprod
           PR_MIGRATION_HASH=$(find yaci-indexer/src/main/resources/db/store -type f -name 'V*.sql' -exec md5sum {} \; | sort | md5sum | cut -d' ' -f1)
 
           if [ "$STABLE_MIGRATION_HASH" != "$PR_MIGRATION_HASH" ]; then
@@ -242,7 +303,7 @@ jobs:
             # Start only the database
             docker compose \
               --env-file .env.docker-compose-preprod \
-              --env-file .env.docker-compose-profile-entry-level \
+              --env-file .env.docker-compose-profile-mid-level \
               -f docker-compose.yaml \
               up -d db
 
@@ -262,33 +323,34 @@ jobs:
               for VERSION in $CHANGED_MIGRATIONS; do
                 echo "Removing Flyway metadata for version: $VERSION"
                 docker exec cardano-rosetta-java-db-1 sh -c \
-                  "PGPASSWORD=weakpwd#123_d psql -U rosetta_db_admin -d rosetta-java \
+                  "PGPASSWORD=weakpwd#123_d psql -U rosetta_db_admin -d rosetta-java -p 5433 \
                    -c \"DELETE FROM preprod.flyway_schema_history WHERE version LIKE '${VERSION}%';\""
               done
 
               echo "✓ Flyway metadata cleaned - all blockchain data preserved"
             else
               echo "No specific migration versions detected - truncating flyway history"
               docker exec cardano-rosetta-java-db-1 sh -c \
-                "PGPASSWORD=weakpwd#123_d psql -U rosetta_db_admin -d rosetta-java \
+                "PGPASSWORD=weakpwd#123_d psql -U rosetta_db_admin -d rosetta-java -p 5433 \
                  -c 'TRUNCATE preprod.flyway_schema_history;'"
             fi
 
             docker compose \
               --env-file .env.docker-compose-preprod \
-              --env-file .env.docker-compose-profile-entry-level \
+              --env-file .env.docker-compose-profile-mid-level \
               -f docker-compose.yaml \
               down
           fi
 
           # Now safe to rollback
           LAST_TAG=$(git tag --sort=-version:refname | head -1)
           echo "Rolling back to stable version: $LAST_TAG"
-          git checkout $LAST_TAG
+          git checkout -f $LAST_TAG
+          sed -i "s#^DB_PORT=.*#DB_PORT=5433#" .env.docker-compose-preprod
 
           docker compose \
             --env-file .env.docker-compose-preprod \
-            --env-file .env.docker-compose-profile-entry-level \
+            --env-file .env.docker-compose-profile-mid-level \
             -f docker-compose.yaml \
             up -d
 
@@ -305,4 +367,10 @@ jobs:
             sleep 5
           done
 
-          echo "✅ Rollback to $LAST_TAG completed"
+          echo "✅ Rollback to $LAST_TAG completed"
+
+      - name: Restore baseline configuration
+        if: always()
+        run: |
+          cd /home/integration/git/cardano-rosetta-java
+          git restore .env.docker-compose-preprod

CLAUDE.md

@@ -75,6 +75,8 @@ docker exec rosetta tail -f /logs/indexer.log
 - Controller implementations in `api/{domain}/controller/` implement generated interfaces
 - Always use @Nullable annotation in case of optional fields for function methods parameter inputs and outputs, records, DTOs, and entities
 - Avoid if { return } else {} , if we already have a return statement, we can just return the value, no need for else block
+- Use @NotNull annotation everywhere where we can be sure that value will not be null, use @Nullable in case value can be null sometimes
+- Considering that we will have @NotNull and @Nullable annotations, just put nulls checks only when you actually need it, if a field / property is annotated with @NonNull, there is no need for a null check in the code
 
 ### Database Architecture
 - **Hibernate JPA** for standard ORM operations

README.md

@@ -100,15 +100,15 @@ For every Release we provide pre-built docker images stored in the DockerHub Rep
 To start it use the following command:
 
 ```bash
-    docker run --name rosetta -v {CUSTOM_MOUNT_PATH}:/node --env-file ./docker/.env.dockerfile --env-file ./docker/.env.docker-profile-mid-level -p 8082:8082 --shm-size=4g -d cardanofoundation/cardano-rosetta-java:1.3.3
+    docker run --name rosetta -v {CUSTOM_MOUNT_PATH}:/node --env-file ./docker/.env.dockerfile --env-file ./docker/.env.docker-profile-mid-level -p 8082:8082 --shm-size=4g -d cardanofoundation/cardano-rosetta-java:1.4.0
 ```
 
 Changes to the configuration can be made by adjusting the `docker/.env.dockerfile` file. For more information on the environment variables, please refer to the [documentation](https://cardano-foundation.github.io/cardano-rosetta-java/docs/install-and-deploy/env-vars).
 
 If you want to use the `cardano-submit-api` you can additionally expose port `8090`. It can then be used to submit raw cbor transaction (API documentation here: [Link](https://input-output-hk.github.io/cardano-rest/submit-api/))
 
 ```bash
-    docker run --name rosetta -v {CUSTOM_MOUNT_PATH}:/node --env-file ./docker/.env.dockerfile --env-file ./docker/.env.docker-profile-mid-level -p 8090:8090 -p 8082:8082 --shm-size=4g -d cardanofoundation/cardano-rosetta-java:1.3.3
+    docker run --name rosetta -v {CUSTOM_MOUNT_PATH}:/node --env-file ./docker/.env.dockerfile --env-file ./docker/.env.docker-profile-mid-level -p 8090:8090 -p 8082:8082 --shm-size=4g -d cardanofoundation/cardano-rosetta-java:1.4.0
 ```
 
 ### Docker compose