Fix docker compose up --wait failing when Trillian server isn't healthy

As noted in docker/compose#12424, compose
--wait doesn't seem to honor healthchecks with restart:always, when the
server crashes and restarts a few times and eventually becomes healthy.
This was happening with Rekor:

* MySQL was not yet healthy because the healthcheck wasn't working as
  expected. docker-library/mysql#930 (comment)
  suggested using 127.0.0.1 instead of localhost
* trillian-log-server was not yet healthy even when MySQL reported as
  healthy, causing trillian-log-server to crash and restart a few times.
  There was no healthcheck for either Trillian service because the image
  we're using is based on Distroless, which has no curl/wget.
* rekor-server tried to start up with an unhealthy trillian-log-server,
  and crashed. The healthcheck reported as unhealthy, and even though
  the server eventually became healthy because of the restart:always
  policy, the healthcheck reported the startup as unhealthy.

This change adds healthchecks to trillian-log-server and log-signer by
pulling the binaries out of the images and putting them into Debian
12 containers that include curl, so we can curl the /healthz endpoint.
This also fixes the MySQL healthcheck as noted above. Now, docker
compose up --wait properly waits for a healthy MySQL before starting
trillian-log-server, and a healthy Trillian before starting Rekor.

Also fix minor Dockerfile linting errors.

Signed-off-by: Hayden B <8418760+haydentherapper@users.noreply.github.com>

Author: haydentherapper

Dockerfile

@@ -31,7 +31,7 @@ RUN CGO_ENABLED=0 go build -gcflags "all=-N -l" -ldflags "${SERVER_LDFLAGS}" -o
 RUN go test -c -ldflags "${SERVER_LDFLAGS}" -cover -covermode=count -coverpkg=./... -o rekor-server_test ./cmd/rekor-server
 
 # Multi-Stage production build
-FROM golang:1.24.2@sha256:30baaea08c5d1e858329c50f29fe381e9b7d7bced11a0f5f1f69a1504cdfbf5e as deploy
+FROM golang:1.24.2@sha256:30baaea08c5d1e858329c50f29fe381e9b7d7bced11a0f5f1f69a1504cdfbf5e AS deploy
 
 # Retrieve the binary from the previous stage
 COPY --from=builder /opt/app-root/src/rekor-server /usr/local/bin/rekor-server
@@ -40,12 +40,12 @@ COPY --from=builder /opt/app-root/src/rekor-server /usr/local/bin/rekor-server
 CMD ["rekor-server", "serve"]
 
 # debug compile options & debugger
-FROM deploy as debug
+FROM deploy AS debug
 RUN go install github.com/go-delve/delve/cmd/dlv@v1.22.1
 
 # overwrite server and include debugger
 COPY --from=builder /opt/app-root/src/rekor-server_debug /usr/local/bin/rekor-server
 
-FROM deploy as test
+FROM deploy AS test
 # overwrite server with test build with code coverage
 COPY --from=builder /opt/app-root/src/rekor-server_test /usr/local/bin/rekor-server

Dockerfile.trillian-log-server

@@ -0,0 +1,21 @@
+# Copyright 2025 The Sigstore Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM ghcr.io/sigstore/scaffolding/trillian_log_server:v1.7.2@sha256:ff64f73b4a8acae7546ecfb5b73c90933b614130a3b43c764a35535e4f60451b AS server
+
+FROM golang:1.24.2@sha256:30baaea08c5d1e858329c50f29fe381e9b7d7bced11a0f5f1f69a1504cdfbf5e AS deploy
+
+COPY --from=server /ko-app/trillian_log_server /usr/local/bin/trillian-log-server
+
+ENTRYPOINT ["trillian-log-server"]

Dockerfile.trillian-log-signer

@@ -0,0 +1,21 @@
+# Copyright 2025 The Sigstore Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM ghcr.io/sigstore/scaffolding/trillian_log_signer:v1.7.2@sha256:bfcc659dc08f87a0f4a4797edf88c93426a95f0d004032779a028bdce7b7e821 AS server
+
+FROM golang:1.24.2@sha256:30baaea08c5d1e858329c50f29fe381e9b7d7bced11a0f5f1f69a1504cdfbf5e AS deploy
+
+COPY --from=server /ko-app/trillian_log_signer /usr/local/bin/trillian-log-signer
+
+ENTRYPOINT ["trillian-log-signer"]

docker-compose.yml

@@ -24,9 +24,9 @@ services:
       - MYSQL_PASSWORD=zaphod
     restart: always # keep the MySQL server running
     healthcheck:
-      # better healthcheck for mysql. See https://github.com/docker-library/mysql/issues/930.
-      test: ["CMD", "mysqladmin", "-h", "localhost", "-u$MYSQL_USER",  "-p$MYSQL_ROOT_PASSWORD", "-s", "ping"]
-      interval: 10s
+      # better healthcheck for MySQL. See https://github.com/docker-library/mysql/issues/930.
+      test: ["CMD", "mysqladmin", "-h", "127.0.0.1", "-u$MYSQL_USER",  "-p$MYSQL_ROOT_PASSWORD", "-s", "ping"]
+      interval: 5s
       timeout: 3s
       retries: 15
       start_period: 90s
@@ -50,7 +50,9 @@ services:
       retries: 3
       start_period: 5s
   trillian-log-server:
-    image: ghcr.io/sigstore/scaffolding/trillian_log_server@sha256:beffee16bb07b5cb051dc4e476d3a1063521ed5ae0b670efc7fe6f3507d94d2b # v1.6.0
+    build:
+      context: .
+      dockerfile: Dockerfile.trillian-log-server
     command: [
       "--quota_system=noop",
       "--storage_system=mysql",
@@ -59,15 +61,23 @@ services:
       "--http_endpoint=0.0.0.0:8091",
       "--alsologtostderr",
     ]
-    restart: always # retry while mysql is starting up
+    restart: always # keep the Trillian log server up
     ports:
       - "8090:8090"
       - "8091:8091"
     depends_on:
       mysql:
         condition: service_healthy
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8091/healthz"]
+      interval: 5s
+      timeout: 3s
+      retries: 15
+      start_period: 15s
   trillian-log-signer:
-    image: ghcr.io/sigstore/scaffolding/trillian_log_signer@sha256:79d57af375cfa997ed5452cc0c02c0396d909fcc91d11065586f119490aa9214 # v1.6.0
+    build:
+      context: .
+      dockerfile: Dockerfile.trillian-log-signer
     command: [
       "--quota_system=noop",
       "--storage_system=mysql",
@@ -77,12 +87,18 @@ services:
       "--force_master",
       "--alsologtostderr",
     ]
-    restart: always # retry while mysql is starting up
+    restart: always # keep the log signer up
     ports:
       - "8092:8091"
     depends_on:
       mysql:
         condition: service_healthy
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8091/healthz"]
+      interval: 5s
+      timeout: 3s
+      retries: 15
+      start_period: 15s
   rekor-server:
     build:
       context: .
@@ -119,7 +135,7 @@ services:
       redis-server:
         condition: service_healthy
       trillian-log-server:
-        condition: service_started
+        condition: service_healthy
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:3000/ping"]
       interval: 10s

tests/client-algos-e2e-test.sh

@@ -30,7 +30,7 @@ function waitForRekorServer () {
   echo -n "* waiting up to 60 sec for system to start"
   count=0
 
-  until [ $(docker ps -a | grep -c "(healthy)") == 3 ];
+  until [ $(docker ps -a | grep -c "(healthy)") == 5 ];
   do
       if [ $count -eq 6 ]; then
         echo "! timeout reached"

tests/index-test-utils.sh

@@ -73,8 +73,8 @@ docker_up () {
     local count=0
     echo "waiting up to 2 min for system to start"
     until [ $(${docker_compose} ps | \
-       grep -E "(rekor[-_]mysql|rekor[-_]redis|rekor[-_]rekor-server)" | \
-       grep -c "(healthy)" ) == 3 ];
+       grep -E "(rekor[-_]mysql|rekor[-_]redis|rekor[-_]rekor-server|rekor[-_]trillian)" | \
+       grep -c "(healthy)" ) == 5 ];
     do
         if [ $count -eq 24 ]; then
            echo "! timeout reached"

tests/issue-872-e2e-test.sh

@@ -33,7 +33,7 @@ function waitForRekorServer () {
   echo -n "* waiting up to 60 sec for system to start"
   count=0
 
-  until [ $(docker ps -a | grep -c "(healthy)") == 3 ];
+  until [ $(docker ps -a | grep -c "(healthy)") == 5 ];
   do
       if [ $count -eq 6 ]; then
         echo "! timeout reached"

tests/rekor-harness.sh

@@ -46,7 +46,7 @@ function start_server () {
 
     count=0
     echo -n "waiting up to 60 sec for system to start"
-    until [ $(${docker_compose} ps | grep -c "(healthy)") == 3 ];
+    until [ $(${docker_compose} ps | grep -c "(healthy)") == 5 ];
     do
         if [ $count -eq 6 ]; then
             echo "! timeout reached"

tests/sharding-e2e-test.sh

@@ -79,7 +79,7 @@ function waitForRekorServer () {
   count=0
 
   echo -n "waiting up to 60 sec for system to start"
-  until [ $(${docker_compose} ps | grep -c "(healthy)") == 3 ];
+  until [ $(${docker_compose} ps | grep -c "(healthy)") == 5 ];
   do
       if [ $count -eq 6 ]; then
         echo "! timeout reached"