Skip to content

Commit aa4c1c5

Browse files
pinglinpinglin
committed
ci(workflows): adopt short commit hash for service version (#1034)
Because - the service version for the repo commit head at the moment is tagged using the long commit hash. We'd like to keep it short. This commit - update the build-time versioning logic.
1 parent dff15b1 commit aa4c1c5

File tree

11 files changed

+165
-296
lines changed

11 files changed

+165
-296
lines changed

‎.github/workflows/coverage.yml

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,12 @@
11
name: Coverage
22

3-
on: [push, pull_request]
3+
on:
4+
push:
5+
branches:
6+
- main
7+
pull_request:
8+
branches:
9+
- main
410

511
jobs:
612
codecov:

‎.github/workflows/golangci-lint.yml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,11 @@ name: golangci-lint
22

33
on:
44
push:
5+
branches:
6+
- main
57
pull_request:
8+
branches:
9+
- main
610

711
permissions:
812
contents: read

‎.github/workflows/images.yml

Lines changed: 61 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,25 @@ jobs:
4444
username: drop@instill-ai.com
4545
password: ${{ secrets.botDockerHubPassword }}
4646

47+
- name: Set short commit SHA
48+
if: github.ref == 'refs/heads/main'
49+
run: |
50+
echo "COMMIT_SHORT_SHA=${GITHUB_SHA:0:7}" >> $GITHUB_ENV
51+
52+
- name: Build and push amd64 (latest)
53+
if: github.ref == 'refs/heads/main'
54+
uses: docker/build-push-action@v6
55+
with:
56+
platforms: linux/amd64
57+
context: .
58+
push: true
59+
build-args: |
60+
SERVICE_NAME=${{ env.SERVICE_NAME }}
61+
SERVICE_VERSION=${{ env.COMMIT_SHORT_SHA }}
62+
tags: instill/${{ env.SERVICE_NAME }}:latest-amd64
63+
cache-from: type=registry,ref=instill/${{ env.SERVICE_NAME }}:buildcache
64+
cache-to: type=registry,ref=instill/${{ env.SERVICE_NAME }}:buildcache,mode=max
65+
4766
- name: Set Versions
4867
if: github.event_name == 'release' || startsWith(github.ref, 'refs/tags/')
4968
uses: actions/github-script@v6
@@ -55,20 +74,6 @@ jobs:
5574
core.setOutput('tag', tag)
5675
core.setOutput('no_v_tag', no_v_tag)
5776
58-
- name: Build and push amd64 (latest)
59-
if: github.ref == 'refs/heads/main'
60-
uses: docker/build-push-action@v6
61-
with:
62-
platforms: linux/amd64
63-
context: .
64-
push: true
65-
build-args: |
66-
SERVICE_NAME=pipeline-backend
67-
SERVICE_VERSION=${{ github.sha }}
68-
tags: instill/pipeline-backend:latest-amd64
69-
cache-from: type=registry,ref=instill/pipeline-backend:buildcache
70-
cache-to: type=registry,ref=instill/pipeline-backend:buildcache,mode=max
71-
7277
- name: Build and push amd64 (rc/release)
7378
if: startsWith(github.ref, 'refs/tags/') || github.event_name == 'release'
7479
uses: docker/build-push-action@v6
@@ -77,11 +82,11 @@ jobs:
7782
context: .
7883
push: true
7984
build-args: |
80-
SERVICE_NAME=pipeline-backend
85+
SERVICE_NAME=${{ env.SERVICE_NAME }}
8186
SERVICE_VERSION=${{ steps.set_version.outputs.no_v_tag }}
82-
tags: instill/pipeline-backend:${{ steps.set_version.outputs.no_v_tag }}-amd64
83-
cache-from: type=registry,ref=instill/pipeline-backend:buildcache
84-
cache-to: type=registry,ref=instill/pipeline-backend:buildcache,mode=max
87+
tags: instill/${{ env.SERVICE_NAME }}:${{ steps.set_version.outputs.no_v_tag }}-amd64
88+
cache-from: type=registry,ref=instill/${{ env.SERVICE_NAME }}:buildcache
89+
cache-to: type=registry,ref=instill/${{ env.SERVICE_NAME }}:buildcache,mode=max
8590

8691
build-arm64:
8792
runs-on: ubuntu-24.04-arm
@@ -114,6 +119,25 @@ jobs:
114119
username: dropletbot
115120
password: ${{ secrets.botDockerHubPassword }}
116121

122+
- name: Set short commit SHA
123+
if: github.ref == 'refs/heads/main'
124+
run: |
125+
echo "COMMIT_SHORT_SHA=${GITHUB_SHA:0:7}" >> $GITHUB_ENV
126+
127+
- name: Build and push arm64 (latest)
128+
if: github.ref == 'refs/heads/main'
129+
uses: docker/build-push-action@v6
130+
with:
131+
platforms: linux/arm64
132+
context: .
133+
push: true
134+
build-args: |
135+
SERVICE_NAME=${{ env.SERVICE_NAME }}
136+
SERVICE_VERSION=${{ env.COMMIT_SHORT_SHA }}
137+
tags: instill/${{ env.SERVICE_NAME }}:latest-arm64
138+
cache-from: type=registry,ref=instill/${{ env.SERVICE_NAME }}:buildcache
139+
cache-to: type=registry,ref=instill/${{ env.SERVICE_NAME }}:buildcache,mode=max
140+
117141
- name: Set Versions
118142
if: github.event_name == 'release' || startsWith(github.ref, 'refs/tags/')
119143
uses: actions/github-script@v6
@@ -125,20 +149,6 @@ jobs:
125149
core.setOutput('tag', tag)
126150
core.setOutput('no_v_tag', no_v_tag)
127151
128-
- name: Build and push arm64 (latest)
129-
if: github.ref == 'refs/heads/main'
130-
uses: docker/build-push-action@v6
131-
with:
132-
platforms: linux/arm64
133-
context: .
134-
push: true
135-
build-args: |
136-
SERVICE_NAME=pipeline-backend
137-
SERVICE_VERSION=${{ github.sha }}
138-
tags: instill/pipeline-backend:latest-arm64
139-
cache-from: type=registry,ref=instill/pipeline-backend:buildcache
140-
cache-to: type=registry,ref=instill/pipeline-backend:buildcache,mode=max
141-
142152
- name: Build and push arm64 (rc/release)
143153
if: startsWith(github.ref, 'refs/tags/') || github.event_name == 'release'
144154
uses: docker/build-push-action@v6
@@ -147,16 +157,25 @@ jobs:
147157
context: .
148158
push: true
149159
build-args: |
150-
SERVICE_NAME=pipeline-backend
160+
SERVICE_NAME=${{ env.SERVICE_NAME }}
151161
SERVICE_VERSION=${{ steps.set_version.outputs.no_v_tag }}
152-
tags: instill/pipeline-backend:${{ steps.set_version.outputs.no_v_tag }}-arm64
153-
cache-from: type=registry,ref=instill/pipeline-backend:buildcache
154-
cache-to: type=registry,ref=instill/pipeline-backend:buildcache,mode=max
162+
tags: instill/${{ env.SERVICE_NAME }}:${{ steps.set_version.outputs.no_v_tag }}-arm64
163+
cache-from: type=registry,ref=instill/${{ env.SERVICE_NAME }}:buildcache
164+
cache-to: type=registry,ref=instill/${{ env.SERVICE_NAME }}:buildcache,mode=max
155165

156166
merge-manifests:
157167
needs: [build-amd64, build-arm64]
158168
runs-on: ubuntu-24.04
159169
steps:
170+
- uses: actions/checkout@v4
171+
with:
172+
token: ${{ secrets.botGitHubToken }}
173+
174+
- name: Load .env file
175+
uses: cardinalby/export-env-action@v2
176+
with:
177+
envFile: .env
178+
160179
- name: Login to DockerHub
161180
uses: docker/login-action@v3
162181
with:
@@ -166,9 +185,9 @@ jobs:
166185
- name: Create and push multi-arch manifest (latest)
167186
if: github.ref == 'refs/heads/main'
168187
run: |
169-
docker buildx imagetools create -t instill/pipeline-backend:latest \
170-
instill/pipeline-backend:latest-amd64 \
171-
instill/pipeline-backend:latest-arm64
188+
docker buildx imagetools create -t instill/${{ env.SERVICE_NAME }}:latest \
189+
instill/${{ env.SERVICE_NAME }}:latest-amd64 \
190+
instill/${{ env.SERVICE_NAME }}:latest-arm64
172191
173192
- name: Set Versions
174193
if: startsWith(github.ref, 'refs/tags/') || github.event_name == 'release'
@@ -184,6 +203,6 @@ jobs:
184203
- name: Create and push multi-arch manifest (rc/release)
185204
if: startsWith(github.ref, 'refs/tags/') || github.event_name == 'release'
186205
run: |
187-
docker buildx imagetools create -t instill/pipeline-backend:${{ steps.set_version.outputs.no_v_tag }} \
188-
instill/pipeline-backend:${{ steps.set_version.outputs.no_v_tag }}-amd64 \
189-
instill/pipeline-backend:${{ steps.set_version.outputs.no_v_tag }}-arm64
206+
docker buildx imagetools create -t instill/${{ env.SERVICE_NAME }}:${{ steps.set_version.outputs.no_v_tag }} \
207+
instill/${{ env.SERVICE_NAME }}:${{ steps.set_version.outputs.no_v_tag }}-amd64 \
208+
instill/${{ env.SERVICE_NAME }}:${{ steps.set_version.outputs.no_v_tag }}-arm64

‎.github/workflows/integration-test.yml

Lines changed: 10 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -9,14 +9,8 @@ on:
99
- main
1010

1111
jobs:
12-
build-push-image:
13-
if: github.ref == 'refs/heads/main'
14-
name: Build and push image
15-
uses: instill-ai/pipeline-backend/.github/workflows/images.yml@main
16-
secrets: inherit
17-
pr-head:
18-
if: github.event_name == 'pull_request'
19-
name: PR head branch
12+
integration-test:
13+
name: Integration test
2014
runs-on: ubuntu-latest
2115
steps:
2216
- name: Maximize build space
@@ -77,14 +71,19 @@ jobs:
7771
- name: Set up Docker Buildx
7872
uses: docker/setup-buildx-action@v3
7973

74+
- name: Set short commit SHA
75+
if: github.ref == 'refs/heads/main'
76+
run: |
77+
echo "COMMIT_SHORT_SHA=${GITHUB_SHA:0:7}" >> $GITHUB_ENV
78+
8079
- name: Build image
8180
uses: docker/build-push-action@v6
8281
with:
8382
context: pipeline-backend
8483
load: true
8584
build-args: |
86-
SERVICE_NAME=pipeline-backend
87-
SERVICE_VERSION=${{ github.sha }}
85+
SERVICE_NAME=${{ env.SERVICE_NAME }}
86+
SERVICE_VERSION=${{ env.COMMIT_SHORT_SHA }}
8887
tags: instill/pipeline-backend:latest
8988
cache-from: |
9089
type=registry,ref=instill/pipeline-backend:buildcache
@@ -94,7 +93,7 @@ jobs:
9493
- name: Launch Instill Core CE (latest)
9594
working-directory: instill-core
9695
run: |
97-
make latest EDITION=local-ce:test ENV_SECRETS_COMPONENT=.env.secrets.component.test
96+
make latest EDITION=docker-ce:test ENV_SECRETS_COMPONENT=.env.secrets.component.test
9897
9998
- name: Run integration-test
10099
working-directory: pipeline-backend

‎Dockerfile

Lines changed: 20 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -37,37 +37,35 @@ ENV C_INCLUDE_PATH=${ONNXRUNTIME_ROOT_PATH}/include
3737
ENV LD_RUN_PATH=${ONNXRUNTIME_ROOT_PATH}/lib
3838
ENV LIBRARY_PATH=${ONNXRUNTIME_ROOT_PATH}/lib
3939

40-
WORKDIR /src
41-
42-
COPY go.mod go.sum ./
43-
RUN go mod download
44-
COPY . .
40+
WORKDIR /build
4541

4642
ARG SERVICE_NAME SERVICE_VERSION TARGETOS TARGETARCH
47-
RUN --mount=target=. \
43+
44+
RUN --mount=type=bind,target=. \
45+
--mount=type=cache,target=/go/pkg/mod \
4846
--mount=type=cache,target=/root/.cache/go-build \
49-
--mount=type=cache,target=/go/pkg \
5047
GOOS=$TARGETOS GOARCH=$TARGETARCH \
5148
go build -ldflags "-w -X main.version=${SERVICE_VERSION} -X main.serviceName=${SERVICE_NAME}" \
5249
-tags=ocr,onnx -o /${SERVICE_NAME} ./cmd/main
5350

54-
RUN --mount=target=. \
51+
RUN --mount=type=bind,target=. \
52+
--mount=type=cache,target=/go/pkg/mod \
5553
--mount=type=cache,target=/root/.cache/go-build \
56-
--mount=type=cache,target=/go/pkg \
5754
GOOS=$TARGETOS GOARCH=$TARGETARCH \
55+
go mod download && \
5856
go build -ldflags "-w -X main.version=${SERVICE_VERSION} -X main.serviceName=${SERVICE_NAME}-worker" \
5957
-tags=ocr,onnx -o /${SERVICE_NAME}-worker ./cmd/worker
6058

61-
RUN --mount=target=. \
59+
RUN --mount=type=bind,target=. \
60+
--mount=type=cache,target=/go/pkg/mod \
6261
--mount=type=cache,target=/root/.cache/go-build \
63-
--mount=type=cache,target=/go/pkg \
6462
GOOS=$TARGETOS GOARCH=$TARGETARCH \
6563
go build -ldflags "-w -X main.version=${SERVICE_VERSION} -X main.serviceName=${SERVICE_NAME}-migrate" \
6664
-tags=ocr,onnx -o /${SERVICE_NAME}-migrate ./cmd/migration
6765

68-
RUN --mount=target=. \
66+
RUN --mount=type=bind,target=. \
67+
--mount=type=cache,target=/go/pkg/mod \
6968
--mount=type=cache,target=/root/.cache/go-build \
70-
--mount=type=cache,target=/go/pkg \
7169
GOOS=$TARGETOS GOARCH=$TARGETARCH \
7270
go build -ldflags "-w -X main.version=${SERVICE_VERSION} -X main.serviceName=${SERVICE_NAME}-init" \
7371
-tags=ocr,onnx -o /${SERVICE_NAME}-init ./cmd/init
@@ -117,14 +115,14 @@ ENV BASE_DOCLING_PATH=/home/nobody
117115
RUN mkdir -p ${BASE_DOCLING_PATH}/.EasyOCR/model && chown -R nobody:nogroup ${BASE_DOCLING_PATH}
118116

119117
RUN apt update && \
120-
apt install -y wget unzip && \
118+
apt install -y unzip && \
121119
wget https://github.com/JaidedAI/EasyOCR/releases/download/v1.3/latin_g2.zip && \
122120
unzip latin_g2.zip -d ${BASE_DOCLING_PATH}/.EasyOCR/model/ && \
123121
rm latin_g2.zip && \
124122
wget https://github.com/JaidedAI/EasyOCR/releases/download/pre-v1.1.6/craft_mlt_25k.zip && \
125123
unzip craft_mlt_25k.zip -d ${BASE_DOCLING_PATH}/.EasyOCR/model/ && \
126124
rm craft_mlt_25k.zip && \
127-
apt remove -y wget unzip && \
125+
apt remove -y unzip && \
128126
apt autoremove -y && \
129127
rm -rf /var/lib/apt/lists/*
130128

@@ -135,22 +133,24 @@ RUN /opt/venv/bin/python import_artifacts.py && rm import_artifacts.py
135133

136134
USER nobody:nogroup
137135

138-
ARG SERVICE_NAME
136+
ARG SERVICE_NAME SERVICE_VERSION
139137

140138
ENV HOME=${BASE_DOCLING_PATH}
141139
WORKDIR /${SERVICE_NAME}
142140

143141
ENV GODEBUG=tlsrsakex=1
144142

145-
COPY --from=build --chown=nobody:nogroup /src/config ./config
146-
COPY --from=build --chown=nobody:nogroup /src/release-please ./release-please
147-
COPY --from=build --chown=nobody:nogroup /src/pkg/db/migration ./pkg/db/migration
148-
149143
COPY --from=build --chown=nobody:nogroup /${SERVICE_NAME}-migrate ./
150144
COPY --from=build --chown=nobody:nogroup /${SERVICE_NAME}-init ./
151145
COPY --from=build --chown=nobody:nogroup /${SERVICE_NAME}-worker ./
152146
COPY --from=build --chown=nobody:nogroup /${SERVICE_NAME} ./
153147

148+
COPY --chown=nobody:nogroup ./config ./config
149+
COPY --chown=nobody:nogroup ./pkg/db/migration ./pkg/db/migration
150+
154151
# Set up ONNX model and environment variable
155152
COPY --chown=nobody:nogroup ./pkg/component/resources/onnx/silero_vad.onnx /${SERVICE_NAME}/pkg/component/resources/onnx/silero_vad.onnx
156153
ENV ONNX_MODEL_FOLDER_PATH=/${SERVICE_NAME}/pkg/component/resources/onnx
154+
155+
ENV SERVICE_NAME=${SERVICE_NAME}
156+
ENV SERVICE_VERSION=${SERVICE_VERSION}

‎Makefile

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ GOTEST_FLAGS := CFG_DATABASE_HOST=${TEST_DBHOST} CFG_DATABASE_NAME=${TEST_DBNAME
1313
.PHONY: dev
1414
dev: ## Run dev container
1515
@docker compose ls -q | grep -q "instill-core" && true || \
16-
(echo "Error: Run \"make latest PROFILE=exclude-pipeline\" in instill-core repository (https://github.com/instill-ai/instill-core) in your local machine first." && exit 1)
16+
(echo "Error: Run \"make latest\" in instill-core repository (https://github.com/instill-ai/instill-core) in your local machine first and run \"docker rm -f ${SERVICE_NAME} ${SERVICE_NAME}-worker\"." && exit 1)
1717
@docker inspect --type container ${SERVICE_NAME} >/dev/null 2>&1 && echo "A container named ${SERVICE_NAME} is already running." || \
1818
echo "Run dev container ${SERVICE_NAME}. To stop it, run \"make stop\"."
1919
@docker run -d --rm \
@@ -28,12 +28,17 @@ dev: ## Run dev container
2828
.PHONY: latest
2929
latest: ## Run latest container
3030
@docker compose ls -q | grep -q "instill-core" && true || \
31-
(echo "Error: Run \"make latest PROFILE=exclude-pipeline\" in instill-core repository (https://github.com/instill-ai/instill-core) in your local machine first." && exit 1)
31+
(echo "Error: Run \"make latest\" in instill-core repository (https://github.com/instill-ai/instill-core) in your local machine first and run \"docker rm -f ${SERVICE_NAME} ${SERVICE_NAME}-worker\"." && exit 1)
3232
@docker inspect --type container ${SERVICE_NAME} >/dev/null 2>&1 && echo "A container named ${SERVICE_NAME} is already running." || \
3333
echo "Run latest container ${SERVICE_NAME} and ${SERVICE_NAME}-worker. To stop it, run \"make stop\"."
3434
@docker run --network=instill-network \
3535
--name ${SERVICE_NAME} \
36-
-d instill/${SERVICE_NAME}:latest ./${SERVICE_NAME}
36+
-d instill/${SERVICE_NAME}:latest \
37+
/bin/sh -c "\
38+
./${SERVICE_NAME}-migrate && \
39+
./${SERVICE_NAME}-init && \
40+
./${SERVICE_NAME} \
41+
"
3742
@docker run --network=instill-network \
3843
--name ${SERVICE_NAME}-worker \
3944
-d instill/${SERVICE_NAME}:latest ./${SERVICE_NAME}-worker
@@ -56,7 +61,8 @@ build-dev: ## Build dev docker image
5661
build-latest: ## Build latest docker image
5762
@docker build \
5863
--build-arg SERVICE_NAME=${SERVICE_NAME} \
59-
-t instill/pipeline-backend:latest .
64+
--build-arg SERVICE_VERSION=dev \
65+
-t instill/${SERVICE_NAME}:latest .
6066

6167
.PHONY: go-gen
6268
go-gen: ## Generate codes
@@ -116,7 +122,7 @@ test: ## Run unit test
116122
.PHONY: integration-test
117123
integration-test: ## Run integration test
118124
@ # DB_HOST points to localhost by default. Override this variable if
119-
@ # pipeline-backend's database isn't accessible at that host.
125+
@ # ${SERVICE_NAME}'s database isn't accessible at that host.
120126
@TEST_FOLDER_ABS_PATH=${PWD} k6 run \
121127
-e API_GATEWAY_PROTOCOL=${API_GATEWAY_PROTOCOL} \
122128
-e API_GATEWAY_URL=${API_GATEWAY_URL} \

0 commit comments

Comments
 (0)