Skip to content

Commit 3a41821

Browse files
aucampiaaucampia
authored
build: docker images for latest release and main branch (#2116)
This PR adds docker image building to the github actions workflow. Two images are built: - unstable: This is built from the current branch - latest: This is built from the version of rdflib in `docker/latest/requirements.txt`, but as this will be updated by dependabot it will essentially always be the latest version. The resulting image references are: - `ghcr.io/rdflib/rdflib:unstable` - `ghcr.io/rdflib/rdflib:latest` with an alias `ghcr.io/rdflib/rdflib:${rdflib_version}`, e.g. `ghcr.io/rdflib/rdflib:6.2.0`. As dependabot is conifgured for both pyton and docker dependencies, any change/update to dependencies will result in a new image being built. To avoid spurious changes in image digests the existing image is used as the cache source, so if the requirements.txt and Dockerfile for `ghcr.io/rdflib/rdflib:latest` has not changed then a new image should not be published for it even if the build runs. I have tested this in my forked repo, but there may be some kinks to still work out.
1 parent 91e9842 commit 3a41821

File tree

11 files changed

+251
-20
lines changed

11 files changed

+251
-20
lines changed

‎.dockerignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,3 +3,5 @@
33
.mypy_cache
44
.pytest_cache
55
.git
6+
docs/
7+
test/

‎.github/dependabot.yml‎

Lines changed: 30 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,32 @@
11
version: 2
22
updates:
3-
- package-ecosystem: pip
4-
directory: "/"
5-
schedule:
6-
interval: weekly
7-
open-pull-requests-limit: 10
8-
ignore:
9-
- dependency-name: sphinx
10-
versions:
11-
- 3.4.3
12-
- 3.5.2
13-
- package-ecosystem: github-actions
14-
directory: "/"
15-
schedule:
16-
interval: weekly
3+
- package-ecosystem: pip
4+
directory: "/"
5+
schedule:
6+
interval: weekly
7+
open-pull-requests-limit: 10
8+
ignore:
9+
- dependency-name: sphinx
10+
versions:
11+
- 3.4.3
12+
- 3.5.2
13+
- package-ecosystem: github-actions
14+
directory: "/"
15+
schedule:
16+
interval: weekly
17+
- package-ecosystem: docker
18+
directory: docker/latest/
19+
schedule:
20+
interval: weekly
21+
- package-ecosystem: pip
22+
directory: docker/latest/
23+
schedule:
24+
interval: weekly
25+
- package-ecosystem: docker
26+
directory: docker/unstable/
27+
schedule:
28+
interval: weekly
29+
- package-ecosystem: pip
30+
directory: docker/unstable/
31+
schedule:
32+
interval: weekly

‎.github/workflows/docker-images.yaml‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
name: Docker Images
2+
3+
on:
4+
push:
5+
branches: ["master"]
6+
pull_request:
7+
workflow_dispatch:
8+
9+
concurrency:
10+
group: ${{ github.workflow }}-${{ github.ref }}
11+
cancel-in-progress: true
12+
13+
jobs:
14+
setup:
15+
permissions:
16+
contents: read
17+
runs-on: ubuntu-latest
18+
outputs:
19+
docker-push: ${{ steps.vars.outputs.docker-push }}
20+
oci-reference: ${{ steps.vars.outputs.oci-reference }}
21+
steps:
22+
- id: vars
23+
run: |
24+
echo ::set-output name=github-ref::${{ github.ref }}
25+
echo ::set-output name=docker-push::${{ github.ref == 'refs/heads/main' }}
26+
echo ::set-output name=oci-reference::ghcr.io/$( echo "${{ github.repository}}" | tr '[:upper:]' '[:lower:]' )
27+
- id: vars-dump
28+
run: |
29+
echo "steps.vars.github-ref = ${{ steps.vars.outputs.github-ref }}"
30+
echo "steps.vars.docker-push = ${{ steps.vars.outputs.docker-push }}"
31+
echo "steps.vars.oci-reference = ${{ steps.vars.outputs.oci-reference }}"
32+
33+
build:
34+
needs: setup
35+
permissions:
36+
contents: read
37+
packages: read
38+
runs-on: ubuntu-latest
39+
steps:
40+
- uses: actions/checkout@v3
41+
with:
42+
fetch-depth: 1
43+
- name: Install Task
44+
uses: arduino/setup-task@v1
45+
with:
46+
repo-token: ${{ secrets.GITHUB_TOKEN }}
47+
- name: Build images
48+
shell: bash
49+
run: |
50+
task \
51+
OCI_REFERENCE="${{ needs.setup.outputs.oci-reference }}" \
52+
docker:latest docker:unstable
53+
push:
54+
if: "${{ needs.setup.outputs.docker-push == 'true' }}"
55+
needs: [ setup, build ]
56+
permissions:
57+
id-token: write
58+
packages: write
59+
contents: read
60+
runs-on: ubuntu-latest
61+
steps:
62+
- uses: actions/checkout@v3
63+
with:
64+
fetch-depth: 1
65+
- name: Install Task
66+
uses: arduino/setup-task@v1
67+
with:
68+
repo-token: ${{ secrets.GITHUB_TOKEN }}
69+
- name: Login to GitHub Container Registry
70+
uses: docker/login-action@v2
71+
with:
72+
registry: ghcr.io
73+
username: ${{ github.actor }}
74+
password: ${{ secrets.GITHUB_TOKEN }}
75+
- name: Push images
76+
shell: bash
77+
run: |
78+
task \
79+
DOCKER_PUSH=true \
80+
OCI_REFERENCE="${{ needs.setup.outputs.oci-reference }}" \
81+
docker:latest docker:unstable

‎.github/workflows/validate.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,10 @@ env:
1111
XDG_CACHE_HOME: ${{ github.workspace }}/cache
1212
PIP_CACHE_DIR: ${{ github.workspace }}/pip-cache
1313

14+
concurrency:
15+
group: ${{ github.workflow }}-${{ github.ref }}
16+
cancel-in-progress: true
17+
1418
jobs:
1519
validate:
1620
permissions:

‎Taskfile.yml‎

Lines changed: 67 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ vars:
1616
VENV_PYTHON: '{{if (eq OS "windows")}}{{.VENV_BINPREFIX}}python.exe{{else}}{{.VENV_BINPREFIX}}python{{end}}'
1717
# The python interpreter to use.
1818
PYTHON: python
19-
_PYTHON: '{{if (mustFromJson .WITH_VENV)}}{{.VENV_PYTHON}}{{else}}{{.PYTHON}}{{end}}'
19+
_PYTHON: "{{if (mustFromJson .WITH_VENV)}}{{.VENV_PYTHON}}{{else}}{{.PYTHON}}{{end}}"
2020
# Truthish values ("true", "1", etc.) results in java being installed as a
2121
# system dependency.
2222
INSTALL_SYSTEM_DEPS_JAVA: "false"
@@ -36,7 +36,9 @@ vars:
3636
# Truthish values ("true", "1", etc.) results in coverage being generated for
3737
# relevant commands.
3838
WITH_COVERAGE: "false"
39-
39+
PIP_COMPILE: pip-compile
40+
DOCKER: docker
41+
OCI_REFERENCE: ghcr.io/rdflib/rdflib
4042
tasks:
4143
install:system-deps:
4244
desc: Install system dependencies
@@ -59,12 +61,12 @@ tasks:
5961
install:tox:
6062
desc: Install tox
6163
cmds:
62-
- '{{._PYTHON | shellQuote}} -m pip install tox {{if (mustFromJson .WITH_GITHUB_ACTIONS)}}tox-gh-actions{{end}}'
64+
- "{{._PYTHON | shellQuote}} -m pip install tox {{if (mustFromJson .WITH_GITHUB_ACTIONS)}}tox-gh-actions{{end}}"
6365

6466
install:pip-deps:
6567
desc: Install pip dependencies
6668
cmds:
67-
- '{{._PYTHON | shellQuote}} -m pip install --upgrade -r requirements.txt -r requirements.dev.txt -r docs/sphinx-requirements.txt {{if (mustFromJson .INSTALL_PIP_EXTRAS)}}-r requirements.dev-extra.txt{{end}}'
69+
- "{{._PYTHON | shellQuote}} -m pip install --upgrade -r requirements.txt -r requirements.dev.txt -r docs/sphinx-requirements.txt {{if (mustFromJson .INSTALL_PIP_EXTRAS)}}-r requirements.dev-extra.txt{{end}}"
6870

6971
install:flake8:
7072
desc: Install flake8 dependencies
@@ -78,7 +80,6 @@ tasks:
7880
- task: install:tox
7981
- task: install:pip-deps
8082

81-
8283
venv:create:
8384
desc: Create a venv to VENV_PATH(={{.VENV_PATH}})
8485
cmds:
@@ -275,7 +276,6 @@ tasks:
275276
COVERAGE_FILE: ".coverage"
276277
- cmd: coveralls
277278

278-
279279
gha:flake8:
280280
desc: GitHub Actions flake8 workflow
281281
cmds:
@@ -287,6 +287,67 @@ tasks:
287287
cmds:
288288
- cmd: "{{._PYTHON | shellQuote}} -m rdflib.tools.rdfpipe {{.CLI_ARGS}}"
289289

290+
pip-compile:
291+
cmds:
292+
- cmd: "{{.PIP_COMPILE}} --quiet --annotate --emit-options {{.CLI_ARGS}}"
293+
docker:prepare:
294+
cmds:
295+
- task: pip-compile
296+
vars:
297+
{ CLI_ARGS: "-o docker/unstable/requirements.txt requirements.txt" }
298+
- task: pip-compile
299+
vars: { CLI_ARGS: "docker/latest/requirements.in" }
300+
docker:unstable:
301+
desc: ...
302+
cmds:
303+
- cmd: |
304+
# fetch for caching ...
305+
{{.DOCKER}} image pull {{.OCI_REFERENCE}}:unstable || :
306+
307+
set -eo pipefail
308+
309+
pipx run --spec=build build --wheel
310+
{{.DOCKER}} buildx build \
311+
--cache-to=type=inline \
312+
--cache-from=type=registry,ref={{.OCI_REFERENCE}}:unstable \
313+
--tag {{.OCI_REFERENCE}}:unstable \
314+
--progress plain \
315+
-f docker/unstable/Dockerfile \
316+
{{.DOCKER_BUILD_ARGS}} \
317+
.
318+
319+
if {{.DOCKER_PUSH | default "false"}}
320+
then
321+
{{.DOCKER}} image push {{.OCI_REFERENCE}}:unstable
322+
fi
323+
docker:latest:
324+
desc: ...
325+
cmds:
326+
- cmd: |
327+
# fetch for caching ...
328+
{{.DOCKER}} image pull {{.OCI_REFERENCE}}:latest || :
329+
330+
set -eo pipefail
331+
332+
{{.DOCKER}} buildx build \
333+
--cache-to=type=inline \
334+
--cache-from=type=registry,ref={{.OCI_REFERENCE}}:latest \
335+
--tag {{.OCI_REFERENCE}}:latest \
336+
--progress plain \
337+
-f docker/latest/Dockerfile \
338+
{{.DOCKER_BUILD_ARGS}} \
339+
.
340+
341+
_latest_rdflib_version=$({{.DOCKER}} run --entrypoint= {{.OCI_REFERENCE}}:latest bash -c 'pip show rdflib | sed -n "s/^Version: //gp"')
342+
echo "_latest_rdflib_version=${_latest_rdflib_version}"
343+
344+
{{.DOCKER}} image tag {{.OCI_REFERENCE}}:latest {{.OCI_REFERENCE}}:${_latest_rdflib_version}
345+
346+
if {{.DOCKER_PUSH | default "false"}}
347+
then
348+
{{.DOCKER}} image push {{.OCI_REFERENCE}}:latest
349+
{{.DOCKER}} image push {{.OCI_REFERENCE}}:${_latest_rdflib_version}
350+
fi
290351
_rimraf:
291352
# This task is a utility task for recursively removing directories, it is
292353
# similar to rm -rf but not identical and it should work wherever there is

‎docker/latest/Dockerfile‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
FROM docker.io/library/python:3.10-slim@sha256:c212a18a48efd4de1d6455f197538563d9cc312775761f14c50866c9a7eabd0e
2+
3+
COPY docker/latest/requirements.txt /var/tmp/build/
4+
5+
RUN \
6+
pip install --no-deps -r /var/tmp/build/requirements.txt && \
7+
rdfpipe --version
8+
9+
ENTRYPOINT [ ]

‎docker/latest/requirements.in‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
rdflib==6.2.0
2+
html5lib

‎docker/latest/requirements.txt‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
#
2+
# This file is autogenerated by pip-compile with python 3.10
3+
# To update, run:
4+
#
5+
# pip-compile docker/latest/requirements.in
6+
#
7+
html5lib==1.1
8+
# via -r docker/latest/requirements.in
9+
isodate==0.6.1
10+
# via rdflib
11+
pyparsing==3.0.9
12+
# via rdflib
13+
rdflib==6.2.0
14+
# via -r docker/latest/requirements.in
15+
six==1.16.0
16+
# via html5lib
17+
webencodings==0.5.1
18+
# via html5lib
19+
20+
# The following packages are considered to be unsafe in a requirements file:
21+
# setuptools
22+

‎docker/unstable/Dockerfile‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
FROM docker.io/library/python:3.10-slim@sha256:c212a18a48efd4de1d6455f197538563d9cc312775761f14c50866c9a7eabd0e
2+
3+
COPY docker/unstable/requirements.txt /var/tmp/build/
4+
5+
RUN \
6+
pip install -r /var/tmp/build/requirements.txt
7+
8+
COPY dist/*.whl /var/tmp/build/
9+
10+
RUN \
11+
pip install --no-deps /var/tmp/build/*.whl && \
12+
rdfpipe --version
13+
14+
ENTRYPOINT [ ]

‎docker/unstable/requirements.txt‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
#
2+
# This file is autogenerated by pip-compile with python 3.10
3+
# To update, run:
4+
#
5+
# pip-compile --output-file=docker/unstable/requirements.txt requirements.txt
6+
#
7+
html5lib==1.1
8+
# via -r requirements.txt
9+
isodate==0.6.1
10+
# via -r requirements.txt
11+
pyparsing==3.0.9
12+
# via -r requirements.txt
13+
six==1.16.0
14+
# via
15+
# html5lib
16+
# isodate
17+
webencodings==0.5.1
18+
# via html5lib
19+

0 commit comments

Comments
 (0)