From 58fcdc8b789529f329713223e14148f32a92bd68 Mon Sep 17 00:00:00 2001 From: Azat Safin Date: Fri, 29 Mar 2024 01:02:59 +0300 Subject: [PATCH 1/8] add Docker file with smile serde --- api/Dockerfile-Serdes | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 api/Dockerfile-Serdes diff --git a/api/Dockerfile-Serdes b/api/Dockerfile-Serdes new file mode 100644 index 000000000..f2d3dafa9 --- /dev/null +++ b/api/Dockerfile-Serdes @@ -0,0 +1,15 @@ +ARG BASE_UI_IMAGE=ghcr.io/kafbat/kafka-ui:main +FROM $BASE_UI_IMAGE + +ARG SERDE_SMILE_JAR=https://github.com/kafbat/ui-serde-smile/releases/download/v1.0.1/serde-smile-v1.0.1-jar-with-dependencies.jar +ARG SERDE_SMILE_VERSION=1.0.1 +USER root +RUN mkdir /usr/lib/serdes; \ + chown -R kafkaui:kafkaui /usr/lib/serdes +USER kafkaui + +RUN cd /usr/lib/serdes; \ + wget -O "kafka-ui-smile-serde-${SERDE_SMILE_VERSION}.jar" $SERDE_SMILE_JAR +ENV kafka.clusters.0.serde.0.name="Smile" +ENV kafka.clusters.0.serde.0.filePath="/usr/lib/serdes/kafka-ui-smile-serde-${SERDE_SMILE_VERSION}.jar" +ENV kafka.clusters.0.serde.0.className="io.kafbat.serde.smile.SmileSerde" \ No newline at end of file From b2d48f403acf2bc259e51c83811a0d4cfa19cc44 Mon Sep 17 00:00:00 2001 From: Azat Safin Date: Tue, 2 Apr 2024 16:15:10 +0300 Subject: [PATCH 2/8] add build args --- .github/workflows/main.yml | 26 ++++++++++++++++++++++++++ api/Dockerfile-Serdes | 31 +++++++++++++++++++++++++++---- api/build-with-serders.sh | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 86 insertions(+), 4 deletions(-) create mode 100755 api/build-with-serders.sh diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 503656b2d..f171cb375 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -76,3 +76,29 @@ jobs: JAR_FILE=api-${{ steps.build.outputs.version }}.jar cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache + + - name: Get latest Serders + id: get_serders + run: | + ./api/build-with-serders.sh + + - name: Build & push docker image with serder + id: docker_build_and_push_with_serdes + uses: docker/build-push-action@v5 + with: + file: Dockerfile-Serdes + builder: ${{ steps.buildx.outputs.name }} + context: api + platforms: linux/amd64,linux/arm64 + provenance: false + push: true + tags: | + ghcr.io/kafbat/kafka-ui-serders:${{ steps.build.outputs.version }} + build-args: | + BASE_UI_IMAGE=ghcr.io/kafbat/kafka-ui:${{ steps.build.outputs.version }} + SERDE_SMILE_JAR=${{ steps.get_serders.outputs.smile_package_url }} + SERDE_SMILE_VERSION=${{ steps.get_serders.outputs.smile_serde_version }} + SERDE_GLUE_JAR=${{ steps.get_serders.outputs.glue_package_url }} + SERDE_GLUE_VERSION=${{ steps.get_serders.outputs.glue_serde_version }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache \ No newline at end of file diff --git a/api/Dockerfile-Serdes b/api/Dockerfile-Serdes index f2d3dafa9..3be3950d7 100644 --- a/api/Dockerfile-Serdes +++ b/api/Dockerfile-Serdes @@ -1,15 +1,38 @@ ARG BASE_UI_IMAGE=ghcr.io/kafbat/kafka-ui:main FROM $BASE_UI_IMAGE -ARG SERDE_SMILE_JAR=https://github.com/kafbat/ui-serde-smile/releases/download/v1.0.1/serde-smile-v1.0.1-jar-with-dependencies.jar -ARG SERDE_SMILE_VERSION=1.0.1 USER root RUN mkdir /usr/lib/serdes; \ chown -R kafkaui:kafkaui /usr/lib/serdes USER kafkaui +# Install Smile Serde +ARG SERDE_SMILE_JAR=https://github.com/kafbat/ui-serde-smile/releases/download/v1.0.1/serde-smile-v1.0.1-jar-with-dependencies.jar +ARG SERDE_SMILE_VERSION=1.0.1 + RUN cd /usr/lib/serdes; \ wget -O "kafka-ui-smile-serde-${SERDE_SMILE_VERSION}.jar" $SERDE_SMILE_JAR + ENV kafka.clusters.0.serde.0.name="Smile" -ENV kafka.clusters.0.serde.0.filePath="/usr/lib/serdes/kafka-ui-smile-serde-${SERDE_SMILE_VERSION}.jar" -ENV kafka.clusters.0.serde.0.className="io.kafbat.serde.smile.SmileSerde" \ No newline at end of file +ENV kafka.clusters.0.serde.0.filePath="/usr/lib/serdes/kafka-ui-serde-smile-${SERDE_SMILE_VERSION}.jar" +ENV kafka.clusters.0.serde.0.className="io.kafbat.serde.smile.SmileSerde" + +# Install Glue Serde +ARG SERDE_GLUE_JAR=https://github.com/kafbat/ui-serde-glue/releases/download/v1.0.1/serde-glue-v1.0.1-jar-with-dependencies.jar +ARG SERDE_GLUE_VERSION=1.0.1 +RUN cd /usr/lib/serdes; \ + wget -O "kafka-ui-serde-glue-${SERDE_GLUE_VERSION}.jar" $SERDE_GLUE_JAR +ENV kafka.clusters.0.serde.1.name="Glue" +ENV kafka.clusters.0.serde.1.filePath="/usr/lib/serdes/kafka-ui-serde-glue-${SERDE_GLUE_VERSION}.jar" +ENV kafka.clusters.0.serde.1.className="io.kafbat.ui.serde.glue.GlueSerde" +ENV kafka.clusters.0.serde.1.properties.region="us-east-1" +ENV kafka.clusters.0.serde.1.properties.registry="kui-test" +# template that will be used to find schema name for topic key. Optional, default is null (not set). +ENV kafka.clusters.0.serde.0.properties.keySchemaNameTemplate="%s-key" +# template that will be used to find schema name for topic value. Optional, default is '%s' +ENV kafka.clusters.0.serde.0.properties.valueSchemaNameTemplate="%s-value" +# schema name -> topics pattern where it will be used for keys. Optional. +ENV kafka.clusters.0.serde.0.properties.topicKeysSchemas.some-topic-key="some-topic1|some-topic2" +# schema name -> topics pattern where it will be used for values. Optional. +ENV kafka.clusters.0.serde.0.properties.topicValuesSchemas.some-topic-value="some-topic1|some-topic2" +ENV kafka.clusters.0.serde.0.properties.topicValuesSchemas.another-topic-val="another-topic-value" \ No newline at end of file diff --git a/api/build-with-serders.sh b/api/build-with-serders.sh new file mode 100755 index 000000000..468fbf426 --- /dev/null +++ b/api/build-with-serders.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# Smile +smile_latest_release=$(curl -s "https://api.github.com/repos/kafbat/ui-serde-smile/releases/latest") +smile_package_url=$(printf '%s' "$smile_latest_release" | jq -r '.assets[0].browser_download_url') +smile_serde_version=$(printf '%s' "$smile_latest_release" | jq -r '.tag_name') +echo "Smile leatest version: $smile_serde_version" +echo "Smile package link: $smile_package_url" +if [[ ! -z "$CI" ]]; then + echo "smile_serde_version=$smile_serde_version" >>$GITHUB_OUTPUT + echo "smile_package_url=$smile_package_url" >>$GITHUB_OUTPUT +fi + +# Glue +glue_latest_release=$(curl -s "https://api.github.com/repos/kafbat/ui-serde-glue/releases/latest") +glue_package_url=$(printf '%s' "$glue_latest_release" | jq -r '.assets[0].browser_download_url') +glue_serde_version=$(printf '%s' "$glue_latest_release" | jq -r '.tag_name') +echo "Glue leatest version: $glue_serde_version" +echo "Glue package link: $glue_package_url" +if [[ ! -z "$CI" ]]; then + echo "glue_serde_version=$glue_serde_version" >>$GITHUB_OUTPUT + echo "glue_package_url=$glue_package_url" >>$GITHUB_OUTPUT +fi + +# Run build if --build option passed +if [ "$1" == "--build" ]; then + echo "Run build in local docker" + docker build -f ./Dockerfile-Serdes --no-cache -t kafka-ui-serders \ + --build-arg SERDE_SMILE_JAR=$smile_package_url \ + --build-arg SERDE_SMILE_VERSION=$smile_serde_version \ + --build-arg SERDE_GLUE_JAR=$glue_package_url \ + --build-arg SERDE_GLUE_VERSION=$glue_serde_version \ + . +fi From 5c4326a1c308cf92cf673dfb573f26c05f5ac48f Mon Sep 17 00:00:00 2001 From: Azat Safin Date: Tue, 2 Apr 2024 16:17:52 +0300 Subject: [PATCH 3/8] add empty line --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f171cb375..99f20b19d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -101,4 +101,4 @@ jobs: SERDE_GLUE_JAR=${{ steps.get_serders.outputs.glue_package_url }} SERDE_GLUE_VERSION=${{ steps.get_serders.outputs.glue_serde_version }} cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache \ No newline at end of file + cache-to: type=local,dest=/tmp/.buildx-cache From f8d5ef7cd6665e8e1e6e4fb1d3e8fcc8747d40af Mon Sep 17 00:00:00 2001 From: Azat Safin Date: Tue, 2 Apr 2024 17:14:41 +0300 Subject: [PATCH 4/8] use separate workflow --- .github/workflows/build-serders.yml | 71 +++++++++++++++++++++++++++++ .github/workflows/main.yml | 40 ++++++++-------- 2 files changed, 91 insertions(+), 20 deletions(-) create mode 100644 .github/workflows/build-serders.yml diff --git a/.github/workflows/build-serders.yml b/.github/workflows/build-serders.yml new file mode 100644 index 000000000..8a6203338 --- /dev/null +++ b/.github/workflows/build-serders.yml @@ -0,0 +1,71 @@ +name: "Build Serdes Docker image" + +on: + workflow_dispatch: + +permissions: + contents: read + packages: write + +jobs: + build: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + token: ${{ github.token }} + + # docker images + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Cache Docker layers + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Get latest Serders + id: get_serders + run: | + ./api/build-with-serders.sh + + - name: Build & push docker image with serder + id: docker_build_and_push_with_serdes + uses: docker/build-push-action@v5 + with: + file: Dockerfile-Serdes + builder: ${{ steps.buildx.outputs.name }} + context: api + platforms: linux/amd64,linux/arm64 + provenance: false + push: true + tags: | + ghcr.io/kafbat/kafka-ui-serders:${{ steps.build.outputs.version }} + build-args: | + BASE_UI_IMAGE=ghcr.io/kafbat/kafka-ui:mains + SERDE_SMILE_JAR=${{ steps.get_serders.outputs.smile_package_url }} + SERDE_SMILE_VERSION=${{ steps.get_serders.outputs.smile_serde_version }} + SERDE_GLUE_JAR=${{ steps.get_serders.outputs.glue_package_url }} + SERDE_GLUE_VERSION=${{ steps.get_serders.outputs.glue_serde_version }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache \ No newline at end of file diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 99f20b19d..ecfb17b6d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -82,23 +82,23 @@ jobs: run: | ./api/build-with-serders.sh - - name: Build & push docker image with serder - id: docker_build_and_push_with_serdes - uses: docker/build-push-action@v5 - with: - file: Dockerfile-Serdes - builder: ${{ steps.buildx.outputs.name }} - context: api - platforms: linux/amd64,linux/arm64 - provenance: false - push: true - tags: | - ghcr.io/kafbat/kafka-ui-serders:${{ steps.build.outputs.version }} - build-args: | - BASE_UI_IMAGE=ghcr.io/kafbat/kafka-ui:${{ steps.build.outputs.version }} - SERDE_SMILE_JAR=${{ steps.get_serders.outputs.smile_package_url }} - SERDE_SMILE_VERSION=${{ steps.get_serders.outputs.smile_serde_version }} - SERDE_GLUE_JAR=${{ steps.get_serders.outputs.glue_package_url }} - SERDE_GLUE_VERSION=${{ steps.get_serders.outputs.glue_serde_version }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache + # - name: Build & push docker image with serder + # id: docker_build_and_push_with_serdes + # uses: docker/build-push-action@v5 + # with: + # file: Dockerfile-Serdes + # builder: ${{ steps.buildx.outputs.name }} + # context: api + # platforms: linux/amd64,linux/arm64 + # provenance: false + # push: true + # tags: | + # ghcr.io/kafbat/kafka-ui-serders:${{ steps.build.outputs.version }} + # build-args: | + # BASE_UI_IMAGE=ghcr.io/kafbat/kafka-ui:${{ steps.build.outputs.version }} + # SERDE_SMILE_JAR=${{ steps.get_serders.outputs.smile_package_url }} + # SERDE_SMILE_VERSION=${{ steps.get_serders.outputs.smile_serde_version }} + # SERDE_GLUE_JAR=${{ steps.get_serders.outputs.glue_package_url }} + # SERDE_GLUE_VERSION=${{ steps.get_serders.outputs.glue_serde_version }} + # cache-from: type=local,src=/tmp/.buildx-cache + # cache-to: type=local,dest=/tmp/.buildx-cache From 003b03e0226bf154b3dcc6153e4a0fbb613bc7b9 Mon Sep 17 00:00:00 2001 From: Azat Safin Date: Tue, 2 Apr 2024 17:15:52 +0300 Subject: [PATCH 5/8] add empty line --- .github/workflows/build-serders.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-serders.yml b/.github/workflows/build-serders.yml index 8a6203338..944399e06 100644 --- a/.github/workflows/build-serders.yml +++ b/.github/workflows/build-serders.yml @@ -68,4 +68,4 @@ jobs: SERDE_GLUE_JAR=${{ steps.get_serders.outputs.glue_package_url }} SERDE_GLUE_VERSION=${{ steps.get_serders.outputs.glue_serde_version }} cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache \ No newline at end of file + cache-to: type=local,dest=/tmp/.buildx-cache From e6600750fc4a3c4c7987b71973319390c1bf0267 Mon Sep 17 00:00:00 2001 From: Azat Safin Date: Tue, 2 Apr 2024 17:16:58 +0300 Subject: [PATCH 6/8] remove from main --- .github/workflows/main.yml | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ecfb17b6d..61f7853bc 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -77,28 +77,3 @@ jobs: cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache - - name: Get latest Serders - id: get_serders - run: | - ./api/build-with-serders.sh - - # - name: Build & push docker image with serder - # id: docker_build_and_push_with_serdes - # uses: docker/build-push-action@v5 - # with: - # file: Dockerfile-Serdes - # builder: ${{ steps.buildx.outputs.name }} - # context: api - # platforms: linux/amd64,linux/arm64 - # provenance: false - # push: true - # tags: | - # ghcr.io/kafbat/kafka-ui-serders:${{ steps.build.outputs.version }} - # build-args: | - # BASE_UI_IMAGE=ghcr.io/kafbat/kafka-ui:${{ steps.build.outputs.version }} - # SERDE_SMILE_JAR=${{ steps.get_serders.outputs.smile_package_url }} - # SERDE_SMILE_VERSION=${{ steps.get_serders.outputs.smile_serde_version }} - # SERDE_GLUE_JAR=${{ steps.get_serders.outputs.glue_package_url }} - # SERDE_GLUE_VERSION=${{ steps.get_serders.outputs.glue_serde_version }} - # cache-from: type=local,src=/tmp/.buildx-cache - # cache-to: type=local,dest=/tmp/.buildx-cache From 192c053f01b275386ea5cfd855611c0891b84295 Mon Sep 17 00:00:00 2001 From: Azat Safin Date: Tue, 2 Apr 2024 17:49:52 +0300 Subject: [PATCH 7/8] small fixes --- .github/workflows/build-serders.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-serders.yml b/.github/workflows/build-serders.yml index 944399e06..0c6edeb74 100644 --- a/.github/workflows/build-serders.yml +++ b/.github/workflows/build-serders.yml @@ -53,16 +53,15 @@ jobs: id: docker_build_and_push_with_serdes uses: docker/build-push-action@v5 with: - file: Dockerfile-Serdes + file: ./api/Dockerfile-Serdes builder: ${{ steps.buildx.outputs.name }} - context: api platforms: linux/amd64,linux/arm64 provenance: false push: true tags: | - ghcr.io/kafbat/kafka-ui-serders:${{ steps.build.outputs.version }} + ghcr.io/kafbat/kafka-ui-serders:main build-args: | - BASE_UI_IMAGE=ghcr.io/kafbat/kafka-ui:mains + BASE_UI_IMAGE=ghcr.io/kafbat/kafka-ui:main SERDE_SMILE_JAR=${{ steps.get_serders.outputs.smile_package_url }} SERDE_SMILE_VERSION=${{ steps.get_serders.outputs.smile_serde_version }} SERDE_GLUE_JAR=${{ steps.get_serders.outputs.glue_package_url }} From 07ea8371dfbb32b4ef8e8ce7b836b0394a18fecc Mon Sep 17 00:00:00 2001 From: Azat Safin Date: Thu, 11 Apr 2024 20:12:46 +0300 Subject: [PATCH 8/8] fix naming --- .github/workflows/build-serders.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-serders.yml b/.github/workflows/build-serders.yml index 0c6edeb74..36bd79e46 100644 --- a/.github/workflows/build-serders.yml +++ b/.github/workflows/build-serders.yml @@ -44,10 +44,10 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Get latest Serders - id: get_serders + - name: Get latest Serdes + id: get_serdes run: | - ./api/build-with-serders.sh + ./api/build-with-serdes.sh - name: Build & push docker image with serder id: docker_build_and_push_with_serdes @@ -59,12 +59,12 @@ jobs: provenance: false push: true tags: | - ghcr.io/kafbat/kafka-ui-serders:main + ghcr.io/kafbat/kafka-ui-serdes:main build-args: | BASE_UI_IMAGE=ghcr.io/kafbat/kafka-ui:main - SERDE_SMILE_JAR=${{ steps.get_serders.outputs.smile_package_url }} - SERDE_SMILE_VERSION=${{ steps.get_serders.outputs.smile_serde_version }} - SERDE_GLUE_JAR=${{ steps.get_serders.outputs.glue_package_url }} - SERDE_GLUE_VERSION=${{ steps.get_serders.outputs.glue_serde_version }} + SERDE_SMILE_JAR=${{ steps.get_serdes.outputs.smile_package_url }} + SERDE_SMILE_VERSION=${{ steps.get_serdes.outputs.smile_serde_version }} + SERDE_GLUE_JAR=${{ steps.get_serdes.outputs.glue_package_url }} + SERDE_GLUE_VERSION=${{ steps.get_serdes.outputs.glue_serde_version }} cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache