diff --git a/.github/workflows/build-serders.yml b/.github/workflows/build-serders.yml new file mode 100644 index 000000000..36bd79e46 --- /dev/null +++ b/.github/workflows/build-serders.yml @@ -0,0 +1,70 @@ +name: "Build Serdes Docker image" + +on: + workflow_dispatch: + +permissions: + contents: read + packages: write + +jobs: + build: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + token: ${{ github.token }} + + # docker images + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Cache Docker layers + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Get latest Serdes + id: get_serdes + run: | + ./api/build-with-serdes.sh + + - name: Build & push docker image with serder + id: docker_build_and_push_with_serdes + uses: docker/build-push-action@v5 + with: + file: ./api/Dockerfile-Serdes + builder: ${{ steps.buildx.outputs.name }} + platforms: linux/amd64,linux/arm64 + provenance: false + push: true + tags: | + ghcr.io/kafbat/kafka-ui-serdes:main + build-args: | + BASE_UI_IMAGE=ghcr.io/kafbat/kafka-ui:main + SERDE_SMILE_JAR=${{ steps.get_serdes.outputs.smile_package_url }} + SERDE_SMILE_VERSION=${{ steps.get_serdes.outputs.smile_serde_version }} + SERDE_GLUE_JAR=${{ steps.get_serdes.outputs.glue_package_url }} + SERDE_GLUE_VERSION=${{ steps.get_serdes.outputs.glue_serde_version }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 503656b2d..61f7853bc 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -76,3 +76,4 @@ jobs: JAR_FILE=api-${{ steps.build.outputs.version }}.jar cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache + diff --git a/api/Dockerfile-Serdes b/api/Dockerfile-Serdes new file mode 100644 index 000000000..3be3950d7 --- /dev/null +++ b/api/Dockerfile-Serdes @@ -0,0 +1,38 @@ +ARG BASE_UI_IMAGE=ghcr.io/kafbat/kafka-ui:main +FROM $BASE_UI_IMAGE + +USER root +RUN mkdir /usr/lib/serdes; \ + chown -R kafkaui:kafkaui /usr/lib/serdes +USER kafkaui + +# Install Smile Serde +ARG SERDE_SMILE_JAR=https://github.com/kafbat/ui-serde-smile/releases/download/v1.0.1/serde-smile-v1.0.1-jar-with-dependencies.jar +ARG SERDE_SMILE_VERSION=1.0.1 + +RUN cd /usr/lib/serdes; \ + wget -O "kafka-ui-smile-serde-${SERDE_SMILE_VERSION}.jar" $SERDE_SMILE_JAR + +ENV kafka.clusters.0.serde.0.name="Smile" +ENV kafka.clusters.0.serde.0.filePath="/usr/lib/serdes/kafka-ui-serde-smile-${SERDE_SMILE_VERSION}.jar" +ENV kafka.clusters.0.serde.0.className="io.kafbat.serde.smile.SmileSerde" + +# Install Glue Serde +ARG SERDE_GLUE_JAR=https://github.com/kafbat/ui-serde-glue/releases/download/v1.0.1/serde-glue-v1.0.1-jar-with-dependencies.jar +ARG SERDE_GLUE_VERSION=1.0.1 +RUN cd /usr/lib/serdes; \ + wget -O "kafka-ui-serde-glue-${SERDE_GLUE_VERSION}.jar" $SERDE_GLUE_JAR +ENV kafka.clusters.0.serde.1.name="Glue" +ENV kafka.clusters.0.serde.1.filePath="/usr/lib/serdes/kafka-ui-serde-glue-${SERDE_GLUE_VERSION}.jar" +ENV kafka.clusters.0.serde.1.className="io.kafbat.ui.serde.glue.GlueSerde" +ENV kafka.clusters.0.serde.1.properties.region="us-east-1" +ENV kafka.clusters.0.serde.1.properties.registry="kui-test" +# template that will be used to find schema name for topic key. Optional, default is null (not set). +ENV kafka.clusters.0.serde.0.properties.keySchemaNameTemplate="%s-key" +# template that will be used to find schema name for topic value. Optional, default is '%s' +ENV kafka.clusters.0.serde.0.properties.valueSchemaNameTemplate="%s-value" +# schema name -> topics pattern where it will be used for keys. Optional. +ENV kafka.clusters.0.serde.0.properties.topicKeysSchemas.some-topic-key="some-topic1|some-topic2" +# schema name -> topics pattern where it will be used for values. Optional. +ENV kafka.clusters.0.serde.0.properties.topicValuesSchemas.some-topic-value="some-topic1|some-topic2" +ENV kafka.clusters.0.serde.0.properties.topicValuesSchemas.another-topic-val="another-topic-value" \ No newline at end of file diff --git a/api/build-with-serders.sh b/api/build-with-serders.sh new file mode 100755 index 000000000..468fbf426 --- /dev/null +++ b/api/build-with-serders.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# Smile +smile_latest_release=$(curl -s "https://api.github.com/repos/kafbat/ui-serde-smile/releases/latest") +smile_package_url=$(printf '%s' "$smile_latest_release" | jq -r '.assets[0].browser_download_url') +smile_serde_version=$(printf '%s' "$smile_latest_release" | jq -r '.tag_name') +echo "Smile leatest version: $smile_serde_version" +echo "Smile package link: $smile_package_url" +if [[ ! -z "$CI" ]]; then + echo "smile_serde_version=$smile_serde_version" >>$GITHUB_OUTPUT + echo "smile_package_url=$smile_package_url" >>$GITHUB_OUTPUT +fi + +# Glue +glue_latest_release=$(curl -s "https://api.github.com/repos/kafbat/ui-serde-glue/releases/latest") +glue_package_url=$(printf '%s' "$glue_latest_release" | jq -r '.assets[0].browser_download_url') +glue_serde_version=$(printf '%s' "$glue_latest_release" | jq -r '.tag_name') +echo "Glue leatest version: $glue_serde_version" +echo "Glue package link: $glue_package_url" +if [[ ! -z "$CI" ]]; then + echo "glue_serde_version=$glue_serde_version" >>$GITHUB_OUTPUT + echo "glue_package_url=$glue_package_url" >>$GITHUB_OUTPUT +fi + +# Run build if --build option passed +if [ "$1" == "--build" ]; then + echo "Run build in local docker" + docker build -f ./Dockerfile-Serdes --no-cache -t kafka-ui-serders \ + --build-arg SERDE_SMILE_JAR=$smile_package_url \ + --build-arg SERDE_SMILE_VERSION=$smile_serde_version \ + --build-arg SERDE_GLUE_JAR=$glue_package_url \ + --build-arg SERDE_GLUE_VERSION=$glue_serde_version \ + . +fi