diff --git a/.github/workflows/beam_PreCommit_Java_Kafka_IO_Dataflow.yml b/.github/workflows/beam_PreCommit_Java_Kafka_IO_Dataflow.yml new file mode 100644 index 000000000000..b39eba233cfb --- /dev/null +++ b/.github/workflows/beam_PreCommit_Java_Kafka_IO_Dataflow.yml @@ -0,0 +1,134 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit Java Kafka IO Dataflow + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/kafka/**" + - "sdks/java/testing/test-utils/**" + - "sdks/java/expansion-service/**" + - "sdks/java/io/synthetic/**" + - "sdks/java/io/expansion-service/**" + - ".github/workflows/beam_PreCommit_Java_Kafka_IO_Dataflow.yml" + pull_request_target: + branches: ['master', 'release-*'] + paths: + - "sdks/java/io/kafka/**" + - "sdks/java/testing/test-utils/**" + - "sdks/java/expansion-service/**" + - "sdks/java/io/synthetic/**" + - "sdks/java/io/expansion-service/**" + - 'release/trigger_all_tests.json' + - '.github/trigger_files/beam_PreCommit_Java_Kafka_IO_Direct.json' + issue_comment: + types: [created] + schedule: + - cron: '45 1/6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: write + checks: write + contents: read + deployments: read + id-token: none + issues: write + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_Java_Kafka_IO_Dataflow: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + strategy: + matrix: + job_name: ["beam_PreCommit_Java_Kafka_IO_Dataflow"] + job_phrase: ["Run Java_Kafka_IO_Dataflow PreCommit"] + timeout-minutes: 120 + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + (github.event_name == 'schedule' && github.repository == 'apache/beam') || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Java_Kafka_IO_Dataflow PreCommit' + runs-on: [self-hosted, ubuntu-20.04, main] + steps: + - uses: actions/checkout@v4 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Setup environment + uses: ./.github/actions/setup-environment-action + - name: Authenticate with GCP + uses: google-github-actions/auth@v2 + with: + service_account: ${{ secrets.GCP_SA_EMAIL }} + credentials_json: ${{ secrets.GCP_SA_KEY }} + - name: run Kafka IO build script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:kafka:build :sdks:java:io:kafka:gmk-tests:build :sdks:java:io:kafka:gmk-tests:KafkaOnGmkIT + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + --no-parallel \ + - name: Archive JUnit Test Results + uses: actions/upload-artifact@v4 + if: ${{ !success() }} + with: + name: JUnit Test Results + path: "**/build/reports/tests/" + - name: Publish JUnit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: '${{ env.prsha || env.GITHUB_SHA }}' + comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} + files: '**/build/test-results/**/*.xml' + large_files: true + - name: Archive SpotBugs Results + uses: actions/upload-artifact@v4 + if: always() + with: + name: SpotBugs Results + path: '**/build/reports/spotbugs/*.html' + - name: Publish SpotBugs Results + uses: jwgmeligmeyling/spotbugs-github-action@v1.2 + if: always() + with: + name: Publish SpotBugs + path: '**/build/reports/spotbugs/*.html' diff --git a/sdks/java/io/kafka/gmk-tests/build.gradle b/sdks/java/io/kafka/gmk-tests/build.gradle new file mode 100644 index 000000000000..5cfc950d6ca1 --- /dev/null +++ b/sdks/java/io/kafka/gmk-tests/build.gradle @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.beam.gradle.kafka.KafkaTestUtilities +import java.util.stream.Collectors + +plugins { id 'org.apache.beam.module' } +applyJavaNature( + automaticModuleName: 'org.apache.beam.sdk.io.kafka.gmk-tests', + mavenRepositories: [ + [id: 'io.confluent', url: 'https://packages.confluent.io/maven/'] + ], + publish: 'False' +) + +description = "Apache Beam :: SDKs :: Java :: IO :: Kafka :: GMK Tests" +ext { + summary = "Library to read Kafka topics." + // newer versions e.g. 7.6.* require dropping support for older kafka versions. + confluentVersion = "7.5.5" +} + +provideIntegrationTestingDependencies() + +configurations.all { + resolutionStrategy.force 'org.apache.kafka:kafka-clients:3.9.0' +} + +dependencies { + testImplementation library.java.vendored_guava_32_1_2_jre + provided library.java.jackson_dataformat_csv + permitUnusedDeclared library.java.jackson_dataformat_csv + testImplementation project(path: ":sdks:java:core", configuration: "shadow") + testImplementation project(path: ":model:pipeline", configuration: "shadow") + testImplementation project(":sdks:java:extensions:avro") + testImplementation project(":sdks:java:extensions:protobuf") + testImplementation project(":sdks:java:expansion-service") + permitUnusedDeclared project(":sdks:java:expansion-service") // BEAM-11761 + implementation library.java.avro + // Get back to "provided" since 2.14 + provided library.java.kafka_clients + if (JavaVersion.current().compareTo(JavaVersion.VERSION_21) >= 0) { + // this dependency is a provided dependency for kafka-avro-serializer. It is not needed to compile with Java<=17 + // but needed for compile only under Java21, specifically, required for extending from AbstractKafkaAvroDeserializer + compileOnly library.java.kafka + } + testImplementation library.java.kafka_clients + testImplementation project(path: ":runners:core-java") + testImplementation library.java.slf4j_api + testImplementation library.java.joda_time + testImplementation library.java.jackson_annotations + testImplementation library.java.jackson_databind + testImplementation (group: 'com.google.cloud.hosted.kafka', name: 'managed-kafka-auth-login-handler', version: '1.0.5') { + // "kafka-clients" has to be provided since user can use its own version. + exclude group: 'org.apache.kafka', module: 'kafka-clients' + } + testImplementation ("io.confluent:kafka-avro-serializer:${confluentVersion}") { + // zookeeper depends on "spotbugs-annotations:3.1.9" which clashes with current + // "spotbugs-annotations:3.1.12" used in Beam. Not required. + exclude group: "org.apache.zookeeper", module: "zookeeper" + // "kafka-clients" has to be provided since user can use its own version. + exclude group: "org.apache.kafka", module: "kafka-clients" + } + testImplementation ("io.confluent:kafka-schema-registry-client:${confluentVersion}") { + // It depends on "spotbugs-annotations:3.1.9" which clashes with current + // "spotbugs-annotations:3.1.12" used in Beam. Not required. + exclude group: "org.apache.zookeeper", module: "zookeeper" + // "kafka-clients" has to be provided since user can use its own version. + exclude group: "org.apache.kafka", module: "kafka-clients" + } + // everit_json is needed for Kafka Read SchemaTransform tests that rely on JSON-schema translation. + permitUnusedDeclared library.java.everit_json_schema + provided library.java.everit_json_schema + testImplementation project(path: ":sdks:java:core", configuration: "shadowTest") + testImplementation project(":sdks:java:io:synthetic") + testImplementation project(":sdks:java:managed") + testImplementation project(path: ":sdks:java:extensions:avro", configuration: "testRuntimeMigration") + testImplementation project(path: ":sdks:java:extensions:protobuf", configuration: "testRuntimeMigration") + testImplementation project(path: ":sdks:java:io:common") + testImplementation project(path: ":sdks:java:testing:test-utils") + testImplementation project(path: ":sdks:java:io:kafka") + // For testing Cross-language transforms + testImplementation library.java.avro + testImplementation library.java.junit + testImplementation library.java.mockito_core + testRuntimeOnly library.java.mockito_inline + testImplementation library.java.testcontainers_kafka + testRuntimeOnly library.java.slf4j_jdk14 + testRuntimeOnly project(path: ":runners:direct-java", configuration: "shadow") + testImplementation project(path: ":runners:google-cloud-dataflow-java") + testImplementation project(path: ":sdks:java:extensions:google-cloud-platform-core") + testImplementation project(path: ":sdks:java:io:google-cloud-platform") + testImplementation "org.apache.kafka:kafka-clients:3.9.0" + testImplementation 'org.junit.jupiter:junit-jupiter-api:5.8.1' + testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.8.1' + +} + +tasks.register("kafkaOnGmkIT", KafkaTestUtilities.KafkaBatchIT, "3.9.0", "390", true, configurations, project) \ No newline at end of file