From 844e34e01b568336bb8aac5e7a5efe973ce1362e Mon Sep 17 00:00:00 2001 From: Roman Zabaluev Date: Thu, 25 Sep 2025 10:49:37 +0300 Subject: [PATCH 01/20] Impl HeartbeatSerde --- .../kafbat/ui/serdes/SerdesInitializer.java | 4 + .../ui/serdes/builtin/mm2/HeartbeatSerde.java | 106 ++++++++++++++++++ 2 files changed, 110 insertions(+) create mode 100644 api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerde.java diff --git a/api/src/main/java/io/kafbat/ui/serdes/SerdesInitializer.java b/api/src/main/java/io/kafbat/ui/serdes/SerdesInitializer.java index bbdba76c4..2e279d221 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/SerdesInitializer.java +++ b/api/src/main/java/io/kafbat/ui/serdes/SerdesInitializer.java @@ -20,6 +20,7 @@ import io.kafbat.ui.serdes.builtin.UInt32Serde; import io.kafbat.ui.serdes.builtin.UInt64Serde; import io.kafbat.ui.serdes.builtin.UuidBinarySerde; +import io.kafbat.ui.serdes.builtin.mm2.HeartbeatSerde; import io.kafbat.ui.serdes.builtin.sr.SchemaRegistrySerde; import java.util.LinkedHashMap; import java.util.Map; @@ -51,6 +52,9 @@ public SerdesInitializer() { .put(HexSerde.name(), HexSerde.class) .put(UuidBinarySerde.name(), UuidBinarySerde.class) .put(ProtobufRawSerde.name(), ProtobufRawSerde.class) + + // mm2 serdes + .put(HeartbeatSerde.name(), HeartbeatSerde.class) .build(), new CustomSerdeLoader() ); diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerde.java new file mode 100644 index 000000000..6fea68320 --- /dev/null +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerde.java @@ -0,0 +1,106 @@ +package io.kafbat.ui.serdes.builtin.mm2; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.kafbat.ui.serde.api.DeserializeResult; +import io.kafbat.ui.serde.api.SchemaDescription; +import io.kafbat.ui.serdes.BuiltInSerde; +import java.nio.ByteBuffer; +import java.util.Map; +import java.util.Optional; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.common.protocol.types.Field; +import org.apache.kafka.common.protocol.types.Schema; +import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.types.Type; + +@Slf4j +public class HeartbeatSerde implements BuiltInSerde { + + private final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + public static final String SOURCE_CLUSTER_ALIAS_KEY = "sourceClusterAlias"; + public static final String TARGET_CLUSTER_ALIAS_KEY = "targetClusterAlias"; + public static final String TIMESTAMP_KEY = "timestamp"; + public static final String VERSION_KEY = "version"; + public static final short VERSION = 0; + + public static final Schema VALUE_SCHEMA_V0 = new Schema( + new Field(TIMESTAMP_KEY, Type.INT64)); + + public static final Schema KEY_SCHEMA = new Schema( + new Field(SOURCE_CLUSTER_ALIAS_KEY, Type.STRING), + new Field(TARGET_CLUSTER_ALIAS_KEY, Type.STRING)); + + public static final Schema HEADER_SCHEMA = new Schema( + new Field(VERSION_KEY, Type.INT16)); + + public static String name() { + return "Heartbeat"; + } + + @Override + public Optional getDescription() { + return Optional.empty(); + } + + @Override + public Optional getSchema(String topic, Target type) { + return Optional.empty(); + } + + @Override + public boolean canDeserialize(String topic, Target type) { + return true; + } + + @Override + public boolean canSerialize(String topic, Target type) { + return false; + } + + @Override + public Serializer serializer(String topic, Target type) { + throw new UnsupportedOperationException(); + } + + @Override + public Deserializer deserializer(String topic, Target target) { + return (recordHeaders, bytes) -> switch (target) { + + case KEY: { + Struct keyStruct = KEY_SCHEMA.read(ByteBuffer.wrap(bytes)); + String sourceClusterAlias = keyStruct.getString(SOURCE_CLUSTER_ALIAS_KEY); + String targetClusterAlias = keyStruct.getString(TARGET_CLUSTER_ALIAS_KEY); + + var map = Map.of( + "sourceClusterAlias", sourceClusterAlias, + "targetClusterAlias", targetClusterAlias + ); + + try { + var result = OBJECT_MAPPER.writeValueAsString(map); + yield new DeserializeResult(result, DeserializeResult.Type.STRING, Map.of()); + } catch (JsonProcessingException e) { + log.error("Error serializing record", e); + throw new RuntimeException(e); + } + } + + case VALUE: { + ByteBuffer value = ByteBuffer.wrap(bytes); + Struct headerStruct = HEADER_SCHEMA.read(value); + short version = headerStruct.getShort(VERSION_KEY); + Struct valueStruct = valueSchema(version).read(value); + long timestamp = valueStruct.getLong(TIMESTAMP_KEY); + yield new DeserializeResult(String.valueOf(timestamp), DeserializeResult.Type.STRING, Map.of()); + } + + }; + } + + private static Schema valueSchema(short version) { + assert version == 0; + return VALUE_SCHEMA_V0; + } +} From 496aa43fea584b2143bda3cfd65243c925b9f5c0 Mon Sep 17 00:00:00 2001 From: Roman Zabaluev Date: Thu, 25 Sep 2025 11:25:15 +0300 Subject: [PATCH 02/20] Impl OffsetSyncSerde --- .../kafbat/ui/serdes/SerdesInitializer.java | 2 + .../serdes/builtin/mm2/OffsetSyncSerde.java | 104 ++++++++++++++++++ 2 files changed, 106 insertions(+) create mode 100644 api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/OffsetSyncSerde.java diff --git a/api/src/main/java/io/kafbat/ui/serdes/SerdesInitializer.java b/api/src/main/java/io/kafbat/ui/serdes/SerdesInitializer.java index 2e279d221..0b0e6314b 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/SerdesInitializer.java +++ b/api/src/main/java/io/kafbat/ui/serdes/SerdesInitializer.java @@ -21,6 +21,7 @@ import io.kafbat.ui.serdes.builtin.UInt64Serde; import io.kafbat.ui.serdes.builtin.UuidBinarySerde; import io.kafbat.ui.serdes.builtin.mm2.HeartbeatSerde; +import io.kafbat.ui.serdes.builtin.mm2.OffsetSyncSerde; import io.kafbat.ui.serdes.builtin.sr.SchemaRegistrySerde; import java.util.LinkedHashMap; import java.util.Map; @@ -55,6 +56,7 @@ public SerdesInitializer() { // mm2 serdes .put(HeartbeatSerde.name(), HeartbeatSerde.class) + .put(OffsetSyncSerde.name(), OffsetSyncSerde.class) .build(), new CustomSerdeLoader() ); diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/OffsetSyncSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/OffsetSyncSerde.java new file mode 100644 index 000000000..c146a7c40 --- /dev/null +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/OffsetSyncSerde.java @@ -0,0 +1,104 @@ +package io.kafbat.ui.serdes.builtin.mm2; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.kafbat.ui.serde.api.DeserializeResult; +import io.kafbat.ui.serde.api.SchemaDescription; +import io.kafbat.ui.serdes.BuiltInSerde; +import java.nio.ByteBuffer; +import java.util.Map; +import java.util.Optional; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.common.protocol.types.Field; +import org.apache.kafka.common.protocol.types.Schema; +import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.types.Type; + +@Slf4j +public class OffsetSyncSerde implements BuiltInSerde { + + private final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + public static final String TOPIC_KEY = "topic"; + public static final String PARTITION_KEY = "partition"; + public static final String UPSTREAM_OFFSET_KEY = "upstreamOffset"; + public static final String DOWNSTREAM_OFFSET_KEY = "offset"; + public static final Schema VALUE_SCHEMA; + public static final Schema KEY_SCHEMA; + + static { + VALUE_SCHEMA = new Schema(new Field("upstreamOffset", Type.INT64), new Field("offset", Type.INT64)); + KEY_SCHEMA = new Schema(new Field("topic", Type.STRING), new Field("partition", Type.INT32)); + } + + public static String name() { + return "OffsetSync"; + } + + @Override + public Optional getDescription() { + return Optional.empty(); + } + + @Override + public Optional getSchema(String topic, Target type) { + return Optional.empty(); + } + + @Override + public boolean canDeserialize(String topic, Target type) { + return true; + } + + @Override + public boolean canSerialize(String topic, Target type) { + return false; + } + + @Override + public Serializer serializer(String topic, Target type) { + throw new UnsupportedOperationException(); + } + + @Override + public Deserializer deserializer(String topic, Target target) { + return (recordHeaders, bytes) -> switch (target) { + + case KEY: { + Struct keyStruct = KEY_SCHEMA.read(ByteBuffer.wrap(bytes)); + String t = keyStruct.getString(TOPIC_KEY); + int partition = keyStruct.getInt(PARTITION_KEY); + + var map = Map.of( + TOPIC_KEY, t, + PARTITION_KEY, partition + ); + + try { + var result = OBJECT_MAPPER.writeValueAsString(map); + yield new DeserializeResult(result, DeserializeResult.Type.STRING, Map.of()); + } catch (JsonProcessingException e) { + log.error("Error serializing record", e); + throw new RuntimeException(e); + } + } + + case VALUE: { + Struct valueStruct = VALUE_SCHEMA.read(ByteBuffer.wrap(bytes)); + var map = Map.of( + UPSTREAM_OFFSET_KEY, valueStruct.getLong(UPSTREAM_OFFSET_KEY), + DOWNSTREAM_OFFSET_KEY, valueStruct.getLong(DOWNSTREAM_OFFSET_KEY) + ); + + try { + var result = OBJECT_MAPPER.writeValueAsString(map); + yield new DeserializeResult(result, DeserializeResult.Type.STRING, Map.of()); + } catch (JsonProcessingException e) { + log.error("Error serializing record", e); + throw new RuntimeException(e); + } + } + + }; + } +} From 13c951f1201a1ea2df47606d33632e56c5b44759 Mon Sep 17 00:00:00 2001 From: Roman Zabaluev Date: Thu, 25 Sep 2025 11:32:15 +0300 Subject: [PATCH 03/20] Impl CheckpointSerde --- .../kafbat/ui/serdes/SerdesInitializer.java | 2 + .../serdes/builtin/mm2/CheckpointSerde.java | 133 ++++++++++++++++++ 2 files changed, 135 insertions(+) create mode 100644 api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/CheckpointSerde.java diff --git a/api/src/main/java/io/kafbat/ui/serdes/SerdesInitializer.java b/api/src/main/java/io/kafbat/ui/serdes/SerdesInitializer.java index 0b0e6314b..341b75e41 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/SerdesInitializer.java +++ b/api/src/main/java/io/kafbat/ui/serdes/SerdesInitializer.java @@ -20,6 +20,7 @@ import io.kafbat.ui.serdes.builtin.UInt32Serde; import io.kafbat.ui.serdes.builtin.UInt64Serde; import io.kafbat.ui.serdes.builtin.UuidBinarySerde; +import io.kafbat.ui.serdes.builtin.mm2.CheckpointSerde; import io.kafbat.ui.serdes.builtin.mm2.HeartbeatSerde; import io.kafbat.ui.serdes.builtin.mm2.OffsetSyncSerde; import io.kafbat.ui.serdes.builtin.sr.SchemaRegistrySerde; @@ -57,6 +58,7 @@ public SerdesInitializer() { // mm2 serdes .put(HeartbeatSerde.name(), HeartbeatSerde.class) .put(OffsetSyncSerde.name(), OffsetSyncSerde.class) + .put(CheckpointSerde.name(), CheckpointSerde.class) .build(), new CustomSerdeLoader() ); diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/CheckpointSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/CheckpointSerde.java new file mode 100644 index 000000000..092167c1e --- /dev/null +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/CheckpointSerde.java @@ -0,0 +1,133 @@ +package io.kafbat.ui.serdes.builtin.mm2; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.kafbat.ui.serde.api.DeserializeResult; +import io.kafbat.ui.serde.api.SchemaDescription; +import io.kafbat.ui.serdes.BuiltInSerde; +import java.nio.ByteBuffer; +import java.util.Map; +import java.util.Optional; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.common.protocol.types.Field; +import org.apache.kafka.common.protocol.types.Schema; +import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.types.Type; + +@Slf4j +public class CheckpointSerde implements BuiltInSerde { + + private final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + public static final String TOPIC_KEY = "topic"; + public static final String PARTITION_KEY = "partition"; + public static final String CONSUMER_GROUP_ID_KEY = "group"; + public static final String UPSTREAM_OFFSET_KEY = "upstreamOffset"; + public static final String DOWNSTREAM_OFFSET_KEY = "offset"; + public static final String METADATA_KEY = "metadata"; + public static final String VERSION_KEY = "version"; + public static final short VERSION = 0; + + public static final Schema VALUE_SCHEMA_V0 = new Schema( + new Field(UPSTREAM_OFFSET_KEY, Type.INT64), + new Field(DOWNSTREAM_OFFSET_KEY, Type.INT64), + new Field(METADATA_KEY, Type.STRING)); + + public static final Schema KEY_SCHEMA = new Schema( + new Field(CONSUMER_GROUP_ID_KEY, Type.STRING), + new Field(TOPIC_KEY, Type.STRING), + new Field(PARTITION_KEY, Type.INT32)); + + public static final Schema HEADER_SCHEMA = new Schema( + new Field(VERSION_KEY, Type.INT16)); + + public static String name() { + return "Checkpoint"; + } + + @Override + public Optional getDescription() { + return Optional.empty(); + } + + @Override + public Optional getSchema(String topic, Target type) { + return Optional.empty(); + } + + @Override + public boolean canDeserialize(String topic, Target type) { + return true; + } + + @Override + public boolean canSerialize(String topic, Target type) { + return false; + } + + @Override + public Serializer serializer(String topic, Target type) { + throw new UnsupportedOperationException(); + } + + @Override + public Deserializer deserializer(String topic, Target target) { + return (recordHeaders, bytes) -> switch (target) { + + case KEY: { + Struct keyStruct = KEY_SCHEMA.read(ByteBuffer.wrap(bytes)); + + String group = keyStruct.getString(CONSUMER_GROUP_ID_KEY); + String t = keyStruct.getString(TOPIC_KEY); + int partition = keyStruct.getInt(PARTITION_KEY); + + var map = Map.of( + CONSUMER_GROUP_ID_KEY, group, + TOPIC_KEY, t, + PARTITION_KEY, partition + ); + + try { + var result = OBJECT_MAPPER.writeValueAsString(map); + yield new DeserializeResult(result, DeserializeResult.Type.STRING, Map.of()); + } catch (JsonProcessingException e) { + log.error("Error serializing record", e); + throw new RuntimeException(e); + } + } + + case VALUE: { + ByteBuffer value = ByteBuffer.wrap(bytes); + Struct header = HEADER_SCHEMA.read(value); + short version = header.getShort(VERSION_KEY); + Schema valueSchema = valueSchema(version); + Struct valueStruct = valueSchema.read(value); + + long upstreamOffset = valueStruct.getLong(UPSTREAM_OFFSET_KEY); + long downstreamOffset = valueStruct.getLong(DOWNSTREAM_OFFSET_KEY); + String metadata = valueStruct.getString(METADATA_KEY); + + var map = Map.of( + UPSTREAM_OFFSET_KEY, upstreamOffset, + DOWNSTREAM_OFFSET_KEY, downstreamOffset, + METADATA_KEY, metadata + ); + + try { + var result = OBJECT_MAPPER.writeValueAsString(map); + yield new DeserializeResult(result, DeserializeResult.Type.STRING, Map.of()); + } catch (JsonProcessingException e) { + log.error("Error serializing record", e); + throw new RuntimeException(e); + } + } + + }; + } + + private static Schema valueSchema(short version) { + assert version == 0; + return VALUE_SCHEMA_V0; + } + +} From c25604bcbe29b512d63d5543545f07ad666240f7 Mon Sep 17 00:00:00 2001 From: Roman Zabaluev Date: Thu, 25 Sep 2025 17:44:02 +0300 Subject: [PATCH 04/20] lint --- .../serdes/builtin/mm2/CheckpointSerde.java | 105 +++++++++--------- .../ui/serdes/builtin/mm2/HeartbeatSerde.java | 65 +++++------ .../serdes/builtin/mm2/OffsetSyncSerde.java | 79 ++++++------- 3 files changed, 126 insertions(+), 123 deletions(-) diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/CheckpointSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/CheckpointSerde.java index 092167c1e..09df73868 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/CheckpointSerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/CheckpointSerde.java @@ -17,7 +17,7 @@ @Slf4j public class CheckpointSerde implements BuiltInSerde { - private final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); public static final String TOPIC_KEY = "topic"; public static final String PARTITION_KEY = "partition"; @@ -72,57 +72,58 @@ public Serializer serializer(String topic, Target type) { @Override public Deserializer deserializer(String topic, Target target) { - return (recordHeaders, bytes) -> switch (target) { - - case KEY: { - Struct keyStruct = KEY_SCHEMA.read(ByteBuffer.wrap(bytes)); - - String group = keyStruct.getString(CONSUMER_GROUP_ID_KEY); - String t = keyStruct.getString(TOPIC_KEY); - int partition = keyStruct.getInt(PARTITION_KEY); - - var map = Map.of( - CONSUMER_GROUP_ID_KEY, group, - TOPIC_KEY, t, - PARTITION_KEY, partition - ); - - try { - var result = OBJECT_MAPPER.writeValueAsString(map); - yield new DeserializeResult(result, DeserializeResult.Type.STRING, Map.of()); - } catch (JsonProcessingException e) { - log.error("Error serializing record", e); - throw new RuntimeException(e); - } - } - - case VALUE: { - ByteBuffer value = ByteBuffer.wrap(bytes); - Struct header = HEADER_SCHEMA.read(value); - short version = header.getShort(VERSION_KEY); - Schema valueSchema = valueSchema(version); - Struct valueStruct = valueSchema.read(value); - - long upstreamOffset = valueStruct.getLong(UPSTREAM_OFFSET_KEY); - long downstreamOffset = valueStruct.getLong(DOWNSTREAM_OFFSET_KEY); - String metadata = valueStruct.getString(METADATA_KEY); - - var map = Map.of( - UPSTREAM_OFFSET_KEY, upstreamOffset, - DOWNSTREAM_OFFSET_KEY, downstreamOffset, - METADATA_KEY, metadata - ); - - try { - var result = OBJECT_MAPPER.writeValueAsString(map); - yield new DeserializeResult(result, DeserializeResult.Type.STRING, Map.of()); - } catch (JsonProcessingException e) { - log.error("Error serializing record", e); - throw new RuntimeException(e); - } - } - - }; + return (recordHeaders, bytes) -> + switch (target) { + + case KEY: { + Struct keyStruct = KEY_SCHEMA.read(ByteBuffer.wrap(bytes)); + + String group = keyStruct.getString(CONSUMER_GROUP_ID_KEY); + String t = keyStruct.getString(TOPIC_KEY); + int partition = keyStruct.getInt(PARTITION_KEY); + + var map = Map.of( + CONSUMER_GROUP_ID_KEY, group, + TOPIC_KEY, t, + PARTITION_KEY, partition + ); + + try { + var result = OBJECT_MAPPER.writeValueAsString(map); + yield new DeserializeResult(result, DeserializeResult.Type.STRING, Map.of()); + } catch (JsonProcessingException e) { + log.error("Error serializing record", e); + throw new RuntimeException(e); + } + } + + case VALUE: { + ByteBuffer value = ByteBuffer.wrap(bytes); + Struct header = HEADER_SCHEMA.read(value); + short version = header.getShort(VERSION_KEY); + Schema valueSchema = valueSchema(version); + Struct valueStruct = valueSchema.read(value); + + long upstreamOffset = valueStruct.getLong(UPSTREAM_OFFSET_KEY); + long downstreamOffset = valueStruct.getLong(DOWNSTREAM_OFFSET_KEY); + String metadata = valueStruct.getString(METADATA_KEY); + + var map = Map.of( + UPSTREAM_OFFSET_KEY, upstreamOffset, + DOWNSTREAM_OFFSET_KEY, downstreamOffset, + METADATA_KEY, metadata + ); + + try { + var result = OBJECT_MAPPER.writeValueAsString(map); + yield new DeserializeResult(result, DeserializeResult.Type.STRING, Map.of()); + } catch (JsonProcessingException e) { + log.error("Error serializing record", e); + throw new RuntimeException(e); + } + } + + }; } private static Schema valueSchema(short version) { diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerde.java index 6fea68320..b6ce62717 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerde.java @@ -17,7 +17,7 @@ @Slf4j public class HeartbeatSerde implements BuiltInSerde { - private final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); public static final String SOURCE_CLUSTER_ALIAS_KEY = "sourceClusterAlias"; public static final String TARGET_CLUSTER_ALIAS_KEY = "targetClusterAlias"; @@ -66,37 +66,38 @@ public Serializer serializer(String topic, Target type) { @Override public Deserializer deserializer(String topic, Target target) { - return (recordHeaders, bytes) -> switch (target) { - - case KEY: { - Struct keyStruct = KEY_SCHEMA.read(ByteBuffer.wrap(bytes)); - String sourceClusterAlias = keyStruct.getString(SOURCE_CLUSTER_ALIAS_KEY); - String targetClusterAlias = keyStruct.getString(TARGET_CLUSTER_ALIAS_KEY); - - var map = Map.of( - "sourceClusterAlias", sourceClusterAlias, - "targetClusterAlias", targetClusterAlias - ); - - try { - var result = OBJECT_MAPPER.writeValueAsString(map); - yield new DeserializeResult(result, DeserializeResult.Type.STRING, Map.of()); - } catch (JsonProcessingException e) { - log.error("Error serializing record", e); - throw new RuntimeException(e); - } - } - - case VALUE: { - ByteBuffer value = ByteBuffer.wrap(bytes); - Struct headerStruct = HEADER_SCHEMA.read(value); - short version = headerStruct.getShort(VERSION_KEY); - Struct valueStruct = valueSchema(version).read(value); - long timestamp = valueStruct.getLong(TIMESTAMP_KEY); - yield new DeserializeResult(String.valueOf(timestamp), DeserializeResult.Type.STRING, Map.of()); - } - - }; + return (recordHeaders, bytes) -> + switch (target) { + + case KEY: { + Struct keyStruct = KEY_SCHEMA.read(ByteBuffer.wrap(bytes)); + String sourceClusterAlias = keyStruct.getString(SOURCE_CLUSTER_ALIAS_KEY); + String targetClusterAlias = keyStruct.getString(TARGET_CLUSTER_ALIAS_KEY); + + var map = Map.of( + "sourceClusterAlias", sourceClusterAlias, + "targetClusterAlias", targetClusterAlias + ); + + try { + var result = OBJECT_MAPPER.writeValueAsString(map); + yield new DeserializeResult(result, DeserializeResult.Type.STRING, Map.of()); + } catch (JsonProcessingException e) { + log.error("Error serializing record", e); + throw new RuntimeException(e); + } + } + + case VALUE: { + ByteBuffer value = ByteBuffer.wrap(bytes); + Struct headerStruct = HEADER_SCHEMA.read(value); + short version = headerStruct.getShort(VERSION_KEY); + Struct valueStruct = valueSchema(version).read(value); + long timestamp = valueStruct.getLong(TIMESTAMP_KEY); + yield new DeserializeResult(String.valueOf(timestamp), DeserializeResult.Type.STRING, Map.of()); + } + + }; } private static Schema valueSchema(short version) { diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/OffsetSyncSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/OffsetSyncSerde.java index c146a7c40..aa9ebcf93 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/OffsetSyncSerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/OffsetSyncSerde.java @@ -17,7 +17,7 @@ @Slf4j public class OffsetSyncSerde implements BuiltInSerde { - private final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); public static final String TOPIC_KEY = "topic"; public static final String PARTITION_KEY = "partition"; @@ -62,43 +62,44 @@ public Serializer serializer(String topic, Target type) { @Override public Deserializer deserializer(String topic, Target target) { - return (recordHeaders, bytes) -> switch (target) { - - case KEY: { - Struct keyStruct = KEY_SCHEMA.read(ByteBuffer.wrap(bytes)); - String t = keyStruct.getString(TOPIC_KEY); - int partition = keyStruct.getInt(PARTITION_KEY); - - var map = Map.of( - TOPIC_KEY, t, - PARTITION_KEY, partition - ); - - try { - var result = OBJECT_MAPPER.writeValueAsString(map); - yield new DeserializeResult(result, DeserializeResult.Type.STRING, Map.of()); - } catch (JsonProcessingException e) { - log.error("Error serializing record", e); - throw new RuntimeException(e); - } - } - - case VALUE: { - Struct valueStruct = VALUE_SCHEMA.read(ByteBuffer.wrap(bytes)); - var map = Map.of( - UPSTREAM_OFFSET_KEY, valueStruct.getLong(UPSTREAM_OFFSET_KEY), - DOWNSTREAM_OFFSET_KEY, valueStruct.getLong(DOWNSTREAM_OFFSET_KEY) - ); - - try { - var result = OBJECT_MAPPER.writeValueAsString(map); - yield new DeserializeResult(result, DeserializeResult.Type.STRING, Map.of()); - } catch (JsonProcessingException e) { - log.error("Error serializing record", e); - throw new RuntimeException(e); - } - } - - }; + return (recordHeaders, bytes) -> + switch (target) { + + case KEY: { + Struct keyStruct = KEY_SCHEMA.read(ByteBuffer.wrap(bytes)); + String t = keyStruct.getString(TOPIC_KEY); + int partition = keyStruct.getInt(PARTITION_KEY); + + var map = Map.of( + TOPIC_KEY, t, + PARTITION_KEY, partition + ); + + try { + var result = OBJECT_MAPPER.writeValueAsString(map); + yield new DeserializeResult(result, DeserializeResult.Type.STRING, Map.of()); + } catch (JsonProcessingException e) { + log.error("Error serializing record", e); + throw new RuntimeException(e); + } + } + + case VALUE: { + Struct valueStruct = VALUE_SCHEMA.read(ByteBuffer.wrap(bytes)); + var map = Map.of( + UPSTREAM_OFFSET_KEY, valueStruct.getLong(UPSTREAM_OFFSET_KEY), + DOWNSTREAM_OFFSET_KEY, valueStruct.getLong(DOWNSTREAM_OFFSET_KEY) + ); + + try { + var result = OBJECT_MAPPER.writeValueAsString(map); + yield new DeserializeResult(result, DeserializeResult.Type.STRING, Map.of()); + } catch (JsonProcessingException e) { + log.error("Error serializing record", e); + throw new RuntimeException(e); + } + } + + }; } } From de529244d25353b5e1803e8a32873e79644a2af6 Mon Sep 17 00:00:00 2001 From: Roman Zabaluev Date: Fri, 26 Sep 2025 08:36:44 +0300 Subject: [PATCH 05/20] Change result type from string to json in appropriate cases --- .../java/io/kafbat/ui/serdes/builtin/mm2/CheckpointSerde.java | 4 ++-- .../java/io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerde.java | 2 +- .../java/io/kafbat/ui/serdes/builtin/mm2/OffsetSyncSerde.java | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/CheckpointSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/CheckpointSerde.java index 09df73868..a03669313 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/CheckpointSerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/CheckpointSerde.java @@ -90,7 +90,7 @@ public Deserializer deserializer(String topic, Target target) { try { var result = OBJECT_MAPPER.writeValueAsString(map); - yield new DeserializeResult(result, DeserializeResult.Type.STRING, Map.of()); + yield new DeserializeResult(result, DeserializeResult.Type.JSON, Map.of()); } catch (JsonProcessingException e) { log.error("Error serializing record", e); throw new RuntimeException(e); @@ -116,7 +116,7 @@ public Deserializer deserializer(String topic, Target target) { try { var result = OBJECT_MAPPER.writeValueAsString(map); - yield new DeserializeResult(result, DeserializeResult.Type.STRING, Map.of()); + yield new DeserializeResult(result, DeserializeResult.Type.JSON, Map.of()); } catch (JsonProcessingException e) { log.error("Error serializing record", e); throw new RuntimeException(e); diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerde.java index b6ce62717..3dce7ddcb 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerde.java @@ -81,7 +81,7 @@ public Deserializer deserializer(String topic, Target target) { try { var result = OBJECT_MAPPER.writeValueAsString(map); - yield new DeserializeResult(result, DeserializeResult.Type.STRING, Map.of()); + yield new DeserializeResult(result, DeserializeResult.Type.JSON, Map.of()); } catch (JsonProcessingException e) { log.error("Error serializing record", e); throw new RuntimeException(e); diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/OffsetSyncSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/OffsetSyncSerde.java index aa9ebcf93..ec34262d8 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/OffsetSyncSerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/OffsetSyncSerde.java @@ -77,7 +77,7 @@ public Deserializer deserializer(String topic, Target target) { try { var result = OBJECT_MAPPER.writeValueAsString(map); - yield new DeserializeResult(result, DeserializeResult.Type.STRING, Map.of()); + yield new DeserializeResult(result, DeserializeResult.Type.JSON, Map.of()); } catch (JsonProcessingException e) { log.error("Error serializing record", e); throw new RuntimeException(e); @@ -93,7 +93,7 @@ public Deserializer deserializer(String topic, Target target) { try { var result = OBJECT_MAPPER.writeValueAsString(map); - yield new DeserializeResult(result, DeserializeResult.Type.STRING, Map.of()); + yield new DeserializeResult(result, DeserializeResult.Type.JSON, Map.of()); } catch (JsonProcessingException e) { log.error("Error serializing record", e); throw new RuntimeException(e); From 6146073a82d6f67ab9662e22f5b016806bca4169 Mon Sep 17 00:00:00 2001 From: Roman Zabaluev Date: Fri, 26 Sep 2025 09:26:36 +0300 Subject: [PATCH 06/20] Impl tests --- .../builtin/mm2/CheckpointSerdeTest.java | 65 +++++++++++++++++++ .../builtin/mm2/HeartbeatSerdeTest.java | 59 +++++++++++++++++ .../mm2/MirrorMakerSerdesAbstractTest.java | 26 ++++++++ .../builtin/mm2/OffsetSyncSerdeTest.java | 63 ++++++++++++++++++ 4 files changed, 213 insertions(+) create mode 100644 api/src/test/java/io/kafbat/ui/serdes/builtin/mm2/CheckpointSerdeTest.java create mode 100644 api/src/test/java/io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerdeTest.java create mode 100644 api/src/test/java/io/kafbat/ui/serdes/builtin/mm2/MirrorMakerSerdesAbstractTest.java create mode 100644 api/src/test/java/io/kafbat/ui/serdes/builtin/mm2/OffsetSyncSerdeTest.java diff --git a/api/src/test/java/io/kafbat/ui/serdes/builtin/mm2/CheckpointSerdeTest.java b/api/src/test/java/io/kafbat/ui/serdes/builtin/mm2/CheckpointSerdeTest.java new file mode 100644 index 000000000..51ddb9f67 --- /dev/null +++ b/api/src/test/java/io/kafbat/ui/serdes/builtin/mm2/CheckpointSerdeTest.java @@ -0,0 +1,65 @@ +package io.kafbat.ui.serdes.builtin.mm2; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.core.JsonProcessingException; +import io.kafbat.ui.serde.api.DeserializeResult; +import io.kafbat.ui.serde.api.Serde; +import io.kafbat.ui.serdes.PropertyResolverImpl; +import java.util.Map; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class CheckpointSerdeTest extends MirrorMakerSerdesAbstractTest { + + private static final CheckpointSerde SERDE = new CheckpointSerde(); + + @BeforeEach + void init() { + SERDE.configure( + PropertyResolverImpl.empty(), + PropertyResolverImpl.empty(), + PropertyResolverImpl.empty() + ); + } + + @Test + void testCanDeserialize() { + assertTrue(SERDE.canDeserialize(TOPIC, Serde.Target.KEY)); + assertTrue(SERDE.canDeserialize(TOPIC, Serde.Target.VALUE)); + } + + @Test + void testDeserializeKey() throws JsonProcessingException { + var key = decodeBase64("AAVncm91cAAFdG9waWMAAAAD"); + var expected = Map.of( + "partition", 3, + "topic", "topic", + "group", "group" + ); + + var result = SERDE.deserializer(TOPIC, Serde.Target.KEY).deserialize(HEADERS, key); + var resultMap = jsonToMap(result.getResult()); + + assertEquals(DeserializeResult.Type.JSON, result.getType()); + assertEquals(expected, resultMap); + } + + @Test + void testDeserializeValue() throws JsonProcessingException { + var value = decodeBase64("AAAAAAAAA/bkPgAAAAHz1jf7AAA="); + var expected = Map.of( + "offset", 8385869819L, + "upstreamOffset", 66511934, + "metadata", "" + ); + + var result = SERDE.deserializer(TOPIC, Serde.Target.VALUE).deserialize(HEADERS, value); + var resultMap = jsonToMap(result.getResult()); + + assertEquals(DeserializeResult.Type.JSON, result.getType()); + assertEquals(expected, resultMap); + } + +} diff --git a/api/src/test/java/io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerdeTest.java b/api/src/test/java/io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerdeTest.java new file mode 100644 index 000000000..1e778794f --- /dev/null +++ b/api/src/test/java/io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerdeTest.java @@ -0,0 +1,59 @@ +package io.kafbat.ui.serdes.builtin.mm2; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.core.JsonProcessingException; +import io.kafbat.ui.serde.api.DeserializeResult; +import io.kafbat.ui.serde.api.Serde; +import io.kafbat.ui.serdes.PropertyResolverImpl; +import java.util.Map; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class HeartbeatSerdeTest extends MirrorMakerSerdesAbstractTest { + + private static final HeartbeatSerde SERDE = new HeartbeatSerde(); + + @BeforeEach + void init() { + SERDE.configure( + PropertyResolverImpl.empty(), + PropertyResolverImpl.empty(), + PropertyResolverImpl.empty() + ); + } + + @Test + void testCanDeserialize() { + assertTrue(SERDE.canDeserialize(TOPIC, Serde.Target.KEY)); + assertTrue(SERDE.canDeserialize(TOPIC, Serde.Target.VALUE)); + } + + @Test + void testDeserializeKey() throws JsonProcessingException { + var key = decodeBase64("AAZzb3VyY2UABnRhcmdldA=="); + var expected = Map.of( + "sourceClusterAlias", "source", + "targetClusterAlias", "target" + ); + + var result = SERDE.deserializer(TOPIC, Serde.Target.KEY).deserialize(HEADERS, key); + var resultMap = jsonToMap(result.getResult()); + + assertEquals(DeserializeResult.Type.JSON, result.getType()); + assertEquals(expected, resultMap); + } + + @Test + void testDeserializeValue() { + var value = decodeBase64("AAAAAAGZgCEMZA=="); + var expected = "1758791273572"; + + var result = SERDE.deserializer(TOPIC, Serde.Target.VALUE).deserialize(HEADERS, value); + + assertEquals(DeserializeResult.Type.STRING, result.getType()); + assertEquals(expected, result.getResult()); + } + +} diff --git a/api/src/test/java/io/kafbat/ui/serdes/builtin/mm2/MirrorMakerSerdesAbstractTest.java b/api/src/test/java/io/kafbat/ui/serdes/builtin/mm2/MirrorMakerSerdesAbstractTest.java new file mode 100644 index 000000000..91c6634a2 --- /dev/null +++ b/api/src/test/java/io/kafbat/ui/serdes/builtin/mm2/MirrorMakerSerdesAbstractTest.java @@ -0,0 +1,26 @@ +package io.kafbat.ui.serdes.builtin.mm2; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.kafbat.ui.serdes.RecordHeadersImpl; +import java.util.Base64; +import java.util.Map; + +public abstract class MirrorMakerSerdesAbstractTest { + + protected static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + protected static final String TOPIC = "test-topic"; + protected static final RecordHeadersImpl HEADERS = new RecordHeadersImpl(); + + protected Map jsonToMap(String json) throws JsonProcessingException { + //@formatter:off + return OBJECT_MAPPER.readValue(json, new TypeReference<>() {}); + //@formatter:on + } + + protected static byte[] decodeBase64(String base64) { + return Base64.getDecoder().decode(base64.trim()); + } + +} diff --git a/api/src/test/java/io/kafbat/ui/serdes/builtin/mm2/OffsetSyncSerdeTest.java b/api/src/test/java/io/kafbat/ui/serdes/builtin/mm2/OffsetSyncSerdeTest.java new file mode 100644 index 000000000..447b33e92 --- /dev/null +++ b/api/src/test/java/io/kafbat/ui/serdes/builtin/mm2/OffsetSyncSerdeTest.java @@ -0,0 +1,63 @@ +package io.kafbat.ui.serdes.builtin.mm2; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.core.JsonProcessingException; +import io.kafbat.ui.serde.api.DeserializeResult; +import io.kafbat.ui.serde.api.Serde; +import io.kafbat.ui.serdes.PropertyResolverImpl; +import java.util.Map; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class OffsetSyncSerdeTest extends MirrorMakerSerdesAbstractTest { + + private static final OffsetSyncSerde SERDE = new OffsetSyncSerde(); + + @BeforeEach + void init() { + SERDE.configure( + PropertyResolverImpl.empty(), + PropertyResolverImpl.empty(), + PropertyResolverImpl.empty() + ); + } + + @Test + void testCanDeserialize() { + assertTrue(SERDE.canDeserialize(TOPIC, Serde.Target.KEY)); + assertTrue(SERDE.canDeserialize(TOPIC, Serde.Target.VALUE)); + } + + @Test + void testDeserializeKey() throws JsonProcessingException { + var key = decodeBase64("AAl0b3BpY25hbWUAAAAA"); + var expected = Map.of( + "partition", 0, + "topic", "topicname" + ); + + var result = SERDE.deserializer(TOPIC, Serde.Target.KEY).deserialize(HEADERS, key); + var resultMap = jsonToMap(result.getResult()); + + assertEquals(DeserializeResult.Type.JSON, result.getType()); + assertEquals(expected, resultMap); + } + + @Test + void testDeserializeValue() throws JsonProcessingException { + var value = decodeBase64("AAAAAAACXsoAAAAAAAHMfw=="); + var expected = Map.of( + "offset", 117887, + "upstreamOffset", 155338 + ); + + var result = SERDE.deserializer(TOPIC, Serde.Target.VALUE).deserialize(HEADERS, value); + var resultMap = jsonToMap(result.getResult()); + + assertEquals(DeserializeResult.Type.JSON, result.getType()); + assertEquals(expected, resultMap); + } + +} From fc59e6b7f54120e6fb1062b1c53c5d044eab799e Mon Sep 17 00:00:00 2001 From: Roman Zabaluev Date: Fri, 26 Sep 2025 12:44:29 +0300 Subject: [PATCH 07/20] Logs --- .../io/kafbat/ui/serdes/builtin/mm2/CheckpointSerde.java | 8 ++++---- .../io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerde.java | 4 ++-- .../io/kafbat/ui/serdes/builtin/mm2/OffsetSyncSerde.java | 8 ++++---- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/CheckpointSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/CheckpointSerde.java index a03669313..c9fcf530d 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/CheckpointSerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/CheckpointSerde.java @@ -92,8 +92,8 @@ public Deserializer deserializer(String topic, Target target) { var result = OBJECT_MAPPER.writeValueAsString(map); yield new DeserializeResult(result, DeserializeResult.Type.JSON, Map.of()); } catch (JsonProcessingException e) { - log.error("Error serializing record", e); - throw new RuntimeException(e); + log.error("Error deserializing record", e); + throw new RuntimeException("Error deserializing record", e); } } @@ -118,8 +118,8 @@ public Deserializer deserializer(String topic, Target target) { var result = OBJECT_MAPPER.writeValueAsString(map); yield new DeserializeResult(result, DeserializeResult.Type.JSON, Map.of()); } catch (JsonProcessingException e) { - log.error("Error serializing record", e); - throw new RuntimeException(e); + log.error("Error deserializing record", e); + throw new RuntimeException("Error deserializing record", e); } } diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerde.java index 3dce7ddcb..5f2f4986c 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerde.java @@ -83,8 +83,8 @@ public Deserializer deserializer(String topic, Target target) { var result = OBJECT_MAPPER.writeValueAsString(map); yield new DeserializeResult(result, DeserializeResult.Type.JSON, Map.of()); } catch (JsonProcessingException e) { - log.error("Error serializing record", e); - throw new RuntimeException(e); + log.error("Error deserializing record", e); + throw new RuntimeException("Error deserializing record", e); } } diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/OffsetSyncSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/OffsetSyncSerde.java index ec34262d8..d3c8a6171 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/OffsetSyncSerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/OffsetSyncSerde.java @@ -79,8 +79,8 @@ public Deserializer deserializer(String topic, Target target) { var result = OBJECT_MAPPER.writeValueAsString(map); yield new DeserializeResult(result, DeserializeResult.Type.JSON, Map.of()); } catch (JsonProcessingException e) { - log.error("Error serializing record", e); - throw new RuntimeException(e); + log.error("Error deserializing record", e); + throw new RuntimeException("Error deserializing record", e); } } @@ -95,8 +95,8 @@ public Deserializer deserializer(String topic, Target target) { var result = OBJECT_MAPPER.writeValueAsString(map); yield new DeserializeResult(result, DeserializeResult.Type.JSON, Map.of()); } catch (JsonProcessingException e) { - log.error("Error serializing record", e); - throw new RuntimeException(e); + log.error("Error deserializing record", e); + throw new RuntimeException("Error deserializing record", e); } } From 267d52df2b5a1facca2375d3401d4073a9a40e88 Mon Sep 17 00:00:00 2001 From: Roman Zabaluev Date: Fri, 26 Sep 2025 12:50:15 +0300 Subject: [PATCH 08/20] Split k/v methods --- .../serdes/builtin/mm2/CheckpointSerde.java | 97 ++++++++++--------- .../ui/serdes/builtin/mm2/HeartbeatSerde.java | 57 +++++------ .../serdes/builtin/mm2/OffsetSyncSerde.java | 71 +++++++------- 3 files changed, 114 insertions(+), 111 deletions(-) diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/CheckpointSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/CheckpointSerde.java index c9fcf530d..33b21c33e 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/CheckpointSerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/CheckpointSerde.java @@ -74,56 +74,57 @@ public Serializer serializer(String topic, Target type) { public Deserializer deserializer(String topic, Target target) { return (recordHeaders, bytes) -> switch (target) { + case KEY -> deserializeKey(bytes); + case VALUE -> deserializeValue(bytes); + }; + } - case KEY: { - Struct keyStruct = KEY_SCHEMA.read(ByteBuffer.wrap(bytes)); - - String group = keyStruct.getString(CONSUMER_GROUP_ID_KEY); - String t = keyStruct.getString(TOPIC_KEY); - int partition = keyStruct.getInt(PARTITION_KEY); - - var map = Map.of( - CONSUMER_GROUP_ID_KEY, group, - TOPIC_KEY, t, - PARTITION_KEY, partition - ); - - try { - var result = OBJECT_MAPPER.writeValueAsString(map); - yield new DeserializeResult(result, DeserializeResult.Type.JSON, Map.of()); - } catch (JsonProcessingException e) { - log.error("Error deserializing record", e); - throw new RuntimeException("Error deserializing record", e); - } - } - - case VALUE: { - ByteBuffer value = ByteBuffer.wrap(bytes); - Struct header = HEADER_SCHEMA.read(value); - short version = header.getShort(VERSION_KEY); - Schema valueSchema = valueSchema(version); - Struct valueStruct = valueSchema.read(value); - - long upstreamOffset = valueStruct.getLong(UPSTREAM_OFFSET_KEY); - long downstreamOffset = valueStruct.getLong(DOWNSTREAM_OFFSET_KEY); - String metadata = valueStruct.getString(METADATA_KEY); - - var map = Map.of( - UPSTREAM_OFFSET_KEY, upstreamOffset, - DOWNSTREAM_OFFSET_KEY, downstreamOffset, - METADATA_KEY, metadata - ); - - try { - var result = OBJECT_MAPPER.writeValueAsString(map); - yield new DeserializeResult(result, DeserializeResult.Type.JSON, Map.of()); - } catch (JsonProcessingException e) { - log.error("Error deserializing record", e); - throw new RuntimeException("Error deserializing record", e); - } - } + private static DeserializeResult deserializeKey(byte[] bytes) { + Struct keyStruct = KEY_SCHEMA.read(ByteBuffer.wrap(bytes)); + + String group = keyStruct.getString(CONSUMER_GROUP_ID_KEY); + String t = keyStruct.getString(TOPIC_KEY); + int partition = keyStruct.getInt(PARTITION_KEY); + + var map = Map.of( + CONSUMER_GROUP_ID_KEY, group, + TOPIC_KEY, t, + PARTITION_KEY, partition + ); + + try { + var result = OBJECT_MAPPER.writeValueAsString(map); + return new DeserializeResult(result, DeserializeResult.Type.JSON, Map.of()); + } catch (JsonProcessingException e) { + log.error("Error deserializing record", e); + throw new RuntimeException("Error deserializing record", e); + } + } - }; + private static DeserializeResult deserializeValue(byte[] bytes) { + ByteBuffer value = ByteBuffer.wrap(bytes); + Struct header = HEADER_SCHEMA.read(value); + short version = header.getShort(VERSION_KEY); + Schema valueSchema = valueSchema(version); + Struct valueStruct = valueSchema.read(value); + + long upstreamOffset = valueStruct.getLong(UPSTREAM_OFFSET_KEY); + long downstreamOffset = valueStruct.getLong(DOWNSTREAM_OFFSET_KEY); + String metadata = valueStruct.getString(METADATA_KEY); + + var map = Map.of( + UPSTREAM_OFFSET_KEY, upstreamOffset, + DOWNSTREAM_OFFSET_KEY, downstreamOffset, + METADATA_KEY, metadata + ); + + try { + var result = OBJECT_MAPPER.writeValueAsString(map); + return new DeserializeResult(result, DeserializeResult.Type.JSON, Map.of()); + } catch (JsonProcessingException e) { + log.error("Error deserializing record", e); + throw new RuntimeException("Error deserializing record", e); + } } private static Schema valueSchema(short version) { diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerde.java index 5f2f4986c..893910444 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerde.java @@ -68,36 +68,37 @@ public Serializer serializer(String topic, Target type) { public Deserializer deserializer(String topic, Target target) { return (recordHeaders, bytes) -> switch (target) { + case KEY -> deserializeKey(bytes); + case VALUE -> deserializeValue(bytes); + }; + } - case KEY: { - Struct keyStruct = KEY_SCHEMA.read(ByteBuffer.wrap(bytes)); - String sourceClusterAlias = keyStruct.getString(SOURCE_CLUSTER_ALIAS_KEY); - String targetClusterAlias = keyStruct.getString(TARGET_CLUSTER_ALIAS_KEY); - - var map = Map.of( - "sourceClusterAlias", sourceClusterAlias, - "targetClusterAlias", targetClusterAlias - ); - - try { - var result = OBJECT_MAPPER.writeValueAsString(map); - yield new DeserializeResult(result, DeserializeResult.Type.JSON, Map.of()); - } catch (JsonProcessingException e) { - log.error("Error deserializing record", e); - throw new RuntimeException("Error deserializing record", e); - } - } - - case VALUE: { - ByteBuffer value = ByteBuffer.wrap(bytes); - Struct headerStruct = HEADER_SCHEMA.read(value); - short version = headerStruct.getShort(VERSION_KEY); - Struct valueStruct = valueSchema(version).read(value); - long timestamp = valueStruct.getLong(TIMESTAMP_KEY); - yield new DeserializeResult(String.valueOf(timestamp), DeserializeResult.Type.STRING, Map.of()); - } + private static DeserializeResult deserializeKey(byte[] bytes) { + Struct keyStruct = KEY_SCHEMA.read(ByteBuffer.wrap(bytes)); + String sourceClusterAlias = keyStruct.getString(SOURCE_CLUSTER_ALIAS_KEY); + String targetClusterAlias = keyStruct.getString(TARGET_CLUSTER_ALIAS_KEY); + + var map = Map.of( + "sourceClusterAlias", sourceClusterAlias, + "targetClusterAlias", targetClusterAlias + ); + + try { + var result = OBJECT_MAPPER.writeValueAsString(map); + return new DeserializeResult(result, DeserializeResult.Type.JSON, Map.of()); + } catch (JsonProcessingException e) { + log.error("Error deserializing record", e); + throw new RuntimeException("Error deserializing record", e); + } + } - }; + private static DeserializeResult deserializeValue(byte[] bytes) { + ByteBuffer value = ByteBuffer.wrap(bytes); + Struct headerStruct = HEADER_SCHEMA.read(value); + short version = headerStruct.getShort(VERSION_KEY); + Struct valueStruct = valueSchema(version).read(value); + long timestamp = valueStruct.getLong(TIMESTAMP_KEY); + return new DeserializeResult(String.valueOf(timestamp), DeserializeResult.Type.STRING, Map.of()); } private static Schema valueSchema(short version) { diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/OffsetSyncSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/OffsetSyncSerde.java index d3c8a6171..74c28913e 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/OffsetSyncSerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/OffsetSyncSerde.java @@ -64,42 +64,43 @@ public Serializer serializer(String topic, Target type) { public Deserializer deserializer(String topic, Target target) { return (recordHeaders, bytes) -> switch (target) { + case KEY -> deserializeKey(bytes); + case VALUE -> deserializeValue(bytes); + }; + } - case KEY: { - Struct keyStruct = KEY_SCHEMA.read(ByteBuffer.wrap(bytes)); - String t = keyStruct.getString(TOPIC_KEY); - int partition = keyStruct.getInt(PARTITION_KEY); - - var map = Map.of( - TOPIC_KEY, t, - PARTITION_KEY, partition - ); - - try { - var result = OBJECT_MAPPER.writeValueAsString(map); - yield new DeserializeResult(result, DeserializeResult.Type.JSON, Map.of()); - } catch (JsonProcessingException e) { - log.error("Error deserializing record", e); - throw new RuntimeException("Error deserializing record", e); - } - } - - case VALUE: { - Struct valueStruct = VALUE_SCHEMA.read(ByteBuffer.wrap(bytes)); - var map = Map.of( - UPSTREAM_OFFSET_KEY, valueStruct.getLong(UPSTREAM_OFFSET_KEY), - DOWNSTREAM_OFFSET_KEY, valueStruct.getLong(DOWNSTREAM_OFFSET_KEY) - ); - - try { - var result = OBJECT_MAPPER.writeValueAsString(map); - yield new DeserializeResult(result, DeserializeResult.Type.JSON, Map.of()); - } catch (JsonProcessingException e) { - log.error("Error deserializing record", e); - throw new RuntimeException("Error deserializing record", e); - } - } + private static DeserializeResult deserializeKey(byte[] bytes) { + Struct keyStruct = KEY_SCHEMA.read(ByteBuffer.wrap(bytes)); + String t = keyStruct.getString(TOPIC_KEY); + int partition = keyStruct.getInt(PARTITION_KEY); + + var map = Map.of( + TOPIC_KEY, t, + PARTITION_KEY, partition + ); + + try { + var result = OBJECT_MAPPER.writeValueAsString(map); + return new DeserializeResult(result, DeserializeResult.Type.JSON, Map.of()); + } catch (JsonProcessingException e) { + log.error("Error deserializing record", e); + throw new RuntimeException("Error deserializing record", e); + } + } - }; + private static DeserializeResult deserializeValue(byte[] bytes) { + Struct valueStruct = VALUE_SCHEMA.read(ByteBuffer.wrap(bytes)); + var map = Map.of( + UPSTREAM_OFFSET_KEY, valueStruct.getLong(UPSTREAM_OFFSET_KEY), + DOWNSTREAM_OFFSET_KEY, valueStruct.getLong(DOWNSTREAM_OFFSET_KEY) + ); + + try { + var result = OBJECT_MAPPER.writeValueAsString(map); + return new DeserializeResult(result, DeserializeResult.Type.JSON, Map.of()); + } catch (JsonProcessingException e) { + log.error("Error deserializing record", e); + throw new RuntimeException("Error deserializing record", e); + } } } From 89ca82c3f671d3301875ccf93c20744636866beb Mon Sep 17 00:00:00 2001 From: Roman Zabaluev Date: Sat, 27 Sep 2025 12:58:55 +0300 Subject: [PATCH 09/20] Upd fields visibility --- .../serdes/builtin/mm2/CheckpointSerde.java | 22 +++++++++---------- .../ui/serdes/builtin/mm2/HeartbeatSerde.java | 16 +++++++------- .../serdes/builtin/mm2/OffsetSyncSerde.java | 12 +++++----- 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/CheckpointSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/CheckpointSerde.java index 33b21c33e..a939dc8a3 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/CheckpointSerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/CheckpointSerde.java @@ -8,6 +8,7 @@ import java.nio.ByteBuffer; import java.util.Map; import java.util.Optional; +import java.util.regex.Pattern; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.common.protocol.types.Field; import org.apache.kafka.common.protocol.types.Schema; @@ -19,26 +20,25 @@ public class CheckpointSerde implements BuiltInSerde { private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - public static final String TOPIC_KEY = "topic"; - public static final String PARTITION_KEY = "partition"; - public static final String CONSUMER_GROUP_ID_KEY = "group"; - public static final String UPSTREAM_OFFSET_KEY = "upstreamOffset"; - public static final String DOWNSTREAM_OFFSET_KEY = "offset"; - public static final String METADATA_KEY = "metadata"; - public static final String VERSION_KEY = "version"; - public static final short VERSION = 0; + private static final String TOPIC_KEY = "topic"; + private static final String PARTITION_KEY = "partition"; + private static final String CONSUMER_GROUP_ID_KEY = "group"; + private static final String UPSTREAM_OFFSET_KEY = "upstreamOffset"; + private static final String DOWNSTREAM_OFFSET_KEY = "offset"; + private static final String METADATA_KEY = "metadata"; + private static final String VERSION_KEY = "version"; - public static final Schema VALUE_SCHEMA_V0 = new Schema( + private static final Schema VALUE_SCHEMA_V0 = new Schema( new Field(UPSTREAM_OFFSET_KEY, Type.INT64), new Field(DOWNSTREAM_OFFSET_KEY, Type.INT64), new Field(METADATA_KEY, Type.STRING)); - public static final Schema KEY_SCHEMA = new Schema( + private static final Schema KEY_SCHEMA = new Schema( new Field(CONSUMER_GROUP_ID_KEY, Type.STRING), new Field(TOPIC_KEY, Type.STRING), new Field(PARTITION_KEY, Type.INT32)); - public static final Schema HEADER_SCHEMA = new Schema( + private static final Schema HEADER_SCHEMA = new Schema( new Field(VERSION_KEY, Type.INT16)); public static String name() { diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerde.java index 893910444..111fce216 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerde.java @@ -8,6 +8,7 @@ import java.nio.ByteBuffer; import java.util.Map; import java.util.Optional; +import java.util.regex.Pattern; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.common.protocol.types.Field; import org.apache.kafka.common.protocol.types.Schema; @@ -19,20 +20,19 @@ public class HeartbeatSerde implements BuiltInSerde { private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - public static final String SOURCE_CLUSTER_ALIAS_KEY = "sourceClusterAlias"; - public static final String TARGET_CLUSTER_ALIAS_KEY = "targetClusterAlias"; - public static final String TIMESTAMP_KEY = "timestamp"; - public static final String VERSION_KEY = "version"; - public static final short VERSION = 0; + private static final String SOURCE_CLUSTER_ALIAS_KEY = "sourceClusterAlias"; + private static final String TARGET_CLUSTER_ALIAS_KEY = "targetClusterAlias"; + private static final String TIMESTAMP_KEY = "timestamp"; + private static final String VERSION_KEY = "version"; - public static final Schema VALUE_SCHEMA_V0 = new Schema( + private static final Schema VALUE_SCHEMA_V0 = new Schema( new Field(TIMESTAMP_KEY, Type.INT64)); - public static final Schema KEY_SCHEMA = new Schema( + private static final Schema KEY_SCHEMA = new Schema( new Field(SOURCE_CLUSTER_ALIAS_KEY, Type.STRING), new Field(TARGET_CLUSTER_ALIAS_KEY, Type.STRING)); - public static final Schema HEADER_SCHEMA = new Schema( + private static final Schema HEADER_SCHEMA = new Schema( new Field(VERSION_KEY, Type.INT16)); public static String name() { diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/OffsetSyncSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/OffsetSyncSerde.java index 74c28913e..863066e3d 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/OffsetSyncSerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/OffsetSyncSerde.java @@ -19,12 +19,12 @@ public class OffsetSyncSerde implements BuiltInSerde { private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - public static final String TOPIC_KEY = "topic"; - public static final String PARTITION_KEY = "partition"; - public static final String UPSTREAM_OFFSET_KEY = "upstreamOffset"; - public static final String DOWNSTREAM_OFFSET_KEY = "offset"; - public static final Schema VALUE_SCHEMA; - public static final Schema KEY_SCHEMA; + private static final String TOPIC_KEY = "topic"; + private static final String PARTITION_KEY = "partition"; + private static final String UPSTREAM_OFFSET_KEY = "upstreamOffset"; + private static final String DOWNSTREAM_OFFSET_KEY = "offset"; + private static final Schema VALUE_SCHEMA; + private static final Schema KEY_SCHEMA; static { VALUE_SCHEMA = new Schema(new Field("upstreamOffset", Type.INT64), new Field("offset", Type.INT64)); From 1ea2aa6349bcd334d2b8decf01106c020a0517ac Mon Sep 17 00:00:00 2001 From: Roman Zabaluev Date: Mon, 29 Sep 2025 17:38:58 +0300 Subject: [PATCH 10/20] Auto register serdes for well known topic patterns --- .../kafbat/ui/serdes/SerdesInitializer.java | 46 +++++++++++++++++++ .../serdes/builtin/mm2/CheckpointSerde.java | 2 + .../ui/serdes/builtin/mm2/HeartbeatSerde.java | 2 + .../serdes/builtin/mm2/OffsetSyncSerde.java | 3 ++ 4 files changed, 53 insertions(+) diff --git a/api/src/main/java/io/kafbat/ui/serdes/SerdesInitializer.java b/api/src/main/java/io/kafbat/ui/serdes/SerdesInitializer.java index 341b75e41..ea60be788 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/SerdesInitializer.java +++ b/api/src/main/java/io/kafbat/ui/serdes/SerdesInitializer.java @@ -151,6 +151,7 @@ public ClusterSerdes init(Environment env, */ private void registerTopicRelatedSerde(Map serdes) { registerConsumerOffsetsSerde(serdes); + registerMirrorMakerSerdes(serdes); } private void registerConsumerOffsetsSerde(Map serdes) { @@ -167,6 +168,51 @@ private void registerConsumerOffsetsSerde(Map serdes) { ); } + private void registerMirrorMakerSerdes(Map serdes) { + registerHeartbeatSerde(serdes); + registerOffsetSyncSerde(serdes); + registerCheckpointSerde(serdes); + } + + private void registerHeartbeatSerde(Map serdes) { + serdes.put( + HeartbeatSerde.name(), + new SerdeInstance( + HeartbeatSerde.name(), + new HeartbeatSerde(), + HeartbeatSerde.TOPIC_NAME_PATTERN, + HeartbeatSerde.TOPIC_NAME_PATTERN, + null + ) + ); + } + + private void registerOffsetSyncSerde(Map serdes) { + serdes.put( + OffsetSyncSerde.name(), + new SerdeInstance( + OffsetSyncSerde.name(), + new OffsetSyncSerde(), + OffsetSyncSerde.TOPIC_NAME_PATTERN, + OffsetSyncSerde.TOPIC_NAME_PATTERN, + null + ) + ); + } + + private void registerCheckpointSerde(Map serdes) { + serdes.put( + CheckpointSerde.name(), + new SerdeInstance( + CheckpointSerde.name(), + new CheckpointSerde(), + CheckpointSerde.TOPIC_NAME_PATTERN, + CheckpointSerde.TOPIC_NAME_PATTERN, + null + ) + ); + } + private SerdeInstance createFallbackSerde() { StringSerde serde = new StringSerde(); serde.configure(PropertyResolverImpl.empty(), PropertyResolverImpl.empty(), PropertyResolverImpl.empty()); diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/CheckpointSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/CheckpointSerde.java index a939dc8a3..52eca5088 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/CheckpointSerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/CheckpointSerde.java @@ -18,6 +18,8 @@ @Slf4j public class CheckpointSerde implements BuiltInSerde { + public static final Pattern TOPIC_NAME_PATTERN = Pattern.compile(".*\\.checkpoints\\.internal"); + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); private static final String TOPIC_KEY = "topic"; diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerde.java index 111fce216..8cf0aa60f 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerde.java @@ -18,6 +18,8 @@ @Slf4j public class HeartbeatSerde implements BuiltInSerde { + public static final Pattern TOPIC_NAME_PATTERN = Pattern.compile("heartbeats"); + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); private static final String SOURCE_CLUSTER_ALIAS_KEY = "sourceClusterAlias"; diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/OffsetSyncSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/OffsetSyncSerde.java index 863066e3d..09a1293e3 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/OffsetSyncSerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/OffsetSyncSerde.java @@ -8,6 +8,7 @@ import java.nio.ByteBuffer; import java.util.Map; import java.util.Optional; +import java.util.regex.Pattern; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.common.protocol.types.Field; import org.apache.kafka.common.protocol.types.Schema; @@ -17,6 +18,8 @@ @Slf4j public class OffsetSyncSerde implements BuiltInSerde { + public static final Pattern TOPIC_NAME_PATTERN = Pattern.compile("mm2-offset-syncs\\..*\\.internal"); + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); private static final String TOPIC_KEY = "topic"; From 9c987ef01786ab5640c979a9fbbf636269572611 Mon Sep 17 00:00:00 2001 From: Roman Zabaluev Date: Mon, 29 Sep 2025 18:05:48 +0300 Subject: [PATCH 11/20] Simplify serialization --- .../serdes/builtin/mm2/CheckpointSerde.java | 43 ++----------------- .../ui/serdes/builtin/mm2/HeartbeatSerde.java | 23 ++-------- .../serdes/builtin/mm2/MirrorMakerSerde.java | 40 +++++++++++++++++ .../serdes/builtin/mm2/OffsetSyncSerde.java | 41 +++--------------- 4 files changed, 51 insertions(+), 96 deletions(-) create mode 100644 api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/MirrorMakerSerde.java diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/CheckpointSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/CheckpointSerde.java index 52eca5088..6415337c0 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/CheckpointSerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/CheckpointSerde.java @@ -1,7 +1,5 @@ package io.kafbat.ui.serdes.builtin.mm2; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; import io.kafbat.ui.serde.api.DeserializeResult; import io.kafbat.ui.serde.api.SchemaDescription; import io.kafbat.ui.serdes.BuiltInSerde; @@ -16,12 +14,10 @@ import org.apache.kafka.common.protocol.types.Type; @Slf4j -public class CheckpointSerde implements BuiltInSerde { +public class CheckpointSerde extends MirrorMakerSerde implements BuiltInSerde { public static final Pattern TOPIC_NAME_PATTERN = Pattern.compile(".*\\.checkpoints\\.internal"); - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - private static final String TOPIC_KEY = "topic"; private static final String PARTITION_KEY = "partition"; private static final String CONSUMER_GROUP_ID_KEY = "group"; @@ -83,24 +79,7 @@ public Deserializer deserializer(String topic, Target target) { private static DeserializeResult deserializeKey(byte[] bytes) { Struct keyStruct = KEY_SCHEMA.read(ByteBuffer.wrap(bytes)); - - String group = keyStruct.getString(CONSUMER_GROUP_ID_KEY); - String t = keyStruct.getString(TOPIC_KEY); - int partition = keyStruct.getInt(PARTITION_KEY); - - var map = Map.of( - CONSUMER_GROUP_ID_KEY, group, - TOPIC_KEY, t, - PARTITION_KEY, partition - ); - - try { - var result = OBJECT_MAPPER.writeValueAsString(map); - return new DeserializeResult(result, DeserializeResult.Type.JSON, Map.of()); - } catch (JsonProcessingException e) { - log.error("Error deserializing record", e); - throw new RuntimeException("Error deserializing record", e); - } + return new DeserializeResult(toJson(keyStruct), DeserializeResult.Type.JSON, Map.of()); } private static DeserializeResult deserializeValue(byte[] bytes) { @@ -110,23 +89,7 @@ private static DeserializeResult deserializeValue(byte[] bytes) { Schema valueSchema = valueSchema(version); Struct valueStruct = valueSchema.read(value); - long upstreamOffset = valueStruct.getLong(UPSTREAM_OFFSET_KEY); - long downstreamOffset = valueStruct.getLong(DOWNSTREAM_OFFSET_KEY); - String metadata = valueStruct.getString(METADATA_KEY); - - var map = Map.of( - UPSTREAM_OFFSET_KEY, upstreamOffset, - DOWNSTREAM_OFFSET_KEY, downstreamOffset, - METADATA_KEY, metadata - ); - - try { - var result = OBJECT_MAPPER.writeValueAsString(map); - return new DeserializeResult(result, DeserializeResult.Type.JSON, Map.of()); - } catch (JsonProcessingException e) { - log.error("Error deserializing record", e); - throw new RuntimeException("Error deserializing record", e); - } + return new DeserializeResult(toJson(valueStruct), DeserializeResult.Type.JSON, Map.of()); } private static Schema valueSchema(short version) { diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerde.java index 8cf0aa60f..2c9efd46c 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerde.java @@ -1,7 +1,5 @@ package io.kafbat.ui.serdes.builtin.mm2; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; import io.kafbat.ui.serde.api.DeserializeResult; import io.kafbat.ui.serde.api.SchemaDescription; import io.kafbat.ui.serdes.BuiltInSerde; @@ -16,12 +14,10 @@ import org.apache.kafka.common.protocol.types.Type; @Slf4j -public class HeartbeatSerde implements BuiltInSerde { +public class HeartbeatSerde extends MirrorMakerSerde implements BuiltInSerde { public static final Pattern TOPIC_NAME_PATTERN = Pattern.compile("heartbeats"); - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - private static final String SOURCE_CLUSTER_ALIAS_KEY = "sourceClusterAlias"; private static final String TARGET_CLUSTER_ALIAS_KEY = "targetClusterAlias"; private static final String TIMESTAMP_KEY = "timestamp"; @@ -77,21 +73,8 @@ public Deserializer deserializer(String topic, Target target) { private static DeserializeResult deserializeKey(byte[] bytes) { Struct keyStruct = KEY_SCHEMA.read(ByteBuffer.wrap(bytes)); - String sourceClusterAlias = keyStruct.getString(SOURCE_CLUSTER_ALIAS_KEY); - String targetClusterAlias = keyStruct.getString(TARGET_CLUSTER_ALIAS_KEY); - - var map = Map.of( - "sourceClusterAlias", sourceClusterAlias, - "targetClusterAlias", targetClusterAlias - ); - - try { - var result = OBJECT_MAPPER.writeValueAsString(map); - return new DeserializeResult(result, DeserializeResult.Type.JSON, Map.of()); - } catch (JsonProcessingException e) { - log.error("Error deserializing record", e); - throw new RuntimeException("Error deserializing record", e); - } + + return new DeserializeResult(toJson(keyStruct), DeserializeResult.Type.JSON, Map.of()); } private static DeserializeResult deserializeValue(byte[] bytes) { diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/MirrorMakerSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/MirrorMakerSerde.java new file mode 100644 index 000000000..b8adb937c --- /dev/null +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/MirrorMakerSerde.java @@ -0,0 +1,40 @@ +package io.kafbat.ui.serdes.builtin.mm2; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.json.JsonMapper; +import com.fasterxml.jackson.databind.module.SimpleModule; +import java.io.IOException; +import lombok.SneakyThrows; +import org.apache.kafka.common.protocol.types.BoundField; +import org.apache.kafka.common.protocol.types.Struct; + +abstract class MirrorMakerSerde { + + protected static final JsonMapper JSON_MAPPER = createMapper(); + + protected static JsonMapper createMapper() { + var module = new SimpleModule(); + module.addSerializer(Struct.class, new JsonSerializer<>() { + @Override + public void serialize(Struct value, JsonGenerator gen, SerializerProvider serializers) throws IOException { + gen.writeStartObject(); + for (BoundField field : value.schema().fields()) { + var fieldVal = value.get(field); + gen.writeObjectField(field.def.name, fieldVal); + } + gen.writeEndObject(); + } + }); + var mapper = new JsonMapper(); + mapper.registerModule(module); + return mapper; + } + + @SneakyThrows + protected static String toJson(Struct s) { + return JSON_MAPPER.writeValueAsString(s); + } + +} diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/OffsetSyncSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/OffsetSyncSerde.java index 09a1293e3..be9fc8268 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/OffsetSyncSerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/OffsetSyncSerde.java @@ -1,7 +1,5 @@ package io.kafbat.ui.serdes.builtin.mm2; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; import io.kafbat.ui.serde.api.DeserializeResult; import io.kafbat.ui.serde.api.SchemaDescription; import io.kafbat.ui.serdes.BuiltInSerde; @@ -16,16 +14,10 @@ import org.apache.kafka.common.protocol.types.Type; @Slf4j -public class OffsetSyncSerde implements BuiltInSerde { +public class OffsetSyncSerde extends MirrorMakerSerde implements BuiltInSerde { public static final Pattern TOPIC_NAME_PATTERN = Pattern.compile("mm2-offset-syncs\\..*\\.internal"); - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - - private static final String TOPIC_KEY = "topic"; - private static final String PARTITION_KEY = "partition"; - private static final String UPSTREAM_OFFSET_KEY = "upstreamOffset"; - private static final String DOWNSTREAM_OFFSET_KEY = "offset"; private static final Schema VALUE_SCHEMA; private static final Schema KEY_SCHEMA; @@ -74,36 +66,13 @@ public Deserializer deserializer(String topic, Target target) { private static DeserializeResult deserializeKey(byte[] bytes) { Struct keyStruct = KEY_SCHEMA.read(ByteBuffer.wrap(bytes)); - String t = keyStruct.getString(TOPIC_KEY); - int partition = keyStruct.getInt(PARTITION_KEY); - - var map = Map.of( - TOPIC_KEY, t, - PARTITION_KEY, partition - ); - - try { - var result = OBJECT_MAPPER.writeValueAsString(map); - return new DeserializeResult(result, DeserializeResult.Type.JSON, Map.of()); - } catch (JsonProcessingException e) { - log.error("Error deserializing record", e); - throw new RuntimeException("Error deserializing record", e); - } + + return new DeserializeResult(toJson(keyStruct), DeserializeResult.Type.JSON, Map.of()); } private static DeserializeResult deserializeValue(byte[] bytes) { Struct valueStruct = VALUE_SCHEMA.read(ByteBuffer.wrap(bytes)); - var map = Map.of( - UPSTREAM_OFFSET_KEY, valueStruct.getLong(UPSTREAM_OFFSET_KEY), - DOWNSTREAM_OFFSET_KEY, valueStruct.getLong(DOWNSTREAM_OFFSET_KEY) - ); - - try { - var result = OBJECT_MAPPER.writeValueAsString(map); - return new DeserializeResult(result, DeserializeResult.Type.JSON, Map.of()); - } catch (JsonProcessingException e) { - log.error("Error deserializing record", e); - throw new RuntimeException("Error deserializing record", e); - } + + return new DeserializeResult(toJson(valueStruct), DeserializeResult.Type.JSON, Map.of()); } } From 7e25c7c8c42a9e4dde71e9777d98f8fd5eb57f7b Mon Sep 17 00:00:00 2001 From: German Osin Date: Mon, 29 Sep 2025 23:19:39 +0200 Subject: [PATCH 12/20] Reduced duplicate code --- .../serdes/builtin/mm2/CheckpointSerde.java | 69 +++------------- .../ui/serdes/builtin/mm2/HeartbeatSerde.java | 70 +++------------- .../serdes/builtin/mm2/MirrorMakerSerde.java | 80 ++++++++++++++++++- .../serdes/builtin/mm2/OffsetSyncSerde.java | 64 ++++----------- 4 files changed, 121 insertions(+), 162 deletions(-) diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/CheckpointSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/CheckpointSerde.java index 6415337c0..16e1c636c 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/CheckpointSerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/CheckpointSerde.java @@ -1,16 +1,12 @@ package io.kafbat.ui.serdes.builtin.mm2; -import io.kafbat.ui.serde.api.DeserializeResult; -import io.kafbat.ui.serde.api.SchemaDescription; import io.kafbat.ui.serdes.BuiltInSerde; -import java.nio.ByteBuffer; -import java.util.Map; import java.util.Optional; +import java.util.OptionalInt; import java.util.regex.Pattern; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.common.protocol.types.Field; import org.apache.kafka.common.protocol.types.Schema; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.protocol.types.Type; @Slf4j @@ -24,7 +20,6 @@ public class CheckpointSerde extends MirrorMakerSerde implements BuiltInSerde { private static final String UPSTREAM_OFFSET_KEY = "upstreamOffset"; private static final String DOWNSTREAM_OFFSET_KEY = "offset"; private static final String METADATA_KEY = "metadata"; - private static final String VERSION_KEY = "version"; private static final Schema VALUE_SCHEMA_V0 = new Schema( new Field(UPSTREAM_OFFSET_KEY, Type.INT64), @@ -36,65 +31,27 @@ public class CheckpointSerde extends MirrorMakerSerde implements BuiltInSerde { new Field(TOPIC_KEY, Type.STRING), new Field(PARTITION_KEY, Type.INT32)); - private static final Schema HEADER_SCHEMA = new Schema( - new Field(VERSION_KEY, Type.INT16)); + public CheckpointSerde() { + super(true); + } public static String name() { return "Checkpoint"; } @Override - public Optional getDescription() { - return Optional.empty(); - } - - @Override - public Optional getSchema(String topic, Target type) { - return Optional.empty(); - } - - @Override - public boolean canDeserialize(String topic, Target type) { - return true; - } - - @Override - public boolean canSerialize(String topic, Target type) { - return false; + protected Schema getKeySchema() { + return KEY_SCHEMA; } @Override - public Serializer serializer(String topic, Target type) { - throw new UnsupportedOperationException(); - } - - @Override - public Deserializer deserializer(String topic, Target target) { - return (recordHeaders, bytes) -> - switch (target) { - case KEY -> deserializeKey(bytes); - case VALUE -> deserializeValue(bytes); - }; - } - - private static DeserializeResult deserializeKey(byte[] bytes) { - Struct keyStruct = KEY_SCHEMA.read(ByteBuffer.wrap(bytes)); - return new DeserializeResult(toJson(keyStruct), DeserializeResult.Type.JSON, Map.of()); - } - - private static DeserializeResult deserializeValue(byte[] bytes) { - ByteBuffer value = ByteBuffer.wrap(bytes); - Struct header = HEADER_SCHEMA.read(value); - short version = header.getShort(VERSION_KEY); - Schema valueSchema = valueSchema(version); - Struct valueStruct = valueSchema.read(value); - - return new DeserializeResult(toJson(valueStruct), DeserializeResult.Type.JSON, Map.of()); - } - - private static Schema valueSchema(short version) { - assert version == 0; - return VALUE_SCHEMA_V0; + protected Optional getValueSchema(short version) { + if (version == 0) { + return Optional.of(VALUE_SCHEMA_V0); + } else { + log.warn("Unsupported version of CheckpointSerde: {}", version); + return Optional.empty(); + } } } diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerde.java index 2c9efd46c..d4e451f4f 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerde.java @@ -1,16 +1,11 @@ package io.kafbat.ui.serdes.builtin.mm2; -import io.kafbat.ui.serde.api.DeserializeResult; -import io.kafbat.ui.serde.api.SchemaDescription; import io.kafbat.ui.serdes.BuiltInSerde; -import java.nio.ByteBuffer; -import java.util.Map; import java.util.Optional; import java.util.regex.Pattern; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.common.protocol.types.Field; import org.apache.kafka.common.protocol.types.Schema; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.protocol.types.Type; @Slf4j @@ -21,7 +16,6 @@ public class HeartbeatSerde extends MirrorMakerSerde implements BuiltInSerde { private static final String SOURCE_CLUSTER_ALIAS_KEY = "sourceClusterAlias"; private static final String TARGET_CLUSTER_ALIAS_KEY = "targetClusterAlias"; private static final String TIMESTAMP_KEY = "timestamp"; - private static final String VERSION_KEY = "version"; private static final Schema VALUE_SCHEMA_V0 = new Schema( new Field(TIMESTAMP_KEY, Type.INT64)); @@ -30,64 +24,24 @@ public class HeartbeatSerde extends MirrorMakerSerde implements BuiltInSerde { new Field(SOURCE_CLUSTER_ALIAS_KEY, Type.STRING), new Field(TARGET_CLUSTER_ALIAS_KEY, Type.STRING)); - private static final Schema HEADER_SCHEMA = new Schema( - new Field(VERSION_KEY, Type.INT16)); + public HeartbeatSerde() { + super(true); + } public static String name() { return "Heartbeat"; } - @Override - public Optional getDescription() { - return Optional.empty(); - } - - @Override - public Optional getSchema(String topic, Target type) { - return Optional.empty(); - } - - @Override - public boolean canDeserialize(String topic, Target type) { - return true; - } - - @Override - public boolean canSerialize(String topic, Target type) { - return false; - } - - @Override - public Serializer serializer(String topic, Target type) { - throw new UnsupportedOperationException(); - } - - @Override - public Deserializer deserializer(String topic, Target target) { - return (recordHeaders, bytes) -> - switch (target) { - case KEY -> deserializeKey(bytes); - case VALUE -> deserializeValue(bytes); - }; - } - - private static DeserializeResult deserializeKey(byte[] bytes) { - Struct keyStruct = KEY_SCHEMA.read(ByteBuffer.wrap(bytes)); - - return new DeserializeResult(toJson(keyStruct), DeserializeResult.Type.JSON, Map.of()); - } - - private static DeserializeResult deserializeValue(byte[] bytes) { - ByteBuffer value = ByteBuffer.wrap(bytes); - Struct headerStruct = HEADER_SCHEMA.read(value); - short version = headerStruct.getShort(VERSION_KEY); - Struct valueStruct = valueSchema(version).read(value); - long timestamp = valueStruct.getLong(TIMESTAMP_KEY); - return new DeserializeResult(String.valueOf(timestamp), DeserializeResult.Type.STRING, Map.of()); + protected Schema getKeySchema() { + return KEY_SCHEMA; } - private static Schema valueSchema(short version) { - assert version == 0; - return VALUE_SCHEMA_V0; + protected Optional getValueSchema(short version) { + if (version == 0) { + return Optional.of(VALUE_SCHEMA_V0); + } else { + log.warn("Unsupported version of HeartbeatSerde: {}", version); + return Optional.empty(); + } } } diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/MirrorMakerSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/MirrorMakerSerde.java index b8adb937c..dbacfbe1e 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/MirrorMakerSerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/MirrorMakerSerde.java @@ -5,12 +5,22 @@ import com.fasterxml.jackson.databind.SerializerProvider; import com.fasterxml.jackson.databind.json.JsonMapper; import com.fasterxml.jackson.databind.module.SimpleModule; +import io.kafbat.ui.serde.api.DeserializeResult; +import io.kafbat.ui.serde.api.SchemaDescription; +import io.kafbat.ui.serde.api.Serde; +import io.kafbat.ui.serdes.BuiltInSerde; import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Map; +import java.util.Optional; +import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import org.apache.kafka.common.protocol.types.BoundField; +import org.apache.kafka.common.protocol.types.Schema; import org.apache.kafka.common.protocol.types.Struct; -abstract class MirrorMakerSerde { +@RequiredArgsConstructor +abstract class MirrorMakerSerde implements BuiltInSerde { protected static final JsonMapper JSON_MAPPER = createMapper(); @@ -32,9 +42,77 @@ public void serialize(Struct value, JsonGenerator gen, SerializerProvider serial return mapper; } + protected final boolean versioned; + + @Override + public Optional getDescription() { + return Optional.empty(); + } + + @Override + public Optional getSchema(String topic, Serde.Target type) { + return Optional.empty(); + } + + @Override + public boolean canDeserialize(String topic, Serde.Target type) { + return true; + } + + @Override + public boolean canSerialize(String topic, Serde.Target type) { + return false; + } + + @Override + public Serde.Serializer serializer(String topic, Serde.Target type) { + throw new UnsupportedOperationException(); + } + @SneakyThrows protected static String toJson(Struct s) { return JSON_MAPPER.writeValueAsString(s); } + @Override + public Deserializer deserializer(String topic, Target target) { + return (recordHeaders, bytes) -> + switch (target) { + case KEY -> deserializeKey(bytes); + case VALUE -> deserializeValue(bytes); + }; + } + + protected DeserializeResult deserializeKey(byte[] bytes) { + Struct keyStruct = getKeySchema().read(ByteBuffer.wrap(bytes)); + return new DeserializeResult(toJson(keyStruct), DeserializeResult.Type.JSON, Map.of()); + } + + protected DeserializeResult deserializeValue(byte[] bytes) { + ByteBuffer wrap = ByteBuffer.wrap(bytes); + Optional valueSchema; + if (versioned) { + short version = wrap.getShort(); + valueSchema = getValueSchema(version); + } else { + valueSchema = getValueSchema(); + } + if (valueSchema.isPresent()) { + Struct valueStruct = valueSchema.get().read(wrap); + return new DeserializeResult(toJson(valueStruct), DeserializeResult.Type.JSON, Map.of()); + } else { + throw new IllegalStateException("Value schema was not present"); + } + } + + protected abstract Schema getKeySchema(); + + protected Optional getValueSchema() { + return Optional.empty(); + } + + protected Optional getValueSchema(short version) { + return Optional.empty(); + } + } diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/OffsetSyncSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/OffsetSyncSerde.java index be9fc8268..9c1994871 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/OffsetSyncSerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/OffsetSyncSerde.java @@ -1,16 +1,12 @@ package io.kafbat.ui.serdes.builtin.mm2; -import io.kafbat.ui.serde.api.DeserializeResult; -import io.kafbat.ui.serde.api.SchemaDescription; import io.kafbat.ui.serdes.BuiltInSerde; -import java.nio.ByteBuffer; -import java.util.Map; import java.util.Optional; +import java.util.OptionalInt; import java.util.regex.Pattern; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.common.protocol.types.Field; import org.apache.kafka.common.protocol.types.Schema; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.protocol.types.Type; @Slf4j @@ -22,57 +18,31 @@ public class OffsetSyncSerde extends MirrorMakerSerde implements BuiltInSerde { private static final Schema KEY_SCHEMA; static { - VALUE_SCHEMA = new Schema(new Field("upstreamOffset", Type.INT64), new Field("offset", Type.INT64)); - KEY_SCHEMA = new Schema(new Field("topic", Type.STRING), new Field("partition", Type.INT32)); + VALUE_SCHEMA = new Schema( + new Field("upstreamOffset", Type.INT64), + new Field("offset", Type.INT64) + ); + KEY_SCHEMA = new Schema( + new Field("topic", Type.STRING), + new Field("partition", Type.INT32) + ); } - public static String name() { - return "OffsetSync"; - } - - @Override - public Optional getDescription() { - return Optional.empty(); - } - - @Override - public Optional getSchema(String topic, Target type) { - return Optional.empty(); - } - - @Override - public boolean canDeserialize(String topic, Target type) { - return true; + public OffsetSyncSerde() { + super(false); } - @Override - public boolean canSerialize(String topic, Target type) { - return false; + public static String name() { + return "OffsetSync"; } @Override - public Serializer serializer(String topic, Target type) { - throw new UnsupportedOperationException(); + protected Schema getKeySchema() { + return KEY_SCHEMA; } @Override - public Deserializer deserializer(String topic, Target target) { - return (recordHeaders, bytes) -> - switch (target) { - case KEY -> deserializeKey(bytes); - case VALUE -> deserializeValue(bytes); - }; - } - - private static DeserializeResult deserializeKey(byte[] bytes) { - Struct keyStruct = KEY_SCHEMA.read(ByteBuffer.wrap(bytes)); - - return new DeserializeResult(toJson(keyStruct), DeserializeResult.Type.JSON, Map.of()); - } - - private static DeserializeResult deserializeValue(byte[] bytes) { - Struct valueStruct = VALUE_SCHEMA.read(ByteBuffer.wrap(bytes)); - - return new DeserializeResult(toJson(valueStruct), DeserializeResult.Type.JSON, Map.of()); + protected Optional getValueSchema() { + return Optional.of(VALUE_SCHEMA); } } From c5cb67d8697db7478adf75f7c343a0e5657e9821 Mon Sep 17 00:00:00 2001 From: German Osin Date: Mon, 29 Sep 2025 23:39:08 +0200 Subject: [PATCH 13/20] Reduced duplicate code --- .../kafbat/ui/serdes/SerdesInitializer.java | 53 +++++-------------- .../serdes/builtin/ConsumerOffsetsSerde.java | 27 +--------- .../kafbat/ui/serdes/builtin/StructSerde.java | 39 ++++++++++++++ .../serdes/builtin/mm2/CheckpointSerde.java | 1 - .../serdes/builtin/mm2/MirrorMakerSerde.java | 36 +------------ .../serdes/builtin/mm2/OffsetSyncSerde.java | 1 - .../builtin/mm2/HeartbeatSerdeTest.java | 9 ++-- 7 files changed, 61 insertions(+), 105 deletions(-) create mode 100644 api/src/main/java/io/kafbat/ui/serdes/builtin/StructSerde.java diff --git a/api/src/main/java/io/kafbat/ui/serdes/SerdesInitializer.java b/api/src/main/java/io/kafbat/ui/serdes/SerdesInitializer.java index ea60be788..55a9e3d8a 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/SerdesInitializer.java +++ b/api/src/main/java/io/kafbat/ui/serdes/SerdesInitializer.java @@ -169,48 +169,21 @@ private void registerConsumerOffsetsSerde(Map serdes) { } private void registerMirrorMakerSerdes(Map serdes) { - registerHeartbeatSerde(serdes); - registerOffsetSyncSerde(serdes); - registerCheckpointSerde(serdes); - } - - private void registerHeartbeatSerde(Map serdes) { - serdes.put( - HeartbeatSerde.name(), - new SerdeInstance( - HeartbeatSerde.name(), - new HeartbeatSerde(), - HeartbeatSerde.TOPIC_NAME_PATTERN, - HeartbeatSerde.TOPIC_NAME_PATTERN, - null - ) - ); - } - - private void registerOffsetSyncSerde(Map serdes) { - serdes.put( - OffsetSyncSerde.name(), - new SerdeInstance( - OffsetSyncSerde.name(), - new OffsetSyncSerde(), - OffsetSyncSerde.TOPIC_NAME_PATTERN, - OffsetSyncSerde.TOPIC_NAME_PATTERN, - null - ) + Map> mmSerdes = Map.of( + HeartbeatSerde.name(), Map.entry(HeartbeatSerde.TOPIC_NAME_PATTERN, new HeartbeatSerde()), + OffsetSyncSerde.name(), Map.entry(OffsetSyncSerde.TOPIC_NAME_PATTERN, new OffsetSyncSerde()), + CheckpointSerde.name(), Map.entry(CheckpointSerde.TOPIC_NAME_PATTERN, new CheckpointSerde()) ); - } + for (Map.Entry> serde : mmSerdes.entrySet()) { + String name = serde.getKey(); + Pattern pattern = serde.getValue().getKey(); + BuiltInSerde serdeInstance = serde.getValue().getValue(); - private void registerCheckpointSerde(Map serdes) { - serdes.put( - CheckpointSerde.name(), - new SerdeInstance( - CheckpointSerde.name(), - new CheckpointSerde(), - CheckpointSerde.TOPIC_NAME_PATTERN, - CheckpointSerde.TOPIC_NAME_PATTERN, - null - ) - ); + serdes.put( + name, + new SerdeInstance(name, serdeInstance, pattern, pattern, null) + ); + } } private SerdeInstance createFallbackSerde() { diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/ConsumerOffsetsSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/ConsumerOffsetsSerde.java index fdf6d0c79..8e1f3211f 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/ConsumerOffsetsSerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/ConsumerOffsetsSerde.java @@ -23,9 +23,7 @@ // Deserialization logic and message's schemas can be found in // kafka.coordinator.group.GroupMetadataManager (readMessageKey, readOffsetMessageValue, readGroupMessageValue) -public class ConsumerOffsetsSerde implements BuiltInSerde { - - private static final JsonMapper JSON_MAPPER = createMapper(); +public class ConsumerOffsetsSerde extends StructSerde implements BuiltInSerde { private static final String ASSIGNMENT = "assignment"; private static final String CLIENT_HOST = "client_host"; @@ -50,24 +48,6 @@ public static String name() { return "__consumer_offsets"; } - private static JsonMapper createMapper() { - var module = new SimpleModule(); - module.addSerializer(Struct.class, new JsonSerializer<>() { - @Override - public void serialize(Struct value, JsonGenerator gen, SerializerProvider serializers) throws IOException { - gen.writeStartObject(); - for (BoundField field : value.schema().fields()) { - var fieldVal = value.get(field); - gen.writeObjectField(field.def.name, fieldVal); - } - gen.writeEndObject(); - } - }); - var mapper = new JsonMapper(); - mapper.registerModule(module); - return mapper; - } - @Override public Optional getDescription() { return Optional.empty(); @@ -304,8 +284,5 @@ private Deserializer valueDeserializer() { }; } - @SneakyThrows - private String toJson(Struct s) { - return JSON_MAPPER.writeValueAsString(s); - } + } diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/StructSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/StructSerde.java new file mode 100644 index 000000000..5315aedc2 --- /dev/null +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/StructSerde.java @@ -0,0 +1,39 @@ +package io.kafbat.ui.serdes.builtin; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.json.JsonMapper; +import com.fasterxml.jackson.databind.module.SimpleModule; +import java.io.IOException; +import lombok.SneakyThrows; +import org.apache.kafka.common.protocol.types.BoundField; +import org.apache.kafka.common.protocol.types.Struct; + +public abstract class StructSerde { + + private static final JsonMapper JSON_MAPPER = createMapper(); + + private static JsonMapper createMapper() { + var module = new SimpleModule(); + module.addSerializer(Struct.class, new JsonSerializer<>() { + @Override + public void serialize(Struct value, JsonGenerator gen, SerializerProvider serializers) throws IOException { + gen.writeStartObject(); + for (BoundField field : value.schema().fields()) { + var fieldVal = value.get(field); + gen.writeObjectField(field.def.name, fieldVal); + } + gen.writeEndObject(); + } + }); + var mapper = new JsonMapper(); + mapper.registerModule(module); + return mapper; + } + + @SneakyThrows + protected String toJson(Struct s) { + return JSON_MAPPER.writeValueAsString(s); + } +} diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/CheckpointSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/CheckpointSerde.java index 16e1c636c..a4da56c41 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/CheckpointSerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/CheckpointSerde.java @@ -2,7 +2,6 @@ import io.kafbat.ui.serdes.BuiltInSerde; import java.util.Optional; -import java.util.OptionalInt; import java.util.regex.Pattern; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.common.protocol.types.Field; diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/MirrorMakerSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/MirrorMakerSerde.java index dbacfbe1e..63505457e 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/MirrorMakerSerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/MirrorMakerSerde.java @@ -1,46 +1,19 @@ package io.kafbat.ui.serdes.builtin.mm2; -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.databind.JsonSerializer; -import com.fasterxml.jackson.databind.SerializerProvider; -import com.fasterxml.jackson.databind.json.JsonMapper; -import com.fasterxml.jackson.databind.module.SimpleModule; import io.kafbat.ui.serde.api.DeserializeResult; import io.kafbat.ui.serde.api.SchemaDescription; import io.kafbat.ui.serde.api.Serde; import io.kafbat.ui.serdes.BuiltInSerde; -import java.io.IOException; +import io.kafbat.ui.serdes.builtin.StructSerde; import java.nio.ByteBuffer; import java.util.Map; import java.util.Optional; import lombok.RequiredArgsConstructor; -import lombok.SneakyThrows; -import org.apache.kafka.common.protocol.types.BoundField; import org.apache.kafka.common.protocol.types.Schema; import org.apache.kafka.common.protocol.types.Struct; @RequiredArgsConstructor -abstract class MirrorMakerSerde implements BuiltInSerde { - - protected static final JsonMapper JSON_MAPPER = createMapper(); - - protected static JsonMapper createMapper() { - var module = new SimpleModule(); - module.addSerializer(Struct.class, new JsonSerializer<>() { - @Override - public void serialize(Struct value, JsonGenerator gen, SerializerProvider serializers) throws IOException { - gen.writeStartObject(); - for (BoundField field : value.schema().fields()) { - var fieldVal = value.get(field); - gen.writeObjectField(field.def.name, fieldVal); - } - gen.writeEndObject(); - } - }); - var mapper = new JsonMapper(); - mapper.registerModule(module); - return mapper; - } +abstract class MirrorMakerSerde extends StructSerde implements BuiltInSerde { protected final boolean versioned; @@ -69,11 +42,6 @@ public Serde.Serializer serializer(String topic, Serde.Target type) { throw new UnsupportedOperationException(); } - @SneakyThrows - protected static String toJson(Struct s) { - return JSON_MAPPER.writeValueAsString(s); - } - @Override public Deserializer deserializer(String topic, Target target) { return (recordHeaders, bytes) -> diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/OffsetSyncSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/OffsetSyncSerde.java index 9c1994871..b355251b7 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/OffsetSyncSerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/OffsetSyncSerde.java @@ -2,7 +2,6 @@ import io.kafbat.ui.serdes.BuiltInSerde; import java.util.Optional; -import java.util.OptionalInt; import java.util.regex.Pattern; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.common.protocol.types.Field; diff --git a/api/src/test/java/io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerdeTest.java b/api/src/test/java/io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerdeTest.java index 1e778794f..a3dacd149 100644 --- a/api/src/test/java/io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerdeTest.java +++ b/api/src/test/java/io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerdeTest.java @@ -46,14 +46,15 @@ void testDeserializeKey() throws JsonProcessingException { } @Test - void testDeserializeValue() { + void testDeserializeValue() throws JsonProcessingException { var value = decodeBase64("AAAAAAGZgCEMZA=="); - var expected = "1758791273572"; + var expected = Map.of("timestamp", 1758791273572L); var result = SERDE.deserializer(TOPIC, Serde.Target.VALUE).deserialize(HEADERS, value); + var resultMap = jsonToMap(result.getResult()); - assertEquals(DeserializeResult.Type.STRING, result.getType()); - assertEquals(expected, result.getResult()); + assertEquals(DeserializeResult.Type.JSON, result.getType()); + assertEquals(expected, resultMap); } } From 40760cab89ee5c05ef1bc1bc7b56ed0df7ef3b82 Mon Sep 17 00:00:00 2001 From: German Osin Date: Mon, 29 Sep 2025 23:46:59 +0200 Subject: [PATCH 14/20] Reduced duplicate code --- .../serdes/builtin/mm2/CheckpointSerde.java | 2 +- .../ui/serdes/builtin/mm2/HeartbeatSerde.java | 3 ++- .../serdes/builtin/mm2/MirrorMakerSerde.java | 20 +++++++++---------- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/CheckpointSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/CheckpointSerde.java index a4da56c41..ef1cbec62 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/CheckpointSerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/CheckpointSerde.java @@ -44,7 +44,7 @@ protected Schema getKeySchema() { } @Override - protected Optional getValueSchema(short version) { + protected Optional getVersionedValueSchema(short version) { if (version == 0) { return Optional.of(VALUE_SCHEMA_V0); } else { diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerde.java index d4e451f4f..f8e54352d 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerde.java @@ -36,7 +36,8 @@ protected Schema getKeySchema() { return KEY_SCHEMA; } - protected Optional getValueSchema(short version) { + @Override + protected Optional getVersionedValueSchema(short version) { if (version == 0) { return Optional.of(VALUE_SCHEMA_V0); } else { diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/MirrorMakerSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/MirrorMakerSerde.java index 63505457e..d915a5e22 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/MirrorMakerSerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/MirrorMakerSerde.java @@ -45,29 +45,27 @@ public Serde.Serializer serializer(String topic, Serde.Target type) { @Override public Deserializer deserializer(String topic, Target target) { return (recordHeaders, bytes) -> - switch (target) { + new DeserializeResult(toJson(switch (target) { case KEY -> deserializeKey(bytes); case VALUE -> deserializeValue(bytes); - }; + }), DeserializeResult.Type.JSON, Map.of()); } - protected DeserializeResult deserializeKey(byte[] bytes) { - Struct keyStruct = getKeySchema().read(ByteBuffer.wrap(bytes)); - return new DeserializeResult(toJson(keyStruct), DeserializeResult.Type.JSON, Map.of()); + protected Struct deserializeKey(byte[] bytes) { + return getKeySchema().read(ByteBuffer.wrap(bytes)); } - protected DeserializeResult deserializeValue(byte[] bytes) { + protected Struct deserializeValue(byte[] bytes) { ByteBuffer wrap = ByteBuffer.wrap(bytes); Optional valueSchema; if (versioned) { short version = wrap.getShort(); - valueSchema = getValueSchema(version); + valueSchema = getVersionedValueSchema(version); } else { valueSchema = getValueSchema(); } if (valueSchema.isPresent()) { - Struct valueStruct = valueSchema.get().read(wrap); - return new DeserializeResult(toJson(valueStruct), DeserializeResult.Type.JSON, Map.of()); + return valueSchema.get().read(wrap); } else { throw new IllegalStateException("Value schema was not present"); } @@ -79,8 +77,8 @@ protected Optional getValueSchema() { return Optional.empty(); } - protected Optional getValueSchema(short version) { - return Optional.empty(); + protected Optional getVersionedValueSchema(short version) { + throw new UnsupportedOperationException("Versioned value schema is not supported"); } } From 68a8a269a753f9f75a88424ed8ac2dda95a3ad5e Mon Sep 17 00:00:00 2001 From: German Osin Date: Mon, 29 Sep 2025 23:50:41 +0200 Subject: [PATCH 15/20] Reduced duplicate code --- .../serdes/builtin/ConsumerOffsetsSerde.java | 20 ------------ .../kafbat/ui/serdes/builtin/StructSerde.java | 31 ++++++++++++++++++- .../serdes/builtin/mm2/MirrorMakerSerde.java | 27 ---------------- 3 files changed, 30 insertions(+), 48 deletions(-) diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/ConsumerOffsetsSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/ConsumerOffsetsSerde.java index 8e1f3211f..2071912b5 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/ConsumerOffsetsSerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/ConsumerOffsetsSerde.java @@ -48,31 +48,11 @@ public static String name() { return "__consumer_offsets"; } - @Override - public Optional getDescription() { - return Optional.empty(); - } - - @Override - public Optional getSchema(String topic, Target type) { - return Optional.empty(); - } - @Override public boolean canDeserialize(String topic, Target type) { return topic.equals(TOPIC); } - @Override - public boolean canSerialize(String topic, Target type) { - return false; - } - - @Override - public Serializer serializer(String topic, Target type) { - throw new UnsupportedOperationException(); - } - @Override public Deserializer deserializer(String topic, Target type) { return switch (type) { diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/StructSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/StructSerde.java index 5315aedc2..f236fd58c 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/StructSerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/StructSerde.java @@ -5,12 +5,16 @@ import com.fasterxml.jackson.databind.SerializerProvider; import com.fasterxml.jackson.databind.json.JsonMapper; import com.fasterxml.jackson.databind.module.SimpleModule; +import io.kafbat.ui.serde.api.SchemaDescription; +import io.kafbat.ui.serde.api.Serde; +import io.kafbat.ui.serdes.BuiltInSerde; import java.io.IOException; +import java.util.Optional; import lombok.SneakyThrows; import org.apache.kafka.common.protocol.types.BoundField; import org.apache.kafka.common.protocol.types.Struct; -public abstract class StructSerde { +public abstract class StructSerde implements BuiltInSerde { private static final JsonMapper JSON_MAPPER = createMapper(); @@ -32,6 +36,31 @@ public void serialize(Struct value, JsonGenerator gen, SerializerProvider serial return mapper; } + @Override + public Optional getDescription() { + return Optional.empty(); + } + + @Override + public Optional getSchema(String topic, Serde.Target type) { + return Optional.empty(); + } + + @Override + public boolean canDeserialize(String topic, Serde.Target type) { + return true; + } + + @Override + public boolean canSerialize(String topic, Serde.Target type) { + return false; + } + + @Override + public Serde.Serializer serializer(String topic, Serde.Target type) { + throw new UnsupportedOperationException(); + } + @SneakyThrows protected String toJson(Struct s) { return JSON_MAPPER.writeValueAsString(s); diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/MirrorMakerSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/MirrorMakerSerde.java index d915a5e22..c0cfc8e02 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/MirrorMakerSerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/MirrorMakerSerde.java @@ -1,8 +1,6 @@ package io.kafbat.ui.serdes.builtin.mm2; import io.kafbat.ui.serde.api.DeserializeResult; -import io.kafbat.ui.serde.api.SchemaDescription; -import io.kafbat.ui.serde.api.Serde; import io.kafbat.ui.serdes.BuiltInSerde; import io.kafbat.ui.serdes.builtin.StructSerde; import java.nio.ByteBuffer; @@ -17,31 +15,6 @@ abstract class MirrorMakerSerde extends StructSerde implements BuiltInSerde { protected final boolean versioned; - @Override - public Optional getDescription() { - return Optional.empty(); - } - - @Override - public Optional getSchema(String topic, Serde.Target type) { - return Optional.empty(); - } - - @Override - public boolean canDeserialize(String topic, Serde.Target type) { - return true; - } - - @Override - public boolean canSerialize(String topic, Serde.Target type) { - return false; - } - - @Override - public Serde.Serializer serializer(String topic, Serde.Target type) { - throw new UnsupportedOperationException(); - } - @Override public Deserializer deserializer(String topic, Target target) { return (recordHeaders, bytes) -> From dfa9cbbcc271565b39d4e5764438a6eadc569fa1 Mon Sep 17 00:00:00 2001 From: German Osin Date: Mon, 29 Sep 2025 23:51:08 +0200 Subject: [PATCH 16/20] Reduced duplicate code --- .../ui/serdes/builtin/ConsumerOffsetsSerde.java | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/ConsumerOffsetsSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/ConsumerOffsetsSerde.java index 2071912b5..38f719b5c 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/ConsumerOffsetsSerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/ConsumerOffsetsSerde.java @@ -1,24 +1,13 @@ package io.kafbat.ui.serdes.builtin; -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.databind.JsonSerializer; -import com.fasterxml.jackson.databind.SerializerProvider; -import com.fasterxml.jackson.databind.json.JsonMapper; -import com.fasterxml.jackson.databind.module.SimpleModule; import io.kafbat.ui.serde.api.DeserializeResult; -import io.kafbat.ui.serde.api.SchemaDescription; import io.kafbat.ui.serdes.BuiltInSerde; -import java.io.IOException; import java.nio.ByteBuffer; import java.util.Map; -import java.util.Optional; -import lombok.SneakyThrows; import org.apache.kafka.common.protocol.types.ArrayOf; -import org.apache.kafka.common.protocol.types.BoundField; import org.apache.kafka.common.protocol.types.CompactArrayOf; import org.apache.kafka.common.protocol.types.Field; import org.apache.kafka.common.protocol.types.Schema; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.protocol.types.Type; // Deserialization logic and message's schemas can be found in From 6b2431c19a1c5b5673659ba935119c6efd17d4f5 Mon Sep 17 00:00:00 2001 From: German Osin Date: Tue, 30 Sep 2025 10:10:36 +0200 Subject: [PATCH 17/20] Reduced duplicate code --- .../io/kafbat/ui/serdes/BuiltInSerde.java | 23 +++++++++++++ .../kafbat/ui/serdes/SerdesInitializer.java | 33 +++++++++---------- .../ui/serdes/builtin/AvroEmbeddedSerde.java | 15 --------- .../kafbat/ui/serdes/builtin/Base64Serde.java | 10 ------ .../io/kafbat/ui/serdes/builtin/HexSerde.java | 10 ------ .../kafbat/ui/serdes/builtin/Int32Serde.java | 5 --- .../kafbat/ui/serdes/builtin/Int64Serde.java | 5 --- .../ui/serdes/builtin/ProtobufFileSerde.java | 5 --- .../ui/serdes/builtin/ProtobufRawSerde.java | 19 ----------- .../kafbat/ui/serdes/builtin/StringSerde.java | 10 ------ .../kafbat/ui/serdes/builtin/StructSerde.java | 22 ------------- .../kafbat/ui/serdes/builtin/UInt32Serde.java | 5 --- .../kafbat/ui/serdes/builtin/UInt64Serde.java | 5 --- .../ui/serdes/builtin/UuidBinarySerde.java | 10 ------ .../builtin/sr/SchemaRegistrySerde.java | 7 +--- 15 files changed, 39 insertions(+), 145 deletions(-) diff --git a/api/src/main/java/io/kafbat/ui/serdes/BuiltInSerde.java b/api/src/main/java/io/kafbat/ui/serdes/BuiltInSerde.java index 0b4755ba3..439f69ded 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/BuiltInSerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/BuiltInSerde.java @@ -1,7 +1,9 @@ package io.kafbat.ui.serdes; import io.kafbat.ui.serde.api.PropertyResolver; +import io.kafbat.ui.serde.api.SchemaDescription; import io.kafbat.ui.serde.api.Serde; +import java.util.Optional; public interface BuiltInSerde extends Serde { @@ -24,4 +26,25 @@ default void configure(PropertyResolver serdeProperties, PropertyResolver kafkaClusterProperties, PropertyResolver globalProperties) { } + + @Override + default boolean canSerialize(String topic, Serde.Target type) { + return false; + } + + @Override + default Serde.Serializer serializer(String topic, Serde.Target type) { + throw new UnsupportedOperationException(); + } + + @Override + default Optional getSchema(String topic, Serde.Target type) { + return Optional.empty(); + } + + @Override + default Optional getDescription() { + return Optional.empty(); + } + } diff --git a/api/src/main/java/io/kafbat/ui/serdes/SerdesInitializer.java b/api/src/main/java/io/kafbat/ui/serdes/SerdesInitializer.java index 55a9e3d8a..0b9158fdb 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/SerdesInitializer.java +++ b/api/src/main/java/io/kafbat/ui/serdes/SerdesInitializer.java @@ -150,13 +150,13 @@ public ClusterSerdes init(Environment env, * Registers serdse that should only be used for specific (hard-coded) topics, like ConsumerOffsetsSerde. */ private void registerTopicRelatedSerde(Map serdes) { - registerConsumerOffsetsSerde(serdes); - registerMirrorMakerSerdes(serdes); + serdes.putAll(consumerOffsetsSerde(serdes)); + serdes.putAll(mirrorMakerSerdes(serdes)); } - private void registerConsumerOffsetsSerde(Map serdes) { + private Map consumerOffsetsSerde(Map serdes) { var pattern = Pattern.compile(ConsumerOffsetsSerde.TOPIC); - serdes.put( + return Map.of( ConsumerOffsetsSerde.name(), new SerdeInstance( ConsumerOffsetsSerde.name(), @@ -168,22 +168,19 @@ private void registerConsumerOffsetsSerde(Map serdes) { ); } - private void registerMirrorMakerSerdes(Map serdes) { - Map> mmSerdes = Map.of( - HeartbeatSerde.name(), Map.entry(HeartbeatSerde.TOPIC_NAME_PATTERN, new HeartbeatSerde()), - OffsetSyncSerde.name(), Map.entry(OffsetSyncSerde.TOPIC_NAME_PATTERN, new OffsetSyncSerde()), - CheckpointSerde.name(), Map.entry(CheckpointSerde.TOPIC_NAME_PATTERN, new CheckpointSerde()) + private Map mirrorMakerSerdes(Map serdes) { + return Map.of( + HeartbeatSerde.name(), + mirrorSerde(HeartbeatSerde.name(), HeartbeatSerde.TOPIC_NAME_PATTERN, new HeartbeatSerde()), + OffsetSyncSerde.name(), + mirrorSerde(HeartbeatSerde.name(), OffsetSyncSerde.TOPIC_NAME_PATTERN, new OffsetSyncSerde()), + CheckpointSerde.name(), + mirrorSerde(HeartbeatSerde.name(), CheckpointSerde.TOPIC_NAME_PATTERN, new CheckpointSerde()) ); - for (Map.Entry> serde : mmSerdes.entrySet()) { - String name = serde.getKey(); - Pattern pattern = serde.getValue().getKey(); - BuiltInSerde serdeInstance = serde.getValue().getValue(); + } - serdes.put( - name, - new SerdeInstance(name, serdeInstance, pattern, pattern, null) - ); - } + private SerdeInstance mirrorSerde(String name, Pattern pattern, BuiltInSerde serde) { + return new SerdeInstance(name, serde, pattern, pattern, null); } private SerdeInstance createFallbackSerde() { diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/AvroEmbeddedSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/AvroEmbeddedSerde.java index 8d5d3dfa5..6c82b9979 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/AvroEmbeddedSerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/AvroEmbeddedSerde.java @@ -18,26 +18,11 @@ public static String name() { return "Avro (Embedded)"; } - @Override - public Optional getDescription() { - return Optional.empty(); - } - - @Override - public Optional getSchema(String topic, Target type) { - return Optional.empty(); - } - @Override public boolean canDeserialize(String topic, Target type) { return true; } - @Override - public boolean canSerialize(String topic, Target type) { - return false; - } - @Override public Serializer serializer(String topic, Target type) { throw new IllegalStateException(); diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/Base64Serde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/Base64Serde.java index 515354695..7fd707022 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/Base64Serde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/Base64Serde.java @@ -14,16 +14,6 @@ public static String name() { return "Base64"; } - @Override - public Optional getDescription() { - return Optional.empty(); - } - - @Override - public Optional getSchema(String topic, Serde.Target type) { - return Optional.empty(); - } - @Override public boolean canDeserialize(String topic, Serde.Target type) { return true; diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/HexSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/HexSerde.java index ab7f66ebb..5c084fc76 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/HexSerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/HexSerde.java @@ -37,16 +37,6 @@ private void configure(String delim, boolean uppercase) { } } - @Override - public Optional getDescription() { - return Optional.empty(); - } - - @Override - public Optional getSchema(String topic, Target type) { - return Optional.empty(); - } - @Override public boolean canDeserialize(String topic, Target type) { return true; diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/Int32Serde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/Int32Serde.java index d402a78b9..ccf5f05a1 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/Int32Serde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/Int32Serde.java @@ -13,11 +13,6 @@ public static String name() { return "Int32"; } - @Override - public Optional getDescription() { - return Optional.empty(); - } - @Override public Optional getSchema(String topic, Target type) { return Optional.of( diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/Int64Serde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/Int64Serde.java index 0616eff2f..ba4e30de9 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/Int64Serde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/Int64Serde.java @@ -14,11 +14,6 @@ public static String name() { return "Int64"; } - @Override - public Optional getDescription() { - return Optional.empty(); - } - @Override public Optional getSchema(String topic, Serde.Target type) { return Optional.of( diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/ProtobufFileSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/ProtobufFileSerde.java index 9b78cc7b2..f76b233f0 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/ProtobufFileSerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/ProtobufFileSerde.java @@ -121,11 +121,6 @@ void configure(Configuration configuration) { this.keyMessageDescriptorMap = configuration.keyMessageDescriptorMap(); } - @Override - public Optional getDescription() { - return Optional.empty(); - } - private Optional descriptorFor(String topic, Serde.Target type) { return type == Serde.Target.KEY ? diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/ProtobufRawSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/ProtobufRawSerde.java index f55ddd93a..25a668a32 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/ProtobufRawSerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/ProtobufRawSerde.java @@ -17,31 +17,12 @@ public static String name() { return "ProtobufDecodeRaw"; } - @Override - public Optional getDescription() { - return Optional.empty(); - } - - @Override - public Optional getSchema(String topic, Serde.Target type) { - return Optional.empty(); - } - - @Override - public boolean canSerialize(String topic, Serde.Target type) { - return false; - } @Override public boolean canDeserialize(String topic, Serde.Target type) { return true; } - @Override - public Serde.Serializer serializer(String topic, Serde.Target type) { - throw new UnsupportedOperationException(); - } - @Override public Serde.Deserializer deserializer(String topic, Serde.Target type) { return new Serde.Deserializer() { diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/StringSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/StringSerde.java index d4351bbc7..a3d1b4780 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/StringSerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/StringSerde.java @@ -26,16 +26,6 @@ public void configure(PropertyResolver serdeProperties, .ifPresent(e -> StringSerde.this.encoding = e); } - @Override - public Optional getDescription() { - return Optional.empty(); - } - - @Override - public Optional getSchema(String topic, Target type) { - return Optional.empty(); - } - @Override public boolean canDeserialize(String topic, Target type) { return true; diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/StructSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/StructSerde.java index f236fd58c..6e7e049ee 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/StructSerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/StructSerde.java @@ -5,11 +5,9 @@ import com.fasterxml.jackson.databind.SerializerProvider; import com.fasterxml.jackson.databind.json.JsonMapper; import com.fasterxml.jackson.databind.module.SimpleModule; -import io.kafbat.ui.serde.api.SchemaDescription; import io.kafbat.ui.serde.api.Serde; import io.kafbat.ui.serdes.BuiltInSerde; import java.io.IOException; -import java.util.Optional; import lombok.SneakyThrows; import org.apache.kafka.common.protocol.types.BoundField; import org.apache.kafka.common.protocol.types.Struct; @@ -36,31 +34,11 @@ public void serialize(Struct value, JsonGenerator gen, SerializerProvider serial return mapper; } - @Override - public Optional getDescription() { - return Optional.empty(); - } - - @Override - public Optional getSchema(String topic, Serde.Target type) { - return Optional.empty(); - } - @Override public boolean canDeserialize(String topic, Serde.Target type) { return true; } - @Override - public boolean canSerialize(String topic, Serde.Target type) { - return false; - } - - @Override - public Serde.Serializer serializer(String topic, Serde.Target type) { - throw new UnsupportedOperationException(); - } - @SneakyThrows protected String toJson(Struct s) { return JSON_MAPPER.writeValueAsString(s); diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/UInt32Serde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/UInt32Serde.java index a767e1a62..cbb898c31 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/UInt32Serde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/UInt32Serde.java @@ -15,11 +15,6 @@ public static String name() { return "UInt32"; } - @Override - public Optional getDescription() { - return Optional.empty(); - } - @Override public Optional getSchema(String topic, Serde.Target type) { return Optional.of( diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/UInt64Serde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/UInt64Serde.java index 4b090329a..8ee3f734a 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/UInt64Serde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/UInt64Serde.java @@ -15,11 +15,6 @@ public static String name() { return "UInt64"; } - @Override - public Optional getDescription() { - return Optional.empty(); - } - @Override public Optional getSchema(String topic, Target type) { return Optional.of( diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/UuidBinarySerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/UuidBinarySerde.java index f94232aee..1d70e118d 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/UuidBinarySerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/UuidBinarySerde.java @@ -28,16 +28,6 @@ public void configure(PropertyResolver serdeProperties, .ifPresent(msb -> UuidBinarySerde.this.mostSignificantBitsFirst = msb); } - @Override - public Optional getDescription() { - return Optional.empty(); - } - - @Override - public Optional getSchema(String topic, Serde.Target type) { - return Optional.empty(); - } - @Override public boolean canDeserialize(String topic, Serde.Target type) { return true; diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/sr/SchemaRegistrySerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/sr/SchemaRegistrySerde.java index d6f7a3699..30c4113fe 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/sr/SchemaRegistrySerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/sr/SchemaRegistrySerde.java @@ -174,11 +174,6 @@ private static SchemaRegistryClient createSchemaRegistryClient(List urls ); } - @Override - public Optional getDescription() { - return Optional.empty(); - } - @Override public boolean canDeserialize(String topic, Target type) { String subject = schemaSubject(topic, type); @@ -213,7 +208,7 @@ public Optional getSchema(String topic, Target type) { @SneakyThrows private String convertSchema(SchemaMetadata schema, ParsedSchema parsedSchema) { - URI basePath = new URI(schemaRegistryUrls.get(0)) + URI basePath = new URI(schemaRegistryUrls.getFirst()) .resolve(Integer.toString(schema.getId())); SchemaType schemaType = SchemaType.fromString(schema.getSchemaType()) .orElseThrow(() -> new IllegalStateException("Unknown schema type: " + schema.getSchemaType())); From 35f6c081a2b6d19a675382a313d47bf5d08fbca7 Mon Sep 17 00:00:00 2001 From: German Osin Date: Tue, 30 Sep 2025 10:27:10 +0200 Subject: [PATCH 18/20] Reduced duplicate code --- .../io/kafbat/ui/serdes/ClusterSerdes.java | 4 +- .../kafbat/ui/serdes/SerdesInitializer.java | 58 +++++++-------- .../ui/serdes/builtin/AvroEmbeddedSerde.java | 5 +- .../kafbat/ui/serdes/builtin/Base64Serde.java | 7 +- .../serdes/builtin/ConsumerOffsetsSerde.java | 5 +- .../io/kafbat/ui/serdes/builtin/HexSerde.java | 7 +- .../kafbat/ui/serdes/builtin/Int32Serde.java | 5 +- .../kafbat/ui/serdes/builtin/Int64Serde.java | 5 +- .../ui/serdes/builtin/ProtobufFileSerde.java | 7 +- .../ui/serdes/builtin/ProtobufRawSerde.java | 7 +- .../kafbat/ui/serdes/builtin/StringSerde.java | 7 +- .../kafbat/ui/serdes/builtin/UInt32Serde.java | 5 +- .../kafbat/ui/serdes/builtin/UInt64Serde.java | 5 +- .../ui/serdes/builtin/UuidBinarySerde.java | 7 +- .../serdes/builtin/mm2/CheckpointSerde.java | 6 +- .../ui/serdes/builtin/mm2/HeartbeatSerde.java | 6 +- .../serdes/builtin/mm2/OffsetSyncSerde.java | 6 +- .../builtin/sr/SchemaRegistrySerde.java | 8 +-- .../java/io/kafbat/ui/emitter/CursorTest.java | 6 +- .../kafbat/ui/emitter/TailingEmitterTest.java | 4 +- .../ui/serdes/SerdesInitializerTest.java | 4 +- .../ui/service/MessagesServiceTest.java | 10 +-- .../kafbat/ui/service/RecordEmitterTest.java | 6 +- .../kafbat/ui/service/SendAndReadTests.java | 72 +++++++++---------- 24 files changed, 101 insertions(+), 161 deletions(-) diff --git a/api/src/main/java/io/kafbat/ui/serdes/ClusterSerdes.java b/api/src/main/java/io/kafbat/ui/serdes/ClusterSerdes.java index fbdfdda48..b73ec193d 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/ClusterSerdes.java +++ b/api/src/main/java/io/kafbat/ui/serdes/ClusterSerdes.java @@ -64,12 +64,12 @@ public Stream all() { public SerdeInstance suggestSerdeForSerialize(String topic, Serde.Target type) { return findSerdeByPatternsOrDefault(topic, type, s -> s.canSerialize(topic, type)) - .orElse(serdes.get(StringSerde.name())); + .orElse(serdes.get(StringSerde.NAME)); } public SerdeInstance suggestSerdeForDeserialize(String topic, Serde.Target type) { return findSerdeByPatternsOrDefault(topic, type, s -> s.canDeserialize(topic, type)) - .orElse(serdes.get(StringSerde.name())); + .orElse(serdes.get(StringSerde.NAME)); } @Override diff --git a/api/src/main/java/io/kafbat/ui/serdes/SerdesInitializer.java b/api/src/main/java/io/kafbat/ui/serdes/SerdesInitializer.java index 0b9158fdb..25d9bc73f 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/SerdesInitializer.java +++ b/api/src/main/java/io/kafbat/ui/serdes/SerdesInitializer.java @@ -42,23 +42,23 @@ public class SerdesInitializer { public SerdesInitializer() { this( ImmutableMap.>builder() - .put(StringSerde.name(), StringSerde.class) - .put(SchemaRegistrySerde.name(), SchemaRegistrySerde.class) - .put(ProtobufFileSerde.name(), ProtobufFileSerde.class) - .put(Int32Serde.name(), Int32Serde.class) - .put(Int64Serde.name(), Int64Serde.class) - .put(UInt32Serde.name(), UInt32Serde.class) - .put(UInt64Serde.name(), UInt64Serde.class) - .put(AvroEmbeddedSerde.name(), AvroEmbeddedSerde.class) - .put(Base64Serde.name(), Base64Serde.class) - .put(HexSerde.name(), HexSerde.class) - .put(UuidBinarySerde.name(), UuidBinarySerde.class) - .put(ProtobufRawSerde.name(), ProtobufRawSerde.class) + .put(StringSerde.NAME, StringSerde.class) + .put(SchemaRegistrySerde.NAME, SchemaRegistrySerde.class) + .put(ProtobufFileSerde.NAME, ProtobufFileSerde.class) + .put(Int32Serde.NAME, Int32Serde.class) + .put(Int64Serde.NAME, Int64Serde.class) + .put(UInt32Serde.NAME, UInt32Serde.class) + .put(UInt64Serde.NAME, UInt64Serde.class) + .put(AvroEmbeddedSerde.NAME, AvroEmbeddedSerde.class) + .put(Base64Serde.NAME, Base64Serde.class) + .put(HexSerde.NAME, HexSerde.class) + .put(UuidBinarySerde.NAME, UuidBinarySerde.class) + .put(ProtobufRawSerde.NAME, ProtobufRawSerde.class) // mm2 serdes - .put(HeartbeatSerde.name(), HeartbeatSerde.class) - .put(OffsetSyncSerde.name(), OffsetSyncSerde.class) - .put(CheckpointSerde.name(), CheckpointSerde.class) + .put(HeartbeatSerde.NAME, HeartbeatSerde.class) + .put(OffsetSyncSerde.NAME, OffsetSyncSerde.class) + .put(CheckpointSerde.NAME, CheckpointSerde.class) .build(), new CustomSerdeLoader() ); @@ -139,8 +139,8 @@ public ClusterSerdes init(Environment env, .orElse(null), Optional.ofNullable(clusterProperties.getDefaultValueSerde()) .map(name -> Preconditions.checkNotNull(registeredSerdes.get(name), "Default value serde not found")) - .or(() -> Optional.ofNullable(registeredSerdes.get(SchemaRegistrySerde.name()))) - .or(() -> Optional.ofNullable(registeredSerdes.get(ProtobufFileSerde.name()))) + .or(() -> Optional.ofNullable(registeredSerdes.get(SchemaRegistrySerde.NAME))) + .or(() -> Optional.ofNullable(registeredSerdes.get(ProtobufFileSerde.NAME))) .orElse(null), createFallbackSerde() ); @@ -150,16 +150,16 @@ public ClusterSerdes init(Environment env, * Registers serdse that should only be used for specific (hard-coded) topics, like ConsumerOffsetsSerde. */ private void registerTopicRelatedSerde(Map serdes) { - serdes.putAll(consumerOffsetsSerde(serdes)); - serdes.putAll(mirrorMakerSerdes(serdes)); + serdes.putAll(consumerOffsetsSerde()); + serdes.putAll(mirrorMakerSerdes()); } - private Map consumerOffsetsSerde(Map serdes) { + private Map consumerOffsetsSerde() { var pattern = Pattern.compile(ConsumerOffsetsSerde.TOPIC); return Map.of( - ConsumerOffsetsSerde.name(), + ConsumerOffsetsSerde.NAME, new SerdeInstance( - ConsumerOffsetsSerde.name(), + ConsumerOffsetsSerde.NAME, new ConsumerOffsetsSerde(), pattern, pattern, @@ -168,14 +168,14 @@ private Map consumerOffsetsSerde(Map mirrorMakerSerdes(Map serdes) { + private Map mirrorMakerSerdes() { return Map.of( - HeartbeatSerde.name(), - mirrorSerde(HeartbeatSerde.name(), HeartbeatSerde.TOPIC_NAME_PATTERN, new HeartbeatSerde()), - OffsetSyncSerde.name(), - mirrorSerde(HeartbeatSerde.name(), OffsetSyncSerde.TOPIC_NAME_PATTERN, new OffsetSyncSerde()), - CheckpointSerde.name(), - mirrorSerde(HeartbeatSerde.name(), CheckpointSerde.TOPIC_NAME_PATTERN, new CheckpointSerde()) + HeartbeatSerde.NAME, + mirrorSerde(HeartbeatSerde.NAME, HeartbeatSerde.TOPIC_NAME_PATTERN, new HeartbeatSerde()), + OffsetSyncSerde.NAME, + mirrorSerde(HeartbeatSerde.NAME, OffsetSyncSerde.TOPIC_NAME_PATTERN, new OffsetSyncSerde()), + CheckpointSerde.NAME, + mirrorSerde(HeartbeatSerde.NAME, CheckpointSerde.TOPIC_NAME_PATTERN, new CheckpointSerde()) ); } diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/AvroEmbeddedSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/AvroEmbeddedSerde.java index 6c82b9979..60874d3cd 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/AvroEmbeddedSerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/AvroEmbeddedSerde.java @@ -13,10 +13,7 @@ import org.apache.avro.generic.GenericDatumReader; public class AvroEmbeddedSerde implements BuiltInSerde { - - public static String name() { - return "Avro (Embedded)"; - } + public static final String NAME = "Avro (Embedded)"; @Override public boolean canDeserialize(String topic, Target type) { diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/Base64Serde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/Base64Serde.java index 7fd707022..fff282714 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/Base64Serde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/Base64Serde.java @@ -1,18 +1,13 @@ package io.kafbat.ui.serdes.builtin; import io.kafbat.ui.serde.api.DeserializeResult; -import io.kafbat.ui.serde.api.SchemaDescription; import io.kafbat.ui.serde.api.Serde; import io.kafbat.ui.serdes.BuiltInSerde; import java.util.Base64; import java.util.Map; -import java.util.Optional; public class Base64Serde implements BuiltInSerde { - - public static String name() { - return "Base64"; - } + public static final String NAME = "Base64"; @Override public boolean canDeserialize(String topic, Serde.Target type) { diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/ConsumerOffsetsSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/ConsumerOffsetsSerde.java index 38f719b5c..9246f705e 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/ConsumerOffsetsSerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/ConsumerOffsetsSerde.java @@ -13,6 +13,7 @@ // Deserialization logic and message's schemas can be found in // kafka.coordinator.group.GroupMetadataManager (readMessageKey, readOffsetMessageValue, readGroupMessageValue) public class ConsumerOffsetsSerde extends StructSerde implements BuiltInSerde { + public static final String NAME = "__consumer_offsets"; private static final String ASSIGNMENT = "assignment"; private static final String CLIENT_HOST = "client_host"; @@ -33,10 +34,6 @@ public class ConsumerOffsetsSerde extends StructSerde implements BuiltInSerde { public static final String TOPIC = "__consumer_offsets"; - public static String name() { - return "__consumer_offsets"; - } - @Override public boolean canDeserialize(String topic, Target type) { return topic.equals(TOPIC); diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/HexSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/HexSerde.java index 5c084fc76..ab6467b52 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/HexSerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/HexSerde.java @@ -2,20 +2,15 @@ import io.kafbat.ui.serde.api.DeserializeResult; import io.kafbat.ui.serde.api.PropertyResolver; -import io.kafbat.ui.serde.api.SchemaDescription; import io.kafbat.ui.serdes.BuiltInSerde; import java.util.HexFormat; import java.util.Map; -import java.util.Optional; public class HexSerde implements BuiltInSerde { + public static final String NAME = "Hex"; private HexFormat deserializeHexFormat; - public static String name() { - return "Hex"; - } - @Override public void autoConfigure(PropertyResolver kafkaClusterProperties, PropertyResolver globalProperties) { configure(" ", true); diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/Int32Serde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/Int32Serde.java index ccf5f05a1..d5f6163f0 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/Int32Serde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/Int32Serde.java @@ -8,10 +8,7 @@ import java.util.Optional; public class Int32Serde implements BuiltInSerde { - - public static String name() { - return "Int32"; - } + public static final String NAME = "Int32"; @Override public Optional getSchema(String topic, Target type) { diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/Int64Serde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/Int64Serde.java index ba4e30de9..e4ffbfe92 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/Int64Serde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/Int64Serde.java @@ -9,10 +9,7 @@ import java.util.Optional; public class Int64Serde implements BuiltInSerde { - - public static String name() { - return "Int64"; - } + public static final String NAME = "Int64"; @Override public Optional getSchema(String topic, Serde.Target type) { diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/ProtobufFileSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/ProtobufFileSerde.java index f76b233f0..a2432c9f9 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/ProtobufFileSerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/ProtobufFileSerde.java @@ -69,10 +69,7 @@ @Slf4j public class ProtobufFileSerde implements BuiltInSerde { - - public static String name() { - return "ProtobufFile"; - } + public static final String NAME = "ProtobufFile"; private static final ProtobufSchemaConverter SCHEMA_CONVERTER = new ProtobufSchemaConverter(); @@ -112,7 +109,7 @@ void configure(Configuration configuration) { && configuration.defaultKeyMessageDescriptor() == null && configuration.messageDescriptorMap().isEmpty() && configuration.keyMessageDescriptorMap().isEmpty()) { - throw new ValidationException("Neither default, nor per-topic descriptors defined for " + name() + " serde"); + throw new ValidationException("Neither default, nor per-topic descriptors defined for " + NAME + " serde"); } this.defaultMessageDescriptor = configuration.defaultMessageDescriptor(); this.defaultKeyMessageDescriptor = configuration.defaultKeyMessageDescriptor(); diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/ProtobufRawSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/ProtobufRawSerde.java index 25a668a32..bb2e1eac3 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/ProtobufRawSerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/ProtobufRawSerde.java @@ -12,17 +12,14 @@ import lombok.SneakyThrows; public class ProtobufRawSerde implements BuiltInSerde { - - public static String name() { - return "ProtobufDecodeRaw"; - } - + public static final String NAME = "ProtobufDecodeRaw"; @Override public boolean canDeserialize(String topic, Serde.Target type) { return true; } + @Override public Serde.Deserializer deserializer(String topic, Serde.Target type) { return new Serde.Deserializer() { diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/StringSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/StringSerde.java index a3d1b4780..2ce85e763 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/StringSerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/StringSerde.java @@ -2,18 +2,13 @@ import io.kafbat.ui.serde.api.DeserializeResult; import io.kafbat.ui.serde.api.PropertyResolver; -import io.kafbat.ui.serde.api.SchemaDescription; import io.kafbat.ui.serdes.BuiltInSerde; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.util.Map; -import java.util.Optional; public class StringSerde implements BuiltInSerde { - - public static String name() { - return "String"; - } + public static final String NAME = "String"; private Charset encoding = StandardCharsets.UTF_8; diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/UInt32Serde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/UInt32Serde.java index cbb898c31..63dce83dc 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/UInt32Serde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/UInt32Serde.java @@ -10,10 +10,7 @@ import java.util.Optional; public class UInt32Serde implements BuiltInSerde { - - public static String name() { - return "UInt32"; - } + public static final String NAME = "UInt32"; @Override public Optional getSchema(String topic, Serde.Target type) { diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/UInt64Serde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/UInt64Serde.java index 8ee3f734a..e89274c31 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/UInt64Serde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/UInt64Serde.java @@ -10,10 +10,7 @@ public class UInt64Serde implements BuiltInSerde { - - public static String name() { - return "UInt64"; - } + public static final String NAME = "UInt64"; @Override public Optional getSchema(String topic, Target type) { diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/UuidBinarySerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/UuidBinarySerde.java index 1d70e118d..0a9178fc6 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/UuidBinarySerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/UuidBinarySerde.java @@ -3,20 +3,15 @@ import io.kafbat.ui.exception.ValidationException; import io.kafbat.ui.serde.api.DeserializeResult; import io.kafbat.ui.serde.api.PropertyResolver; -import io.kafbat.ui.serde.api.SchemaDescription; import io.kafbat.ui.serde.api.Serde; import io.kafbat.ui.serdes.BuiltInSerde; import java.nio.ByteBuffer; import java.util.Map; -import java.util.Optional; import java.util.UUID; public class UuidBinarySerde implements BuiltInSerde { - - public static String name() { - return "UUIDBinary"; - } + public static final String NAME = "UUIDBinary"; private boolean mostSignificantBitsFirst = true; diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/CheckpointSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/CheckpointSerde.java index ef1cbec62..2b7d2bea1 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/CheckpointSerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/CheckpointSerde.java @@ -10,7 +10,7 @@ @Slf4j public class CheckpointSerde extends MirrorMakerSerde implements BuiltInSerde { - + public static final String NAME = "Checkpoint"; public static final Pattern TOPIC_NAME_PATTERN = Pattern.compile(".*\\.checkpoints\\.internal"); private static final String TOPIC_KEY = "topic"; @@ -34,10 +34,6 @@ public CheckpointSerde() { super(true); } - public static String name() { - return "Checkpoint"; - } - @Override protected Schema getKeySchema() { return KEY_SCHEMA; diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerde.java index f8e54352d..440ce80a1 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerde.java @@ -10,7 +10,7 @@ @Slf4j public class HeartbeatSerde extends MirrorMakerSerde implements BuiltInSerde { - + public static final String NAME = "Heartbeat"; public static final Pattern TOPIC_NAME_PATTERN = Pattern.compile("heartbeats"); private static final String SOURCE_CLUSTER_ALIAS_KEY = "sourceClusterAlias"; @@ -28,10 +28,6 @@ public HeartbeatSerde() { super(true); } - public static String name() { - return "Heartbeat"; - } - protected Schema getKeySchema() { return KEY_SCHEMA; } diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/OffsetSyncSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/OffsetSyncSerde.java index b355251b7..7a641be21 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/OffsetSyncSerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/OffsetSyncSerde.java @@ -10,7 +10,7 @@ @Slf4j public class OffsetSyncSerde extends MirrorMakerSerde implements BuiltInSerde { - + public static final String NAME = "OffsetSync"; public static final Pattern TOPIC_NAME_PATTERN = Pattern.compile("mm2-offset-syncs\\..*\\.internal"); private static final Schema VALUE_SCHEMA; @@ -31,10 +31,6 @@ public OffsetSyncSerde() { super(false); } - public static String name() { - return "OffsetSync"; - } - @Override protected Schema getKeySchema() { return KEY_SCHEMA; diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/sr/SchemaRegistrySerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/sr/SchemaRegistrySerde.java index 30c4113fe..91c2375d8 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/sr/SchemaRegistrySerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/sr/SchemaRegistrySerde.java @@ -39,14 +39,10 @@ public class SchemaRegistrySerde implements BuiltInSerde { - + public static final String NAME = "SchemaRegistry"; private static final byte SR_PAYLOAD_MAGIC_BYTE = 0x0; private static final int SR_PAYLOAD_PREFIX_LENGTH = 5; - public static String name() { - return "SchemaRegistry"; - } - private static final String SCHEMA_REGISTRY = "schemaRegistry"; private SchemaRegistryClient schemaRegistryClient; @@ -303,7 +299,7 @@ private int extractSchemaIdFromMsg(byte[] data) { throw new ValidationException( String.format( "Data doesn't contain magic byte and schema id prefix, so it can't be deserialized with %s serde", - name()) + NAME) ); } } diff --git a/api/src/test/java/io/kafbat/ui/emitter/CursorTest.java b/api/src/test/java/io/kafbat/ui/emitter/CursorTest.java index 5a262b132..095c373a7 100644 --- a/api/src/test/java/io/kafbat/ui/emitter/CursorTest.java +++ b/api/src/test/java/io/kafbat/ui/emitter/CursorTest.java @@ -181,11 +181,11 @@ private static ConsumerRecordDeserializer createRecordsDeserializer() { Serde s = new StringSerde(); s.configure(PropertyResolverImpl.empty(), PropertyResolverImpl.empty(), PropertyResolverImpl.empty()); return new ConsumerRecordDeserializer( - StringSerde.name(), + StringSerde.NAME, s.deserializer(null, Serde.Target.KEY), - StringSerde.name(), + StringSerde.NAME, s.deserializer(null, Serde.Target.VALUE), - StringSerde.name(), + StringSerde.NAME, s.deserializer(null, Serde.Target.KEY), s.deserializer(null, Serde.Target.VALUE), msg -> msg diff --git a/api/src/test/java/io/kafbat/ui/emitter/TailingEmitterTest.java b/api/src/test/java/io/kafbat/ui/emitter/TailingEmitterTest.java index c1130c9d2..f3c11ff62 100644 --- a/api/src/test/java/io/kafbat/ui/emitter/TailingEmitterTest.java +++ b/api/src/test/java/io/kafbat/ui/emitter/TailingEmitterTest.java @@ -114,8 +114,8 @@ private Flux createTailingFlux( query, null, 0, - StringSerde.name(), - StringSerde.name()); + StringSerde.NAME, + StringSerde.NAME); } private List startTailing(String filterQuery) { diff --git a/api/src/test/java/io/kafbat/ui/serdes/SerdesInitializerTest.java b/api/src/test/java/io/kafbat/ui/serdes/SerdesInitializerTest.java index 0d6fc36a3..8b559caa0 100644 --- a/api/src/test/java/io/kafbat/ui/serdes/SerdesInitializerTest.java +++ b/api/src/test/java/io/kafbat/ui/serdes/SerdesInitializerTest.java @@ -31,8 +31,8 @@ class SerdesInitializerTest { Map.of( "BuiltIn1", BuiltInSerdeWithAutoconfigure.class, "BuiltIn2", BuiltInSerdeMock2NoAutoConfigure.class, - Int32Serde.name(), Int32Serde.class, - StringSerde.name(), StringSerde.class + Int32Serde.NAME, Int32Serde.class, + StringSerde.NAME, StringSerde.class ), customSerdeLoaderMock ); diff --git a/api/src/test/java/io/kafbat/ui/service/MessagesServiceTest.java b/api/src/test/java/io/kafbat/ui/service/MessagesServiceTest.java index c0d02f39a..dc8f37711 100644 --- a/api/src/test/java/io/kafbat/ui/service/MessagesServiceTest.java +++ b/api/src/test/java/io/kafbat/ui/service/MessagesServiceTest.java @@ -97,8 +97,8 @@ void maskingAppliedOnConfiguredClusters() throws Exception { null, null, 100, - StringSerde.name(), - StringSerde.name() + StringSerde.NAME, + StringSerde.NAME ).filter(evt -> evt.getType() == TopicMessageEventDTO.TypeEnum.MESSAGE) .map(TopicMessageEventDTO::getMessage); @@ -128,7 +128,7 @@ void cursorIsRegisteredAfterPollingIsDoneAndCanBeUsedForNextPagePolling(PollingM Flux msgsFlux = messagesService.loadMessages( cluster, testTopic, new ConsumerPosition(mode, testTopic, List.of(), null, null), - null, null, pageSize, StringSerde.name(), StringSerde.name()) + null, null, pageSize, StringSerde.NAME, StringSerde.NAME) .doOnNext(evt -> { if (evt.getType() == TopicMessageEventDTO.TypeEnum.DONE) { assertThat(evt.getCursor()).isNotNull(); @@ -230,9 +230,9 @@ void sendMessageWithProtobufAnyType() { CreateTopicMessageDTO testMessage = new CreateTopicMessageDTO() .key(null) .partition(0) - .keySerde(StringSerde.name()) + .keySerde(StringSerde.NAME) .value(jsonContent) - .valueSerde(ProtobufFileSerde.name()); + .valueSerde(ProtobufFileSerde.NAME); String testTopic = MASKED_TOPICS_PREFIX + UUID.randomUUID(); createTopicWithCleanup(new NewTopic(testTopic, 5, (short) 1)); diff --git a/api/src/test/java/io/kafbat/ui/service/RecordEmitterTest.java b/api/src/test/java/io/kafbat/ui/service/RecordEmitterTest.java index cc395ed41..b10e000fa 100644 --- a/api/src/test/java/io/kafbat/ui/service/RecordEmitterTest.java +++ b/api/src/test/java/io/kafbat/ui/service/RecordEmitterTest.java @@ -106,11 +106,11 @@ private static ConsumerRecordDeserializer createRecordsDeserializer() { Serde s = new StringSerde(); s.configure(PropertyResolverImpl.empty(), PropertyResolverImpl.empty(), PropertyResolverImpl.empty()); return new ConsumerRecordDeserializer( - StringSerde.name(), + StringSerde.NAME, s.deserializer(null, Serde.Target.KEY), - StringSerde.name(), + StringSerde.NAME, s.deserializer(null, Serde.Target.VALUE), - StringSerde.name(), + StringSerde.NAME, s.deserializer(null, Serde.Target.KEY), s.deserializer(null, Serde.Target.VALUE), msg -> msg diff --git a/api/src/test/java/io/kafbat/ui/service/SendAndReadTests.java b/api/src/test/java/io/kafbat/ui/service/SendAndReadTests.java index efed56ded..7e21112c1 100644 --- a/api/src/test/java/io/kafbat/ui/service/SendAndReadTests.java +++ b/api/src/test/java/io/kafbat/ui/service/SendAndReadTests.java @@ -142,9 +142,9 @@ void noSchemaStringKeyStringValue() { .withMsgToSend( new CreateTopicMessageDTO() .key("testKey") - .keySerde(StringSerde.name()) + .keySerde(StringSerde.NAME) .value("testValue") - .valueSerde(StringSerde.name()) + .valueSerde(StringSerde.NAME) ) .doAssert(polled -> { assertThat(polled.getKey()).isEqualTo("testKey"); @@ -158,9 +158,9 @@ void keyIsIntValueIsLong() { .withMsgToSend( new CreateTopicMessageDTO() .key("123") - .keySerde(Int32Serde.name()) + .keySerde(Int32Serde.NAME) .value("21474836470") - .valueSerde(Int64Serde.name()) + .valueSerde(Int64Serde.NAME) ) .doAssert(polled -> { assertThat(polled.getKey()).isEqualTo("123"); @@ -174,9 +174,9 @@ void keyIsNull() { .withMsgToSend( new CreateTopicMessageDTO() .key(null) - .keySerde(StringSerde.name()) + .keySerde(StringSerde.NAME) .value("testValue") - .valueSerde(StringSerde.name()) + .valueSerde(StringSerde.NAME) ) .doAssert(polled -> { assertThat(polled.getKey()).isNull(); @@ -190,9 +190,9 @@ void valueIsNull() { .withMsgToSend( new CreateTopicMessageDTO() .key("testKey") - .keySerde(StringSerde.name()) + .keySerde(StringSerde.NAME) .value(null) - .valueSerde(StringSerde.name()) + .valueSerde(StringSerde.NAME) ) .doAssert(polled -> { assertThat(polled.getKey()).isEqualTo("testKey"); @@ -208,9 +208,9 @@ void primitiveAvroSchemas() { .withMsgToSend( new CreateTopicMessageDTO() .key("\"some string\"") - .keySerde(SchemaRegistrySerde.name()) + .keySerde(SchemaRegistrySerde.NAME) .value("123") - .valueSerde(SchemaRegistrySerde.name()) + .valueSerde(SchemaRegistrySerde.NAME) ) .doAssert(polled -> { assertThat(polled.getKey()).isEqualTo("\"some string\""); @@ -226,9 +226,9 @@ void recordAvroSchema() { .withMsgToSend( new CreateTopicMessageDTO() .key(AVRO_SCHEMA_1_JSON_RECORD) - .keySerde(SchemaRegistrySerde.name()) + .keySerde(SchemaRegistrySerde.NAME) .value(AVRO_SCHEMA_2_JSON_RECORD) - .valueSerde(SchemaRegistrySerde.name()) + .valueSerde(SchemaRegistrySerde.NAME) ) .doAssert(polled -> { assertJsonEqual(polled.getKey(), AVRO_SCHEMA_1_JSON_RECORD); @@ -243,9 +243,9 @@ void keyWithNoSchemaValueWithProtoSchema() { .withMsgToSend( new CreateTopicMessageDTO() .key("testKey") - .keySerde(StringSerde.name()) + .keySerde(StringSerde.NAME) .value(PROTOBUF_SCHEMA_JSON_RECORD) - .valueSerde(SchemaRegistrySerde.name()) + .valueSerde(SchemaRegistrySerde.NAME) ) .doAssert(polled -> { assertThat(polled.getKey()).isEqualTo("testKey"); @@ -261,9 +261,9 @@ void keyWithAvroSchemaValueWithAvroSchemaKeyIsNull() { .withMsgToSend( new CreateTopicMessageDTO() .key(null) - .keySerde(SchemaRegistrySerde.name()) + .keySerde(SchemaRegistrySerde.NAME) .value(AVRO_SCHEMA_2_JSON_RECORD) - .valueSerde(SchemaRegistrySerde.name()) + .valueSerde(SchemaRegistrySerde.NAME) ) .doAssert(polled -> { @@ -278,10 +278,10 @@ void valueWithAvroSchemaShouldThrowExceptionIfArgIsNotValidJsonObject() { .withValueSchema(AVRO_SCHEMA_2) .withMsgToSend( new CreateTopicMessageDTO() - .keySerde(StringSerde.name()) + .keySerde(StringSerde.NAME) // f2 has type int instead of string .value("{ \"f1\": 111, \"f2\": 123 }") - .valueSerde(SchemaRegistrySerde.name()) + .valueSerde(SchemaRegistrySerde.NAME) ) .assertSendThrowsException(); } @@ -294,9 +294,9 @@ void keyWithAvroSchemaValueWithProtoSchema() { .withMsgToSend( new CreateTopicMessageDTO() .key(AVRO_SCHEMA_1_JSON_RECORD) - .keySerde(SchemaRegistrySerde.name()) + .keySerde(SchemaRegistrySerde.NAME) .value(PROTOBUF_SCHEMA_JSON_RECORD) - .valueSerde(SchemaRegistrySerde.name()) + .valueSerde(SchemaRegistrySerde.NAME) ) .doAssert(polled -> { assertJsonEqual(polled.getKey(), AVRO_SCHEMA_1_JSON_RECORD); @@ -311,10 +311,10 @@ void valueWithProtoSchemaShouldThrowExceptionArgIsNotValidJsonObject() { .withMsgToSend( new CreateTopicMessageDTO() .key(null) - .keySerde(StringSerde.name()) + .keySerde(StringSerde.NAME) // f2 field has type object instead of int .value("{ \"f1\" : \"test str\", \"f2\" : {} }") - .valueSerde(SchemaRegistrySerde.name()) + .valueSerde(SchemaRegistrySerde.NAME) ) .assertSendThrowsException(); } @@ -327,9 +327,9 @@ void keyWithProtoSchemaValueWithJsonSchema() { .withMsgToSend( new CreateTopicMessageDTO() .key(PROTOBUF_SCHEMA_JSON_RECORD) - .keySerde(SchemaRegistrySerde.name()) + .keySerde(SchemaRegistrySerde.NAME) .value(JSON_SCHEMA_RECORD) - .valueSerde(SchemaRegistrySerde.name()) + .valueSerde(SchemaRegistrySerde.NAME) ) .doAssert(polled -> { assertJsonEqual(polled.getKey(), PROTOBUF_SCHEMA_JSON_RECORD); @@ -344,10 +344,10 @@ void valueWithJsonSchemaThrowsExceptionIfArgIsNotValidJsonObject() { .withMsgToSend( new CreateTopicMessageDTO() .key(null) - .keySerde(StringSerde.name()) + .keySerde(StringSerde.NAME) // 'f2' field has type object instead of string .value("{ \"f1\": 12, \"f2\": {}, \"schema\": \"some txt\" }") - .valueSerde(SchemaRegistrySerde.name()) + .valueSerde(SchemaRegistrySerde.NAME) ) .assertSendThrowsException(); } @@ -360,9 +360,9 @@ void topicMessageMetadataAvro() { .withMsgToSend( new CreateTopicMessageDTO() .key(AVRO_SCHEMA_1_JSON_RECORD) - .keySerde(SchemaRegistrySerde.name()) + .keySerde(SchemaRegistrySerde.NAME) .value(AVRO_SCHEMA_2_JSON_RECORD) - .valueSerde(SchemaRegistrySerde.name()) + .valueSerde(SchemaRegistrySerde.NAME) ) .doAssert(polled -> { assertJsonEqual(polled.getKey(), AVRO_SCHEMA_1_JSON_RECORD); @@ -385,9 +385,9 @@ void topicMessageMetadataProtobuf() { .withMsgToSend( new CreateTopicMessageDTO() .key(PROTOBUF_SCHEMA_JSON_RECORD) - .keySerde(SchemaRegistrySerde.name()) + .keySerde(SchemaRegistrySerde.NAME) .value(PROTOBUF_SCHEMA_JSON_RECORD) - .valueSerde(SchemaRegistrySerde.name()) + .valueSerde(SchemaRegistrySerde.NAME) ) .doAssert(polled -> { assertJsonEqual(polled.getKey(), PROTOBUF_SCHEMA_JSON_RECORD); @@ -409,9 +409,9 @@ void topicMessageMetadataJson() { .withMsgToSend( new CreateTopicMessageDTO() .key(JSON_SCHEMA_RECORD) - .keySerde(SchemaRegistrySerde.name()) + .keySerde(SchemaRegistrySerde.NAME) .value(JSON_SCHEMA_RECORD) - .valueSerde(SchemaRegistrySerde.name()) + .valueSerde(SchemaRegistrySerde.NAME) .headers(Map.of("header1", "value1")) ) .doAssert(polled -> { @@ -435,9 +435,9 @@ void headerValueNullPresentTest() { .withMsgToSend( new CreateTopicMessageDTO() .key(JSON_SCHEMA_RECORD) - .keySerde(SchemaRegistrySerde.name()) + .keySerde(SchemaRegistrySerde.NAME) .value(JSON_SCHEMA_RECORD) - .valueSerde(SchemaRegistrySerde.name()) + .valueSerde(SchemaRegistrySerde.NAME) .headers(Collections.singletonMap("header123", null)) ) .doAssert(polled -> assertThat(polled.getHeaders().get("header123")).isNull()); @@ -450,9 +450,9 @@ void noKeyAndNoContentPresentTest() { .withMsgToSend( new CreateTopicMessageDTO() .key(null) - .keySerde(StringSerde.name()) // any serde + .keySerde(StringSerde.NAME) // any serde .value(null) - .valueSerde(StringSerde.name()) // any serde + .valueSerde(StringSerde.NAME) // any serde ) .doAssert(polled -> { assertThat(polled.getKey()).isNull(); From 435c4fd15d1dd24b28145332059431aff843929f Mon Sep 17 00:00:00 2001 From: Roman Zabaluev Date: Mon, 6 Oct 2025 11:20:28 +0300 Subject: [PATCH 19/20] Fix typos --- api/src/main/java/io/kafbat/ui/serdes/SerdesInitializer.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/src/main/java/io/kafbat/ui/serdes/SerdesInitializer.java b/api/src/main/java/io/kafbat/ui/serdes/SerdesInitializer.java index 25d9bc73f..3b2eb52fc 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/SerdesInitializer.java +++ b/api/src/main/java/io/kafbat/ui/serdes/SerdesInitializer.java @@ -173,9 +173,9 @@ private Map mirrorMakerSerdes() { HeartbeatSerde.NAME, mirrorSerde(HeartbeatSerde.NAME, HeartbeatSerde.TOPIC_NAME_PATTERN, new HeartbeatSerde()), OffsetSyncSerde.NAME, - mirrorSerde(HeartbeatSerde.NAME, OffsetSyncSerde.TOPIC_NAME_PATTERN, new OffsetSyncSerde()), + mirrorSerde(OffsetSyncSerde.NAME, OffsetSyncSerde.TOPIC_NAME_PATTERN, new OffsetSyncSerde()), CheckpointSerde.NAME, - mirrorSerde(HeartbeatSerde.NAME, CheckpointSerde.TOPIC_NAME_PATTERN, new CheckpointSerde()) + mirrorSerde(CheckpointSerde.NAME, CheckpointSerde.TOPIC_NAME_PATTERN, new CheckpointSerde()) ); } From f510fbd8c4ddc68213e655a65acb08812fd34e68 Mon Sep 17 00:00:00 2001 From: Roman Zabaluev Date: Mon, 6 Oct 2025 15:55:28 +0300 Subject: [PATCH 20/20] Add prefix to serde names --- .../java/io/kafbat/ui/serdes/builtin/mm2/CheckpointSerde.java | 2 +- .../java/io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerde.java | 2 +- .../java/io/kafbat/ui/serdes/builtin/mm2/OffsetSyncSerde.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/CheckpointSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/CheckpointSerde.java index 2b7d2bea1..01ddfad05 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/CheckpointSerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/CheckpointSerde.java @@ -10,7 +10,7 @@ @Slf4j public class CheckpointSerde extends MirrorMakerSerde implements BuiltInSerde { - public static final String NAME = "Checkpoint"; + public static final String NAME = "mm2-Checkpoint"; public static final Pattern TOPIC_NAME_PATTERN = Pattern.compile(".*\\.checkpoints\\.internal"); private static final String TOPIC_KEY = "topic"; diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerde.java index 440ce80a1..92f05e442 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/HeartbeatSerde.java @@ -10,7 +10,7 @@ @Slf4j public class HeartbeatSerde extends MirrorMakerSerde implements BuiltInSerde { - public static final String NAME = "Heartbeat"; + public static final String NAME = "mm2-Heartbeat"; public static final Pattern TOPIC_NAME_PATTERN = Pattern.compile("heartbeats"); private static final String SOURCE_CLUSTER_ALIAS_KEY = "sourceClusterAlias"; diff --git a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/OffsetSyncSerde.java b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/OffsetSyncSerde.java index 7a641be21..d80415c93 100644 --- a/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/OffsetSyncSerde.java +++ b/api/src/main/java/io/kafbat/ui/serdes/builtin/mm2/OffsetSyncSerde.java @@ -10,7 +10,7 @@ @Slf4j public class OffsetSyncSerde extends MirrorMakerSerde implements BuiltInSerde { - public static final String NAME = "OffsetSync"; + public static final String NAME = "mm2-OffsetSync"; public static final Pattern TOPIC_NAME_PATTERN = Pattern.compile("mm2-offset-syncs\\..*\\.internal"); private static final Schema VALUE_SCHEMA;