Skip to content

Commit c499c39

Browse files
dpcollins-googledpcollins-google
authored
feat: add pubsublite.ordering.mode to kafka connector (#228)
* feat: add pubsublite.ordering.mode to kafka connector This is useful for migration cases using the kafka wire protocol. * feat: add pubsublite.ordering.mode to kafka connector This is useful for migration cases using the kafka wire protocol. Also clean up dependency management. * feat: add pubsublite.ordering.mode to kafka connector This is useful for migration cases using the kafka wire protocol. Also clean up dependency management. * feat: add pubsublite.ordering.mode to kafka connector This is useful for migration cases using the kafka wire protocol. Also clean up dependency management. * feat: add pubsublite.ordering.mode to kafka connector This is useful for migration cases using the kafka wire protocol. Also clean up dependency management. * feat: add pubsublite.ordering.mode to kafka connector This is useful for migration cases using the kafka wire protocol. Also clean up dependency management.
1 parent 423141f commit c499c39

File tree

9 files changed

+238
-190
lines changed

9 files changed

+238
-190
lines changed

‎pom.xml

Lines changed: 52 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -15,54 +15,73 @@
1515
<parent>
1616
<groupId>com.google.cloud</groupId>
1717
<artifactId>google-cloud-shared-config</artifactId>
18-
<version>1.5.4</version>
18+
<version>1.5.5</version>
1919
</parent>
2020

2121
<properties>
2222
<maven.compiler.source>1.8</maven.compiler.source>
2323
<maven.compiler.target>1.8</maven.compiler.target>
2424

25-
<kafka.version>3.3.1</kafka.version>
26-
<pubsub.version>1.120.25</pubsub.version>
27-
<pubsublite.version>1.8.0</pubsublite.version>
28-
<cloud-compute.version>1.16.0</cloud-compute.version>
29-
<protobuf-java.vesion>3.21.9</protobuf-java.vesion>
30-
<gax.version>2.19.4</gax.version>
31-
<slf4j.version>2.0.3</slf4j.version>
25+
<kafka.version>3.4.0</kafka.version>
3226
</properties>
3327

28+
<dependencyManagement>
29+
<dependencies>
30+
<dependency>
31+
<groupId>com.google.cloud</groupId>
32+
<artifactId>google-cloud-shared-dependencies</artifactId>
33+
<version>3.3.0</version>
34+
<type>pom</type>
35+
<scope>import</scope>
36+
</dependency>
37+
<dependency>
38+
<groupId>com.google.cloud</groupId>
39+
<artifactId>libraries-bom</artifactId>
40+
<version>26.8.0</version>
41+
<type>pom</type>
42+
<scope>import</scope>
43+
</dependency>
44+
<!--TODO(dpcollins-google): remove this !-->
45+
<dependency>
46+
<groupId>com.google.api.grpc</groupId>
47+
<artifactId>grpc-google-cloud-pubsublite-v1</artifactId>
48+
<version>1.11.1</version>
49+
</dependency>
50+
</dependencies>
51+
</dependencyManagement>
52+
3453
<dependencies>
54+
<dependency>
55+
<groupId>com.google.api.grpc</groupId>
56+
<artifactId>proto-google-cloud-pubsublite-v1</artifactId>
57+
<!--TODO(dpcollins-google): remove explicit version !-->
58+
<version>1.11.1</version>
59+
</dependency>
60+
<dependency>
61+
<groupId>com.google.cloud</groupId>
62+
<artifactId>google-cloud-pubsublite</artifactId>
63+
<!--TODO(dpcollins-google): remove explicit version !-->
64+
<version>1.11.1</version>
65+
</dependency>
66+
<dependency>
67+
<groupId>com.google.cloud</groupId>
68+
<artifactId>pubsublite-kafka</artifactId>
69+
<version>1.1.2</version>
70+
</dependency>
3571
<dependency>
3672
<groupId>com.google.cloud</groupId>
3773
<artifactId>google-cloud-pubsub</artifactId>
38-
<version>${pubsub.version}</version>
3974
</dependency>
4075
<!-- https://mvnrepository.com/artifact/com.google.protobuf/protobuf-java -->
4176
<dependency>
4277
<groupId>com.google.protobuf</groupId>
4378
<artifactId>protobuf-java</artifactId>
44-
<version>${protobuf-java.vesion}</version>
45-
</dependency>
46-
<dependency>
47-
<groupId>com.google.auth</groupId>
48-
<artifactId>google-auth-library-oauth2-http</artifactId>
49-
<version>1.12.1</version>
5079
</dependency>
5180
<dependency>
5281
<groupId>org.apache.kafka</groupId>
5382
<artifactId>kafka-clients</artifactId>
5483
<version>${kafka.version}</version>
5584
</dependency>
56-
<dependency>
57-
<groupId>com.google.cloud</groupId>
58-
<artifactId>pubsublite-kafka</artifactId>
59-
<version>1.0.2</version>
60-
</dependency>
61-
<dependency>
62-
<groupId>com.google.api.grpc</groupId>
63-
<artifactId>proto-google-cloud-pubsublite-v1</artifactId>
64-
<version>${pubsublite.version}</version>
65-
</dependency>
6685
<dependency>
6786
<groupId>com.google.flogger</groupId>
6887
<artifactId>google-extensions</artifactId>
@@ -71,62 +90,51 @@
7190
<dependency>
7291
<groupId>com.google.code.findbugs</groupId>
7392
<artifactId>jsr305</artifactId>
74-
<version>3.0.2</version>
7593
</dependency>
7694
<dependency>
7795
<groupId>com.google.api</groupId>
7896
<artifactId>gax</artifactId>
79-
<version>${gax.version}</version>
8097
</dependency>
8198
<dependency>
8299
<groupId>org.slf4j</groupId>
83100
<artifactId>slf4j-api</artifactId>
84-
<version>${slf4j.version}</version>
101+
<version>2.0.5</version>
85102
</dependency>
86103
<dependency>
87104
<groupId>com.google.api</groupId>
88105
<artifactId>gax-grpc</artifactId>
89-
<version>${gax.version}</version>
90106
</dependency>
91107
<dependency>
92108
<groupId>com.google.auth</groupId>
93109
<artifactId>google-auth-library-credentials</artifactId>
94-
<version>1.13.0</version>
95110
</dependency>
96111
<dependency>
97112
<groupId>com.google.api.grpc</groupId>
98113
<artifactId>proto-google-cloud-pubsub-v1</artifactId>
99-
<version>1.102.25</version>
100114
</dependency>
101115
<dependency>
102116
<groupId>com.google.api</groupId>
103117
<artifactId>api-common</artifactId>
104-
<version>2.2.2</version>
105-
</dependency>
106-
<dependency>
107-
<groupId>com.google.cloud</groupId>
108-
<artifactId>google-cloud-pubsublite</artifactId>
109-
<version>${pubsublite.version}</version>
110118
</dependency>
111119
<dependency>
112120
<groupId>com.google.guava</groupId>
113121
<artifactId>guava</artifactId>
114-
<version>31.1-jre</version>
115122
</dependency>
116123
<dependency>
117124
<groupId>org.threeten</groupId>
118125
<artifactId>threetenbp</artifactId>
119-
<version>1.6.4</version>
120126
</dependency>
121127
<dependency>
122128
<groupId>com.google.errorprone</groupId>
123129
<artifactId>error_prone_annotations</artifactId>
124-
<version>2.16</version>
125130
</dependency>
126131
<dependency>
127132
<groupId>com.google.protobuf</groupId>
128133
<artifactId>protobuf-java-util</artifactId>
129-
<version>${protobuf-java.vesion}</version>
134+
</dependency>
135+
<dependency>
136+
<groupId>com.google.auth</groupId>
137+
<artifactId>google-auth-library-oauth2-http</artifactId>
130138
</dependency>
131139

132140
<!-- Provided dependencies -->
@@ -141,25 +149,24 @@
141149
<dependency>
142150
<groupId>junit</groupId>
143151
<artifactId>junit</artifactId>
144-
<version>4.13.2</version>
145152
<scope>test</scope>
153+
<version>4.13.2</version>
146154
</dependency>
147155
<dependency>
148156
<groupId>org.mockito</groupId>
149157
<artifactId>mockito-core</artifactId>
150-
<version>4.9.0</version>
151158
<scope>test</scope>
159+
<version>4.11.0</version>
152160
</dependency>
153161
<dependency>
154162
<groupId>com.google.truth</groupId>
155163
<artifactId>truth</artifactId>
156-
<version>1.1.3</version>
157164
<scope>test</scope>
165+
<version>1.1.3</version>
158166
</dependency>
159167
<dependency>
160168
<groupId>com.google.cloud</groupId>
161169
<artifactId>google-cloud-core</artifactId>
162-
<version>2.8.28</version>
163170
<scope>test</scope>
164171
</dependency>
165172
<dependency>
@@ -177,13 +184,11 @@
177184
<dependency>
178185
<groupId>com.google.api.grpc</groupId>
179186
<artifactId>proto-google-cloud-compute-v1</artifactId>
180-
<version>${cloud-compute.version}</version>
181187
<scope>test</scope>
182188
</dependency>
183189
<dependency>
184190
<groupId>com.google.cloud</groupId>
185191
<artifactId>google-cloud-compute</artifactId>
186-
<version>${cloud-compute.version}</version>
187192
<scope>test</scope>
188193
</dependency>
189194
<dependency>
@@ -195,13 +200,6 @@
195200
<dependency>
196201
<groupId>com.google.cloud</groupId>
197202
<artifactId>google-cloud-storage</artifactId>
198-
<version>2.15.0</version>
199-
<scope>test</scope>
200-
</dependency>
201-
<dependency>
202-
<groupId>org.slf4j</groupId>
203-
<artifactId>slf4j-log4j12</artifactId>
204-
<version>${slf4j.version}</version>
205203
<scope>test</scope>
206204
</dependency>
207205
</dependencies>

‎src/main/java/com/google/pubsublite/kafka/sink/ConfigDefs.java

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ private ConfigDefs() {}
2525
static final String PROJECT_FLAG = "pubsublite.project";
2626
static final String LOCATION_FLAG = "pubsublite.location";
2727
static final String TOPIC_NAME_FLAG = "pubsublite.topic";
28+
static final String ORDERING_MODE_FLAG = "pubsublite.ordering.mode";
2829

2930
static ConfigDef config() {
3031
return new ConfigDef()
@@ -42,6 +43,12 @@ static ConfigDef config() {
4243
TOPIC_NAME_FLAG,
4344
ConfigDef.Type.STRING,
4445
Importance.HIGH,
45-
"The name of the topic to which to publish.");
46+
"The name of the topic to which to publish.")
47+
.define(
48+
ORDERING_MODE_FLAG,
49+
ConfigDef.Type.STRING,
50+
OrderingMode.DEFAULT.name(),
51+
Importance.HIGH,
52+
"The ordering mode to use for publishing to Pub/Sub Lite. If set to `KAFKA`, messages will be republished to the same partition index they were read from on the source topic. Note that this means the Pub/Sub Lite topic *must* have the same number of partitions as the source Kafka topic.");
4653
}
4754
}

‎src/main/java/com/google/pubsublite/kafka/sink/Constants.java

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,4 @@ private Constants() {}
2424
public static final String KAFKA_OFFSET_HEADER = "x-goog-pubsublite-source-kafka-offset";
2525
public static final String KAFKA_EVENT_TIME_TYPE_HEADER =
2626
"x-goog-pubsublite-source-kafka-event-time-type";
27-
public static final String PUBSUBLITE_KAFKA_SINK_CONNECTOR_NAME =
28-
"JAVA_PUBSUBLITE_KAFKA_SINK_CONNECTOR";
2927
}

‎src/main/java/com/google/pubsublite/kafka/sink/KafkaPartitionRoutingPolicy.java

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
/*
2+
* Copyright 2023 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package com.google.pubsublite.kafka.sink;
17+
18+
import static com.google.cloud.pubsublite.internal.ExtractStatus.toCanonical;
19+
20+
import com.google.api.gax.rpc.StatusCode.Code;
21+
import com.google.cloud.pubsublite.Partition;
22+
import com.google.cloud.pubsublite.internal.CheckedApiException;
23+
import com.google.cloud.pubsublite.internal.RoutingPolicy;
24+
import com.google.cloud.pubsublite.proto.PubSubMessage;
25+
26+
/** A routing policy that extracts the original kafka partition and routes to that partition. */
27+
class KafkaPartitionRoutingPolicy implements RoutingPolicy {
28+
private final long numPartitions;
29+
30+
KafkaPartitionRoutingPolicy(long numPartitions) {
31+
this.numPartitions = numPartitions;
32+
}
33+
34+
@Override
35+
public Partition route(PubSubMessage message) throws CheckedApiException {
36+
Partition partition = getPartition(message);
37+
if (partition.value() >= numPartitions) {
38+
throw new CheckedApiException(
39+
"Kafka topic has more partitions than Pub/Sub Lite topic. OrderingMode.KAFKA cannot be used.",
40+
Code.FAILED_PRECONDITION);
41+
}
42+
return partition;
43+
}
44+
45+
private Partition getPartition(PubSubMessage message) throws CheckedApiException {
46+
try {
47+
return Partition.of(
48+
Long.parseLong(
49+
message
50+
.getAttributesOrThrow(Constants.KAFKA_PARTITION_HEADER)
51+
.getValues(0)
52+
.toStringUtf8()));
53+
} catch (Throwable t) {
54+
throw toCanonical(t);
55+
}
56+
}
57+
}

‎src/main/java/com/google/pubsublite/kafka/sink/OrderingMode.java

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
/*
2+
* Copyright 2023 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package com.google.pubsublite.kafka.sink;
17+
18+
public enum OrderingMode {
19+
/* Order based on the standard Pub/Sub Lite logic. */
20+
DEFAULT,
21+
/* Send messages to the same partition index they were from in Kafka. */
22+
KAFKA
23+
}

‎src/main/java/com/google/pubsublite/kafka/sink/PubSubLiteSinkTask.java

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,10 @@
1818
import static com.google.pubsublite.kafka.sink.Schemas.encodeToBytes;
1919

2020
import com.google.api.core.ApiService.State;
21-
import com.google.cloud.pubsublite.Message;
2221
import com.google.cloud.pubsublite.MessageMetadata;
2322
import com.google.cloud.pubsublite.internal.Publisher;
23+
import com.google.cloud.pubsublite.proto.AttributeValues;
24+
import com.google.cloud.pubsublite.proto.PubSubMessage;
2425
import com.google.common.annotations.VisibleForTesting;
2526
import com.google.common.collect.ImmutableListMultimap;
2627
import com.google.protobuf.ByteString;
@@ -75,7 +76,7 @@ public void put(Collection<SinkRecord> collection) {
7576
}
7677
}
7778
for (SinkRecord record : collection) {
78-
Message.Builder message = Message.builder();
79+
PubSubMessage.Builder message = PubSubMessage.newBuilder();
7980
if (record.key() != null) {
8081
message.setKey(encodeToBytes(record.keySchema(), record.key()));
8182
}
@@ -89,6 +90,7 @@ public void put(Collection<SinkRecord> collection) {
8990
header ->
9091
attributes.put(
9192
header.key(), Schemas.encodeToBytes(header.schema(), header.value())));
93+
9294
if (record.topic() != null) {
9395
attributes.put(Constants.KAFKA_TOPIC_HEADER, ByteString.copyFromUtf8(record.topic()));
9496
}
@@ -106,7 +108,13 @@ public void put(Collection<SinkRecord> collection) {
106108
ByteString.copyFromUtf8(record.timestampType().name));
107109
message.setEventTime(Timestamps.fromMillis(record.timestamp()));
108110
}
109-
message.setAttributes(attributes.build());
111+
attributes
112+
.build()
113+
.asMap()
114+
.forEach(
115+
(key, values) ->
116+
message.putAttributes(
117+
key, AttributeValues.newBuilder().addAllValues(values).build()));
110118
publisher.publish(message.build());
111119
}
112120
}

0 commit comments

Comments
 (0)