Skip to content

Commit af4ee6e

Browse files
kinoxyz1kinoxyz1
authored
[feature-#1690][connection][arctic] added arctic connector (#1696)
1 parent b230914 commit af4ee6e

File tree

10 files changed

+964
-0
lines changed

10 files changed

+964
-0
lines changed

‎chunjun-connectors/chunjun-connector-arctic/pom.xml

Lines changed: 162 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
~ Licensed to the Apache Software Foundation (ASF) under one
4+
~ or more contributor license agreements. See the NOTICE file
5+
~ distributed with this work for additional information
6+
~ regarding copyright ownership. The ASF licenses this file
7+
~ to you under the Apache License, Version 2.0 (the
8+
~ "License"); you may not use this file except in compliance
9+
~ with the License. You may obtain a copy of the License at
10+
~
11+
~ http://www.apache.org/licenses/LICENSE-2.0
12+
~
13+
~ Unless required by applicable law or agreed to in writing,
14+
~ software distributed under the License is distributed on an
15+
~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
~ KIND, either express or implied. See the License for the
17+
~ specific language governing permissions and limitations
18+
~ under the License.
19+
-->
20+
21+
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
22+
xmlns="http://maven.apache.org/POM/4.0.0"
23+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
24+
<parent>
25+
<artifactId>chunjun-connectors</artifactId>
26+
<groupId>com.dtstack.chunjun</groupId>
27+
<version>${revision}</version>
28+
</parent>
29+
<modelVersion>4.0.0</modelVersion>
30+
31+
<artifactId>chunjun-connector-arctic</artifactId>
32+
<name>ChunJun : Connector : Arctic</name>
33+
34+
<properties>
35+
<hadoop.version>2.7.5</hadoop.version>
36+
<iceberg.version>0.13.2</iceberg.version>
37+
<arctic.version>0.4.0</arctic.version>
38+
<connector.dir>arctic</connector.dir>
39+
</properties>
40+
41+
<dependencies>
42+
43+
<dependency>
44+
<groupId>commons-cli</groupId>
45+
<artifactId>commons-cli</artifactId>
46+
<version>1.3.1</version>
47+
</dependency>
48+
<dependency>
49+
<groupId>org.apache.hadoop</groupId>
50+
<artifactId>hadoop-auth</artifactId>
51+
<version>${hadoop3.version}</version>
52+
</dependency>
53+
54+
<dependency>
55+
<groupId>org.apache.avro</groupId>
56+
<artifactId>avro</artifactId>
57+
<version>1.10.0</version>
58+
</dependency>
59+
60+
<dependency>
61+
<groupId>org.apache.hadoop</groupId>
62+
<artifactId>hadoop-hdfs</artifactId>
63+
<version>${hadoop.version}</version>
64+
<exclusions>
65+
<exclusion>
66+
<artifactId>commons-cli</artifactId>
67+
<groupId>commons-cli</groupId>
68+
</exclusion>
69+
</exclusions>
70+
</dependency>
71+
72+
<dependency>
73+
<groupId>org.apache.hadoop</groupId>
74+
<artifactId>hadoop-common</artifactId>
75+
<version>${hadoop.version}</version>
76+
<exclusions>
77+
<exclusion>
78+
<artifactId>commons-cli</artifactId>
79+
<groupId>commons-cli</groupId>
80+
</exclusion>
81+
<exclusion>
82+
<groupId>org.apache.hadoop</groupId>
83+
<artifactId>hadoop-auth</artifactId>
84+
</exclusion>
85+
<exclusion>
86+
<groupId>org.apache.commons</groupId>
87+
<artifactId>commons-math3</artifactId>
88+
</exclusion>
89+
</exclusions>
90+
</dependency>
91+
92+
<dependency>
93+
<groupId>org.apache.hadoop</groupId>
94+
<artifactId>hadoop-mapreduce-client-core</artifactId>
95+
<version>${hadoop2.version}</version>
96+
</dependency>
97+
98+
<dependency>
99+
<groupId>com.netease.arctic</groupId>
100+
<artifactId>arctic-flink-1.12</artifactId>
101+
<version>${arctic.version}</version>
102+
</dependency>
103+
</dependencies>
104+
105+
<build>
106+
<plugins>
107+
<plugin>
108+
<groupId>org.apache.maven.plugins</groupId>
109+
<artifactId>maven-shade-plugin</artifactId>
110+
<version>3.1.0</version>
111+
<executions>
112+
<execution>
113+
<phase>package</phase>
114+
<goals>
115+
<goal>shade</goal>
116+
</goals>
117+
<configuration>
118+
<createDependencyReducedPom>false</createDependencyReducedPom>
119+
<artifactSet>
120+
<excludes>
121+
<!-- <exclude>commons-cli:commons-cli</exclude>-->
122+
<!-- <exclude>org.apache.avro:avro</exclude>-->
123+
<!-- <exclude>org.apache.hadoop:hadoop-mapreduce-client-core</exclude>-->
124+
<exclude>org.slf4j:slf4j-api</exclude>
125+
<exclude>log4j:log4j</exclude>
126+
<exclude>ch.qos.logback:*</exclude>
127+
</excludes>
128+
</artifactSet>
129+
<filters>
130+
<filter>
131+
<artifact>org.apache.hadoop:hadoop-mapreduce-client-core</artifact>
132+
<includes>
133+
<include>**</include>
134+
</includes>
135+
</filter>
136+
<filter>
137+
<artifact>*:*</artifact>
138+
<excludes>
139+
<exclude>META-INF/*.SF</exclude>
140+
<exclude>META-INF/*.DSA</exclude>
141+
<exclude>META-INF/*.RSA</exclude>
142+
</excludes>
143+
</filter>
144+
</filters>
145+
<relocations>
146+
<relocation>
147+
<pattern>org.apache.avro</pattern>
148+
<shadedPattern>org.apache.flink.avro.shaded.org.apache.avro</shadedPattern>
149+
</relocation>
150+
</relocations>
151+
</configuration>
152+
</execution>
153+
</executions>
154+
</plugin>
155+
<plugin>
156+
<groupId>org.apache.maven.plugins</groupId>
157+
<artifactId>maven-antrun-plugin</artifactId>
158+
</plugin>
159+
</plugins>
160+
</build>
161+
162+
</project>

‎chunjun-connectors/chunjun-connector-arctic/src/main/java/com/dtstack/chunjun/connector/arctic/conf/ArcticReaderConf.java

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
package com.dtstack.chunjun.connector.arctic.conf;
19+
20+
import com.dtstack.chunjun.config.CommonConfig;
21+
22+
import java.util.HashMap;
23+
import java.util.Map;
24+
25+
public class ArcticReaderConf extends CommonConfig {
26+
private String path;
27+
28+
private String defaultFS;
29+
private String fileType;
30+
private Map<String, Object> hadoopConfig = new HashMap<>(16);
31+
32+
public String getDefaultFS() {
33+
return defaultFS;
34+
}
35+
36+
public void setDefaultFS(String defaultFS) {
37+
this.defaultFS = defaultFS;
38+
}
39+
40+
public String getFileType() {
41+
return fileType;
42+
}
43+
44+
public void setFileType(String fileType) {
45+
this.fileType = fileType;
46+
}
47+
48+
public Map<String, Object> getHadoopConfig() {
49+
return hadoopConfig;
50+
}
51+
52+
public void setHadoopConfig(Map<String, Object> hadoopConfig) {
53+
this.hadoopConfig = hadoopConfig;
54+
}
55+
56+
public String getPath() {
57+
return path;
58+
}
59+
60+
public void setPath(String path) {
61+
this.path = path;
62+
}
63+
64+
@Override
65+
public String toString() {
66+
return "IcebergReaderConf{" + "path='" + path + '\'' + '}';
67+
}
68+
}

‎chunjun-connectors/chunjun-connector-arctic/src/main/java/com/dtstack/chunjun/connector/arctic/conf/ArcticWriterConf.java

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package com.dtstack.chunjun.connector.arctic.conf;
20+
21+
import com.dtstack.chunjun.config.CommonConfig;
22+
23+
public class ArcticWriterConf extends CommonConfig {
24+
private String amsUrl;
25+
private String tableMode;
26+
private boolean isOverwrite;
27+
private String databaseName;
28+
private String tableName;
29+
public static final String KEYED_TABLE_MODE = "KEYED";
30+
public static final String UNKEYED_TABLE_MODE = "UNKEYED";
31+
32+
public String getAmsUrl() {
33+
return amsUrl;
34+
}
35+
36+
public void setAmsUrl(String amsUrl) {
37+
this.amsUrl = amsUrl;
38+
}
39+
40+
public String getTableMode() {
41+
return tableMode;
42+
}
43+
44+
public void setTableMode(String tableMode) {
45+
this.tableMode = tableMode;
46+
}
47+
48+
public boolean isOverwrite() {
49+
return isOverwrite;
50+
}
51+
52+
public void setOverwrite(boolean overwrite) {
53+
isOverwrite = overwrite;
54+
}
55+
56+
public String getDatabaseName() {
57+
return databaseName;
58+
}
59+
60+
public void setDatabaseName(String databaseName) {
61+
this.databaseName = databaseName;
62+
}
63+
64+
public String getTableName() {
65+
return tableName;
66+
}
67+
68+
public void setTableName(String tableName) {
69+
this.tableName = tableName;
70+
}
71+
72+
@Override
73+
public String toString() {
74+
return "ArcticWriterConf{" + "amsUrl='" + amsUrl + '\'' + '}';
75+
}
76+
}

0 commit comments

Comments
 (0)