From 7903db740a6893aedd27dedfdb552c1b28766f22 Mon Sep 17 00:00:00 2001 From: Jarvis Date: Fri, 1 Sep 2023 16:10:15 +0800 Subject: [PATCH 01/15] [Fix] fix jindo oss connector name (#5385) --- config/plugin_config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/plugin_config b/config/plugin_config index 95b952b31bf..0c7e119ecfc 100644 --- a/config/plugin_config +++ b/config/plugin_config @@ -36,7 +36,7 @@ connector-file-ftp connector-file-hadoop connector-file-local connector-file-oss -connector-file-oss-jindo +connector-file-jindo-oss connector-file-s3 connector-file-sftp connector-google-sheets From d8c92a118231d6796d4ba5be15122775b1f6fbbe Mon Sep 17 00:00:00 2001 From: Guangdong Liu <804167098@qq.com> Date: Fri, 1 Sep 2023 16:14:47 +0800 Subject: [PATCH 02/15] [Bugfix]Fix the problem of "," being divided in [] (#5401) * Fix the problem of "," being divided in [] * Update seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/command/ParameterSplitter.java Co-authored-by: Wenjun Ruan * fix error * fix bug * fix bug --------- Co-authored-by: Wenjun Ruan --- .../config/impl/PropertiesParser.java | 58 +++++++++++-------- .../starter/command/AbstractCommandArgs.java | 1 + .../starter/command/ParameterSplitter.java | 58 +++++++++++++++++++ .../seatunnel/args/ClientCommandArgsTest.java | 8 ++- .../resources/args/user_defined_params.conf | 3 +- 5 files changed, 102 insertions(+), 26 deletions(-) create mode 100644 seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/command/ParameterSplitter.java diff --git a/seatunnel-config/seatunnel-config-shade/src/main/java/org/apache/seatunnel/shade/com/typesafe/config/impl/PropertiesParser.java b/seatunnel-config/seatunnel-config-shade/src/main/java/org/apache/seatunnel/shade/com/typesafe/config/impl/PropertiesParser.java index eceacf99797..3cfdb7dba3b 100644 --- a/seatunnel-config/seatunnel-config-shade/src/main/java/org/apache/seatunnel/shade/com/typesafe/config/impl/PropertiesParser.java +++ b/seatunnel-config/seatunnel-config-shade/src/main/java/org/apache/seatunnel/shade/com/typesafe/config/impl/PropertiesParser.java @@ -7,8 +7,7 @@ import java.io.IOException; import java.io.Reader; import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; +import java.util.Arrays; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; @@ -58,7 +57,15 @@ private static AbstractConfigObject fromEntrySet( } private static Map getPathMap(Set> entries) { - Map pathMap = new LinkedHashMap(); + Map pathMap = new LinkedHashMap<>(); + System.getProperties() + .forEach( + (key, value) -> { + if (key instanceof String) { + Path path = pathFromPropertyKey((String) key); + pathMap.put(path, value); + } + }); for (Map.Entry entry : entries) { Object key = entry.getKey(); if (key instanceof String) { @@ -74,7 +81,7 @@ static AbstractConfigObject fromStringMap(ConfigOrigin origin, Map pathExpressionMap) { - Map pathMap = new LinkedHashMap(); + Map pathMap = new LinkedHashMap<>(); for (Map.Entry entry : pathExpressionMap.entrySet()) { Object keyObj = entry.getKey(); if (!(keyObj instanceof String)) { @@ -93,8 +100,8 @@ private static AbstractConfigObject fromPathMap( * First, build a list of paths that will have values, either string or * object values. */ - Set scopePaths = new LinkedHashSet(); - Set valuePaths = new LinkedHashSet(); + Set scopePaths = new LinkedHashSet<>(); + Set valuePaths = new LinkedHashSet<>(); for (Path path : pathMap.keySet()) { // add value's path valuePaths.add(path); @@ -129,13 +136,11 @@ private static AbstractConfigObject fromPathMap( /* * Create maps for the object-valued values. */ - Map root = new LinkedHashMap(); - Map> scopes = - new LinkedHashMap>(); + Map root = new LinkedHashMap<>(); + Map> scopes = new LinkedHashMap<>(); for (Path path : scopePaths) { - Map scope = - new LinkedHashMap(); + Map scope = new LinkedHashMap<>(); scopes.put(path, scope); } @@ -150,7 +155,17 @@ private static AbstractConfigObject fromPathMap( AbstractConfigValue value; if (convertedFromProperties) { if (rawValue instanceof String) { - value = new ConfigString.Quoted(origin, (String) rawValue); + if (((String) rawValue).startsWith("[") && ((String) rawValue).endsWith("]")) { + List list = + Arrays.asList( + ((String) rawValue) + .substring(1, ((String) rawValue).length() - 1) + .split(",")); + value = ConfigImpl.fromAnyRef(list, origin, FromMapMode.KEYS_ARE_PATHS); + } else { + value = new ConfigString.Quoted(origin, (String) rawValue); + } + } else { // silently ignore non-string values in Properties value = null; @@ -167,19 +182,14 @@ private static AbstractConfigObject fromPathMap( * Make a list of scope paths from longest to shortest, so children go * before parents. */ - List sortedScopePaths = new ArrayList(); - sortedScopePaths.addAll(scopePaths); + List sortedScopePaths = new ArrayList<>(scopePaths); // sort descending by length - Collections.sort( - sortedScopePaths, - new Comparator() { - @Override - public int compare(Path a, Path b) { - // Path.length() is O(n) so in theory this sucks - // but in practice we can make Path precompute length - // if it ever matters. - return b.length() - a.length(); - } + sortedScopePaths.sort( + (a, b) -> { + // Path.length() is O(n) so in theory this sucks + // but in practice we can make Path precompute length + // if it ever matters. + return b.length() - a.length(); }); /* diff --git a/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/command/AbstractCommandArgs.java b/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/command/AbstractCommandArgs.java index 9b818ca95f6..ada15490f0a 100644 --- a/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/command/AbstractCommandArgs.java +++ b/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/command/AbstractCommandArgs.java @@ -41,6 +41,7 @@ public abstract class AbstractCommandArgs extends CommandArgs { /** user-defined parameters */ @Parameter( names = {"-i", "--variable"}, + splitter = ParameterSplitter.class, description = "Variable substitution, such as -i city=beijing, or -i date=20190318") protected List variables = Collections.emptyList(); diff --git a/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/command/ParameterSplitter.java b/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/command/ParameterSplitter.java new file mode 100644 index 00000000000..29263d417e7 --- /dev/null +++ b/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/command/ParameterSplitter.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.seatunnel.core.starter.command; + +import com.beust.jcommander.converters.IParameterSplitter; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class ParameterSplitter implements IParameterSplitter { + + @Override + public List split(String value) { + if (!value.contains(",")) { + return Collections.singletonList(value); + } + + List result = new ArrayList<>(); + StringBuilder currentToken = new StringBuilder(); + boolean insideBrackets = false; + + for (char c : value.toCharArray()) { + if (c == '[') { + insideBrackets = true; + } else if (c == ']') { + insideBrackets = false; + } + + if (c == ',' && !insideBrackets) { + result.add(currentToken.toString().trim()); + currentToken = new StringBuilder(); + } else { + currentToken.append(c); + } + } + + if (currentToken.length() > 0) { + result.add(currentToken.toString().trim()); + } + + return result; + } +} diff --git a/seatunnel-core/seatunnel-starter/src/test/java/org/apache/seatunnel/core/starter/seatunnel/args/ClientCommandArgsTest.java b/seatunnel-core/seatunnel-starter/src/test/java/org/apache/seatunnel/core/starter/seatunnel/args/ClientCommandArgsTest.java index 5f197367d0d..c4bd422f2f1 100644 --- a/seatunnel-core/seatunnel-starter/src/test/java/org/apache/seatunnel/core/starter/seatunnel/args/ClientCommandArgsTest.java +++ b/seatunnel-core/seatunnel-starter/src/test/java/org/apache/seatunnel/core/starter/seatunnel/args/ClientCommandArgsTest.java @@ -40,6 +40,7 @@ public void testUserDefinedParamsCommand() throws URISyntaxException { String password = "dsjr42=4wfskahdsd=w1chh"; String fakeSourceTable = "fake"; String fakeSinkTable = "sink"; + String list = "[par1=20230829,par2=20230829]"; String[] args = { "-c", "/args/user_defined_params.conf", @@ -54,7 +55,9 @@ public void testUserDefinedParamsCommand() throws URISyntaxException { "-i", "password=" + password, "-i", - "username=" + username + "username=" + username, + "-i", + "list=" + list, }; ClientCommandArgs clientCommandArgs = CommandLineUtils.parse(args, new ClientCommandArgs(), "seatunnel-zeta", true); @@ -88,6 +91,9 @@ public void testUserDefinedParamsCommand() throws URISyntaxException { Assertions.assertEquals(sinkConfig.getString("username"), username); Assertions.assertEquals(sinkConfig.getString("password"), password); + List list1 = sinkConfig.getStringList("list"); + Assertions.assertEquals(list1.get(0), "par1=20230829"); + Assertions.assertEquals(list1.get(1), "par2=20230829"); } } } diff --git a/seatunnel-core/seatunnel-starter/src/test/resources/args/user_defined_params.conf b/seatunnel-core/seatunnel-starter/src/test/resources/args/user_defined_params.conf index 9dfde35dd6a..bc2114443f1 100644 --- a/seatunnel-core/seatunnel-starter/src/test/resources/args/user_defined_params.conf +++ b/seatunnel-core/seatunnel-starter/src/test/resources/args/user_defined_params.conf @@ -47,5 +47,6 @@ sink { result_table_name = ${fake_sink_table} username = ${username} password = ${password} + list = ${list} } -} \ No newline at end of file +} From 1d7f44516c3b86f45b9de9d434e21924177b4024 Mon Sep 17 00:00:00 2001 From: Chengyu Yan Date: Tue, 5 Sep 2023 10:15:24 +0800 Subject: [PATCH 03/15] [Docs][HotFix] Replace username by user in the options description of FtpFile (#5421) --- docs/en/connector-v2/source/FtpFile.md | 4 ++-- release-note.md | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/en/connector-v2/source/FtpFile.md b/docs/en/connector-v2/source/FtpFile.md index c692a7483a6..c9fb8e70cdb 100644 --- a/docs/en/connector-v2/source/FtpFile.md +++ b/docs/en/connector-v2/source/FtpFile.md @@ -58,9 +58,9 @@ The target ftp host is required The target ftp port is required -### username [string] +### user [string] -The target ftp username is required +The target ftp user name is required ### password [string] diff --git a/release-note.md b/release-note.md index eb1624b158c..1b797ff3154 100644 --- a/release-note.md +++ b/release-note.md @@ -195,3 +195,4 @@ - [Docs] Redshift add defaultRowFetchSize (#4616) - [Docs] Refactor connector-v2 docs using unified format Mysql (#4590) - [Docs] Add Value types in Java to Schema features (#5087) +- [Docs] Replace username by user in the options of FtpFile (#5421) \ No newline at end of file From 856aedb3c99e1e723003900914b6e9a60827c6e9 Mon Sep 17 00:00:00 2001 From: Jarvis Date: Tue, 5 Sep 2023 12:26:10 +0800 Subject: [PATCH 04/15] [Doc] update iotdb document (#5404) * [Doc] update iotdb document, remove duplicated config * [Doc] update iotdb document * [Doc] update iotdb document --- docs/en/connector-v2/sink/IoTDB.md | 118 +++++++--- docs/en/connector-v2/source/IoTDB.md | 219 +++++++----------- .../seatunnel/iotdb/config/SourceConfig.java | 16 +- .../seatunnel/iotdb/source/IoTDBSource.java | 8 +- .../iotdb/source/IoTDBSourceFactory.java | 4 - .../iotdb/source/IoTDBSourceReader.java | 17 +- .../e2e/connector/iotdb/IoTDBIT.java | 2 +- 7 files changed, 173 insertions(+), 211 deletions(-) diff --git a/docs/en/connector-v2/sink/IoTDB.md b/docs/en/connector-v2/sink/IoTDB.md index e7f5bc277a6..554d0bfd06e 100644 --- a/docs/en/connector-v2/sink/IoTDB.md +++ b/docs/en/connector-v2/sink/IoTDB.md @@ -50,10 +50,10 @@ There is a conflict of thrift version between IoTDB and Spark.Therefore, you nee | Name | Type | Required | Default | Description | |-----------------------------|---------|----------|--------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| node_urls | Array | Yes | - | `IoTDB` cluster address, the format is `["host:port", ...]` | +| node_urls | String | Yes | - | `IoTDB` cluster address, the format is `"host1:port"` or `"host1:port,host2:port"` | | username | String | Yes | - | `IoTDB` user username | | password | String | Yes | - | `IoTDB` user password | -| key_device | String | No | - | Specify field name of the `IoTDB` deviceId in SeaTunnelRow | +| key_device | String | Yes | - | Specify field name of the `IoTDB` deviceId in SeaTunnelRow | | key_timestamp | String | No | processing time | Specify field-name of the `IoTDB` timestamp in SeaTunnelRow. If not specified, use processing-time as timestamp | | key_measurement_fields | Array | No | exclude `device` & `timestamp` | Specify field-name of the `IoTDB` measurement list in SeaTunnelRow. If not specified, include all fields but exclude `device` & `timestamp` | | storage_group | Array | No | - | Specify device storage group(path prefix)
example: deviceId = ${storage_group} + "." + ${key_device} | @@ -68,78 +68,124 @@ There is a conflict of thrift version between IoTDB and Spark.Therefore, you nee | connection_timeout_in_ms | Integer | No | - | The maximum time (in ms) to wait when connecting to `IoTDB` | | common-options | | no | - | Sink plugin common parameters, please refer to [Sink Common Options](common-options.md) for details | -## Task Example +## Examples + +```hocon +env { + execution.parallelism = 2 + job.mode = "BATCH" +} + +source { + FakeSource { + row.num = 16 + bigint.template = [1664035200001] + schema = { + fields { + device_name = "string" + temperature = "float" + moisture = "int" + event_ts = "bigint" + c_string = "string" + c_boolean = "boolean" + c_tinyint = "tinyint" + c_smallint = "smallint" + c_int = "int" + c_bigint = "bigint" + c_float = "float" + c_double = "double" + } + } + } +} + +... + +``` + +Upstream SeaTunnelRow data format is the following: + +| device_name | temperature | moisture | event_ts | c_string | c_boolean | c_tinyint | c_smallint | c_int | c_bigint | c_float | c_double | +|--------------------------|-------------|----------|---------------|----------|-----------|-----------|------------|-------|------------|---------|----------| +| root.test_group.device_a | 36.1 | 100 | 1664035200001 | abc1 | true | 1 | 1 | 1 | 2147483648 | 1.0 | 1.0 | +| root.test_group.device_b | 36.2 | 101 | 1664035200001 | abc2 | false | 2 | 2 | 2 | 2147483649 | 2.0 | 2.0 | +| root.test_group.device_c | 36.3 | 102 | 1664035200001 | abc3 | false | 3 | 3 | 3 | 2147483649 | 3.0 | 3.0 | ### Case1 -Common options: +only fill required config. +use current processing time as timestamp. and include all fields but exclude `device` & `timestamp` as measurement fields ```hocon sink { IoTDB { - node_urls = ["localhost:6667"] + node_urls = "localhost:6667" username = "root" password = "root" - batch_size = 1024 + key_device = "device_name" # specify the `deviceId` use device_name field } } ``` -When you assign `key_device` is `device_name`, for example: +Output to `IoTDB` data format is the following: + +```shell +IoTDB> SELECT * FROM root.test_group.* align by device; ++------------------------+------------------------+--------------+-----------+--------------+---------+----------+----------+-----------+------+-----------+--------+---------+ +| Time| Device| temperature| moisture| event_ts| c_string| c_boolean| c_tinyint| c_smallint| c_int| c_bigint| c_float| c_double| ++------------------------+------------------------+--------------+-----------+--------------+---------+----------+----------+-----------+------+-----------+--------+---------+ +|2023-09-01T00:00:00.001Z|root.test_group.device_a| 36.1| 100| 1664035200001| abc1| true| 1| 1| 1| 2147483648| 1.0| 1.0| +|2023-09-01T00:00:00.001Z|root.test_group.device_b| 36.2| 101| 1664035200001| abc2| false| 2| 2| 2| 2147483649| 2.0| 2.0| +|2023-09-01T00:00:00.001Z|root.test_group.device_c| 36.3| 102| 1664035200001| abc2| false| 3| 3| 3| 2147483649| 3.0| 3.0| ++------------------------+------------------------+--------------+-----------+--------------+---------+---------+-----------+-----------+------+-----------+--------+---------+ +``` + +### Case2 + +use source event's time ```hocon sink { IoTDB { - ... - key_device = "device_name" + node_urls = "localhost:6667" + username = "root" + password = "root" + key_device = "device_name" # specify the `deviceId` use device_name field + key_timestamp = "event_ts" # specify the `timestamp` use event_ts field } } ``` -Upstream SeaTunnelRow data format is the following: - -| device_name | field_1 | field_2 | -|--------------------------|---------|---------| -| root.test_group.device_a | 1001 | 1002 | -| root.test_group.device_b | 2001 | 2002 | -| root.test_group.device_c | 3001 | 3002 | - Output to `IoTDB` data format is the following: ```shell IoTDB> SELECT * FROM root.test_group.* align by device; -+------------------------+------------------------+-----------+----------+ -| Time| Device| field_1| field_2| -+------------------------+------------------------+----------+-----------+ -|2022-09-26T17:50:01.201Z|root.test_group.device_a| 1001| 1002| -|2022-09-26T17:50:01.202Z|root.test_group.device_b| 2001| 2002| -|2022-09-26T17:50:01.203Z|root.test_group.device_c| 3001| 3002| -+------------------------+------------------------+----------+-----------+ ++------------------------+------------------------+--------------+-----------+--------------+---------+----------+----------+-----------+------+-----------+--------+---------+ +| Time| Device| temperature| moisture| event_ts| c_string| c_boolean| c_tinyint| c_smallint| c_int| c_bigint| c_float| c_double| ++------------------------+------------------------+--------------+-----------+--------------+---------+----------+----------+-----------+------+-----------+--------+---------+ +|2022-09-25T00:00:00.001Z|root.test_group.device_a| 36.1| 100| 1664035200001| abc1| true| 1| 1| 1| 2147483648| 1.0| 1.0| +|2022-09-25T00:00:00.001Z|root.test_group.device_b| 36.2| 101| 1664035200001| abc2| false| 2| 2| 2| 2147483649| 2.0| 2.0| +|2022-09-25T00:00:00.001Z|root.test_group.device_c| 36.3| 102| 1664035200001| abc2| false| 3| 3| 3| 2147483649| 3.0| 3.0| ++------------------------+------------------------+--------------+-----------+--------------+---------+---------+-----------+-----------+------+-----------+--------+---------+ ``` -### Case2 +### Case3 -When you assign `key_device`、`key_timestamp`、`key_measurement_fields`, for example: +use source event's time and limit measurement fields ```hocon sink { IoTDB { - ... + node_urls = "localhost:6667" + username = "root" + password = "root" key_device = "device_name" - key_timestamp = "ts" + key_timestamp = "event_ts" key_measurement_fields = ["temperature", "moisture"] } } ``` -Upstream SeaTunnelRow data format is the following: - -| ts | device_name | field_1 | field_2 | temperature | moisture | -|---------------|--------------------------|---------|---------|-------------|----------| -| 1664035200001 | root.test_group.device_a | 1001 | 1002 | 36.1 | 100 | -| 1664035200001 | root.test_group.device_b | 2001 | 2002 | 36.2 | 101 | -| 1664035200001 | root.test_group.device_c | 3001 | 3002 | 36.3 | 102 | - Output to `IoTDB` data format is the following: ```shell diff --git a/docs/en/connector-v2/source/IoTDB.md b/docs/en/connector-v2/source/IoTDB.md index a20680ce638..da0f198d3e1 100644 --- a/docs/en/connector-v2/source/IoTDB.md +++ b/docs/en/connector-v2/source/IoTDB.md @@ -2,14 +2,16 @@ > IoTDB source connector -## Description +## Support Those Engines -Read external data source data through IoTDB. +> Spark
+> Flink
+> SeaTunnel Zeta
## Key features - [x] [batch](../../concept/connector-v2-features.md) -- [ ] [stream](../../concept/connector-v2-features.md) +- [x] [stream](../../concept/connector-v2-features.md) - [x] [exactly-once](../../concept/connector-v2-features.md) - [x] [column projection](../../concept/connector-v2-features.md) @@ -18,106 +20,53 @@ supports query SQL and can achieve projection effect. - [x] [parallelism](../../concept/connector-v2-features.md) - [ ] [support user-defined split](../../concept/connector-v2-features.md) -## Options - -| name | type | required | default value | -|----------------------------|---------|----------|---------------| -| host | string | no | - | -| port | int | no | - | -| node_urls | string | no | - | -| username | string | yes | - | -| password | string | yes | - | -| sql | string | yes | - | -| schema | config | yes | - | -| fetch_size | int | no | - | -| lower_bound | long | no | - | -| upper_bound | long | no | - | -| num_partitions | int | no | - | -| thrift_default_buffer_size | int | no | - | -| enable_cache_leader | boolean | no | - | -| version | string | no | - | -| common-options | | no | - | - -### single node, you need to set host and port to connect to the remote data source. - -**host** [string] the host of the IoTDB when you select host of the IoTDB - -**port** [int] the port of the IoTDB when you select - -### multi node, you need to set node_urls to connect to the remote data source. - -**node_urls** [string] the node_urls of the IoTDB when you select - -e.g. - -```text -127.0.0.1:8080,127.0.0.2:8080 -``` - -### other parameters - -**sql** [string] -execute sql statement e.g. - -``` -select name,age from test -``` - -### schema [config] - -#### fields [Config] - -The schema of the IoTDB that you want to generate - -e.g. - -``` -schema { - fields { - name = string - age = int - } - } -``` - -### option parameters - -### fetch_size [int] - -the fetch_size of the IoTDB when you select - -### username [string] - -the username of the IoTDB when you select - -### password [string] - -the password of the IoTDB when you select - -### lower_bound [long] - -the lower_bound of the IoTDB when you select - -### upper_bound [long] - -the upper_bound of the IoTDB when you select - -### num_partitions [int] - -the num_partitions of the IoTDB when you select - -### thrift_default_buffer_size [int] - -the thrift_default_buffer_size of the IoTDB when you select - -### enable_cache_leader [boolean] - -enable_cache_leader of the IoTDB when you select +## Description -### version [string] +Read external data source data through IoTDB. -Version represents the SQL semantic version used by the client, which is used to be compatible with the SQL semantics of -0.12 when upgrading 0.13. The possible values are: V_0_12, V_0_13. +:::tip + +There is a conflict of thrift version between IoTDB and Spark.Therefore, you need to execute `rm -f $SPARK_HOME/jars/libthrift*` and `cp $IOTDB_HOME/lib/libthrift* $SPARK_HOME/jars/` to resolve it. + +::: + +## Supported DataSource Info + +| Datasource | Supported Versions | Url | +|------------|--------------------|----------------| +| IoTDB | `>= 0.13.0` | localhost:6667 | + +## Data Type Mapping + +| IotDB Data type | SeaTunnel Data type | +|-----------------|---------------------| +| BOOLEAN | BOOLEAN | +| INT32 | TINYINT | +| INT32 | SMALLINT | +| INT32 | INT | +| INT64 | BIGINT | +| FLOAT | FLOAT | +| DOUBLE | DOUBLE | +| TEXT | STRING | + +## Source Options + +| Name | Type | Required | Default Value | Description | +|----------------------------|---------|----------|---------------|------------------------------------------------------------------------------------| +| node_urls | string | yes | - | `IoTDB` cluster address, the format is `"host1:port"` or `"host1:port,host2:port"` | +| username | string | yes | - | `IoTDB` user username | +| password | string | yes | - | `IoTDB` user password | +| sql | string | yes | - | execute sql statement | +| schema | config | yes | - | the data schema | +| fetch_size | int | no | - | the fetch_size of the IoTDB when you select | +| lower_bound | long | no | - | the lower_bound of the IoTDB when you select | +| upper_bound | long | no | - | the upper_bound of the IoTDB when you select | +| num_partitions | int | no | - | the num_partitions of the IoTDB when you select | +| thrift_default_buffer_size | int | no | - | the thrift_default_buffer_size of the IoTDB when you select | +| thrift_max_frame_size | int | no | - | the thrift max frame size | +| enable_cache_leader | boolean | no | - | enable_cache_leader of the IoTDB when you select | +| version | string | no | - | SQL semantic version used by the client, The possible values are: V_0_12, V_0_13 | +| common-options | | no | - | | ### split partitions @@ -157,37 +106,37 @@ Source plugin common parameters, please refer to [Source Common Options](common- ## Examples -### Case1 - -Common options: - ```hocon +env { + execution.parallelism = 2 + job.mode = "BATCH" +} + source { IoTDB { node_urls = "localhost:6667" username = "root" password = "root" + sql = "SELECT temperature, moisture, c_int, c_bigint, c_float, c_double, c_string, c_boolean FROM root.test_group.* WHERE time < 4102329600000 align by device" + schema { + fields { + ts = timestamp + device_name = string + temperature = float + moisture = bigint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_string = string + c_boolean = boolean + } + } } } -``` - -When you assign `sql`、`fields`、`partition`, for example: -```hocon sink { - IoTDB { - ... - sql = "SELECT temperature, moisture FROM root.test_group.* WHERE time < 4102329600000 align by device" - lower_bound = 1 - upper_bound = 4102329600000 - num_partitions = 10 - fields { - ts = bigint - device_name = string - - temperature = float - moisture = bigint - } + Console { } } ``` @@ -195,23 +144,23 @@ sink { Upstream `IoTDB` data format is the following: ```shell -IoTDB> SELECT temperature, moisture FROM root.test_group.* WHERE time < 4102329600000 align by device; -+------------------------+------------------------+--------------+-----------+ -| Time| Device| temperature| moisture| -+------------------------+------------------------+--------------+-----------+ -|2022-09-25T00:00:00.001Z|root.test_group.device_a| 36.1| 100| -|2022-09-25T00:00:00.001Z|root.test_group.device_b| 36.2| 101| -|2022-09-25T00:00:00.001Z|root.test_group.device_c| 36.3| 102| -+------------------------+------------------------+--------------+-----------+ +IoTDB> SELECT temperature, moisture, c_int, c_bigint, c_float, c_double, c_string, c_boolean FROM root.test_group.* WHERE time < 4102329600000 align by device; ++------------------------+------------------------+--------------+-----------+--------+--------------+----------+---------+---------+----------+ +| Time| Device| temperature| moisture| c_int| c_bigint| c_float| c_double| c_string| c_boolean| ++------------------------+------------------------+--------------+-----------+--------+--------------+----------+---------+---------+----------+ +|2022-09-25T00:00:00.001Z|root.test_group.device_a| 36.1| 100| 1| 21474836470| 1.0f| 1.0d| abc| true| +|2022-09-25T00:00:00.001Z|root.test_group.device_b| 36.2| 101| 2| 21474836470| 2.0f| 2.0d| abc| true| +|2022-09-25T00:00:00.001Z|root.test_group.device_c| 36.3| 102| 3| 21474836470| 3.0f| 3.0d| abc| true| ++------------------------+------------------------+--------------+-----------+--------+--------------+----------+---------+---------+----------+ ``` Loaded to SeaTunnelRow data format is the following: -| ts | device_name | temperature | moisture | -|---------------|--------------------------|-------------|----------| -| 1664035200001 | root.test_group.device_a | 36.1 | 100 | -| 1664035200001 | root.test_group.device_b | 36.2 | 101 | -| 1664035200001 | root.test_group.device_c | 36.3 | 102 | +| ts | device_name | temperature | moisture | c_int | c_bigint | c_float | c_double | c_string | c_boolean | +|---------------|--------------------------|-------------|----------|-------|-------------|---------|----------|----------|-----------| +| 1664035200001 | root.test_group.device_a | 36.1 | 100 | 1 | 21474836470 | 1.0f | 1.0d | abc | true | +| 1664035200001 | root.test_group.device_b | 36.2 | 101 | 2 | 21474836470 | 2.0f | 2.0d | abc | true | +| 1664035200001 | root.test_group.device_c | 36.3 | 102 | 3 | 21474836470 | 3.0f | 3.0d | abc | true | ## Changelog diff --git a/seatunnel-connectors-v2/connector-iotdb/src/main/java/org/apache/seatunnel/connectors/seatunnel/iotdb/config/SourceConfig.java b/seatunnel-connectors-v2/connector-iotdb/src/main/java/org/apache/seatunnel/connectors/seatunnel/iotdb/config/SourceConfig.java index be96a7d9187..ac515ef37a9 100644 --- a/seatunnel-connectors-v2/connector-iotdb/src/main/java/org/apache/seatunnel/connectors/seatunnel/iotdb/config/SourceConfig.java +++ b/seatunnel-connectors-v2/connector-iotdb/src/main/java/org/apache/seatunnel/connectors/seatunnel/iotdb/config/SourceConfig.java @@ -31,20 +31,6 @@ public class SourceConfig { public static final Option SQL = Options.key("sql").stringType().noDefaultValue().withDescription("sql"); - /*---------------------- single node configurations -------------------------*/ - - /** The host of the IotDB server. */ - public static final Option HOST = - Options.key("host").stringType().noDefaultValue().withDescription("host"); - - /* - * The port of the IotDB server. - */ - public static final Option PORT = - Options.key("port").intType().noDefaultValue().withDescription("port"); - - /*---------------------- multiple node configurations -------------------------*/ - /** Username for the source. */ public static final Option USERNAME = Options.key("username").stringType().noDefaultValue().withDescription("usernam"); @@ -53,7 +39,7 @@ public class SourceConfig { public static final Option PASSWORD = Options.key("password").stringType().noDefaultValue().withDescription("password"); - /** multiple nodes */ + /** node urls */ public static final Option NODE_URLS = Options.key("node_urls").stringType().noDefaultValue().withDescription("node urls"); diff --git a/seatunnel-connectors-v2/connector-iotdb/src/main/java/org/apache/seatunnel/connectors/seatunnel/iotdb/source/IoTDBSource.java b/seatunnel-connectors-v2/connector-iotdb/src/main/java/org/apache/seatunnel/connectors/seatunnel/iotdb/source/IoTDBSource.java index 7f2960ae007..0c171ada4fc 100644 --- a/seatunnel-connectors-v2/connector-iotdb/src/main/java/org/apache/seatunnel/connectors/seatunnel/iotdb/source/IoTDBSource.java +++ b/seatunnel-connectors-v2/connector-iotdb/src/main/java/org/apache/seatunnel/connectors/seatunnel/iotdb/source/IoTDBSource.java @@ -43,9 +43,7 @@ import java.util.HashMap; import java.util.Map; -import static org.apache.seatunnel.connectors.seatunnel.iotdb.config.SourceConfig.HOST; import static org.apache.seatunnel.connectors.seatunnel.iotdb.config.SourceConfig.NODE_URLS; -import static org.apache.seatunnel.connectors.seatunnel.iotdb.config.SourceConfig.PORT; @AutoService(SeaTunnelSource.class) public class IoTDBSource @@ -66,11 +64,7 @@ public String getPluginName() { @Override public void prepare(Config pluginConfig) throws PrepareFailException { - CheckResult urlCheckResult = - CheckConfigUtil.checkAllExists(pluginConfig, HOST.key(), PORT.key()); - if (!urlCheckResult.isSuccess()) { - urlCheckResult = CheckConfigUtil.checkAllExists(pluginConfig, NODE_URLS.key()); - } + CheckResult urlCheckResult = CheckConfigUtil.checkAllExists(pluginConfig, NODE_URLS.key()); CheckResult schemaCheckResult = CheckConfigUtil.checkAllExists(pluginConfig, CatalogTableUtil.SCHEMA.key()); CheckResult mergedConfigCheck = diff --git a/seatunnel-connectors-v2/connector-iotdb/src/main/java/org/apache/seatunnel/connectors/seatunnel/iotdb/source/IoTDBSourceFactory.java b/seatunnel-connectors-v2/connector-iotdb/src/main/java/org/apache/seatunnel/connectors/seatunnel/iotdb/source/IoTDBSourceFactory.java index c697df70129..2c2a521fd84 100644 --- a/seatunnel-connectors-v2/connector-iotdb/src/main/java/org/apache/seatunnel/connectors/seatunnel/iotdb/source/IoTDBSourceFactory.java +++ b/seatunnel-connectors-v2/connector-iotdb/src/main/java/org/apache/seatunnel/connectors/seatunnel/iotdb/source/IoTDBSourceFactory.java @@ -30,10 +30,8 @@ import static org.apache.seatunnel.connectors.seatunnel.iotdb.config.CommonConfig.USERNAME; import static org.apache.seatunnel.connectors.seatunnel.iotdb.config.SourceConfig.ENABLE_CACHE_LEADER; import static org.apache.seatunnel.connectors.seatunnel.iotdb.config.SourceConfig.FETCH_SIZE; -import static org.apache.seatunnel.connectors.seatunnel.iotdb.config.SourceConfig.HOST; import static org.apache.seatunnel.connectors.seatunnel.iotdb.config.SourceConfig.LOWER_BOUND; import static org.apache.seatunnel.connectors.seatunnel.iotdb.config.SourceConfig.NUM_PARTITIONS; -import static org.apache.seatunnel.connectors.seatunnel.iotdb.config.SourceConfig.PORT; import static org.apache.seatunnel.connectors.seatunnel.iotdb.config.SourceConfig.SQL; import static org.apache.seatunnel.connectors.seatunnel.iotdb.config.SourceConfig.THRIFT_DEFAULT_BUFFER_SIZE; import static org.apache.seatunnel.connectors.seatunnel.iotdb.config.SourceConfig.THRIFT_MAX_FRAME_SIZE; @@ -52,8 +50,6 @@ public OptionRule optionRule() { return OptionRule.builder() .required(NODE_URLS, USERNAME, PASSWORD, SQL, SCHEMA) .optional( - HOST, - PORT, FETCH_SIZE, THRIFT_DEFAULT_BUFFER_SIZE, THRIFT_MAX_FRAME_SIZE, diff --git a/seatunnel-connectors-v2/connector-iotdb/src/main/java/org/apache/seatunnel/connectors/seatunnel/iotdb/source/IoTDBSourceReader.java b/seatunnel-connectors-v2/connector-iotdb/src/main/java/org/apache/seatunnel/connectors/seatunnel/iotdb/source/IoTDBSourceReader.java index c4ecd9dc81d..546487825c3 100644 --- a/seatunnel-connectors-v2/connector-iotdb/src/main/java/org/apache/seatunnel/connectors/seatunnel/iotdb/source/IoTDBSourceReader.java +++ b/seatunnel-connectors-v2/connector-iotdb/src/main/java/org/apache/seatunnel/connectors/seatunnel/iotdb/source/IoTDBSourceReader.java @@ -46,10 +46,8 @@ import static org.apache.seatunnel.connectors.seatunnel.iotdb.config.SourceConfig.ENABLE_CACHE_LEADER; import static org.apache.seatunnel.connectors.seatunnel.iotdb.config.SourceConfig.FETCH_SIZE; -import static org.apache.seatunnel.connectors.seatunnel.iotdb.config.SourceConfig.HOST; import static org.apache.seatunnel.connectors.seatunnel.iotdb.config.SourceConfig.NODE_URLS; import static org.apache.seatunnel.connectors.seatunnel.iotdb.config.SourceConfig.PASSWORD; -import static org.apache.seatunnel.connectors.seatunnel.iotdb.config.SourceConfig.PORT; import static org.apache.seatunnel.connectors.seatunnel.iotdb.config.SourceConfig.THRIFT_DEFAULT_BUFFER_SIZE; import static org.apache.seatunnel.connectors.seatunnel.iotdb.config.SourceConfig.THRIFT_MAX_FRAME_SIZE; import static org.apache.seatunnel.connectors.seatunnel.iotdb.config.SourceConfig.USERNAME; @@ -130,17 +128,10 @@ private void read(IoTDBSourceSplit split, Collector output) throws private Session buildSession(Map conf) { Session.Builder sessionBuilder = new Session.Builder(); - if (conf.containsKey(HOST.key())) { - sessionBuilder - .host((String) conf.get(HOST.key())) - .port(Integer.parseInt(conf.get(PORT.key()).toString())) - .build(); - } else { - String nodeUrlsString = (String) conf.get(NODE_URLS.key()); - List nodes = - Stream.of(nodeUrlsString.split(NODES_SPLIT)).collect(Collectors.toList()); - sessionBuilder.nodeUrls(nodes); - } + String nodeUrlsString = (String) conf.get(NODE_URLS.key()); + List nodes = + Stream.of(nodeUrlsString.split(NODES_SPLIT)).collect(Collectors.toList()); + sessionBuilder.nodeUrls(nodes); if (null != conf.get(FETCH_SIZE.key())) { sessionBuilder.fetchSize(Integer.parseInt(conf.get(FETCH_SIZE.key()).toString())); } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iotdb-e2e/src/test/java/org/apache/seatunnel/e2e/connector/iotdb/IoTDBIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iotdb-e2e/src/test/java/org/apache/seatunnel/e2e/connector/iotdb/IoTDBIT.java index 94bfbe917e2..8b8d6acd77b 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iotdb-e2e/src/test/java/org/apache/seatunnel/e2e/connector/iotdb/IoTDBIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iotdb-e2e/src/test/java/org/apache/seatunnel/e2e/connector/iotdb/IoTDBIT.java @@ -63,7 +63,7 @@ value = {}, type = {EngineType.SPARK}, disabledReason = - "There is a conflict of thrift version between IoTDB and Spark.Therefore. Refactor starter module, so disabled in flink") + "There is a conflict of thrift version between IoTDB and Spark.Therefore. Refactor starter module, so disabled in spark") public class IoTDBIT extends TestSuiteBase implements TestResource { private static final String IOTDB_DOCKER_IMAGE = "apache/iotdb:0.13.1-node"; From c109385bb5138c20d78664fbc63ff86c8404b895 Mon Sep 17 00:00:00 2001 From: ZhilinLi Date: Tue, 5 Sep 2023 12:27:02 +0800 Subject: [PATCH 05/15] [Docs][Connector][Source][mysql]Added the mysql Connector Document version title Example pr (#5249) * [Docs][Connector][Source][mysql]Added the mysql Connector Document version title Example pr * [Docs][Connector][Source][mysql]Added the mysql Connector Document version title Example pr #5249 * Update docs/en/connector-v2/sink/Mysql.md * Update docs/en/connector-v2/source/Mysql.md --------- Co-authored-by: Jia Fan --- docs/en/connector-v2/sink/Mysql.md | 4 ++++ docs/en/connector-v2/source/Mysql.md | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/docs/en/connector-v2/sink/Mysql.md b/docs/en/connector-v2/sink/Mysql.md index 55c825ed168..6c01c35ee8c 100644 --- a/docs/en/connector-v2/sink/Mysql.md +++ b/docs/en/connector-v2/sink/Mysql.md @@ -2,6 +2,10 @@ > JDBC Mysql Sink Connector +## Support Mysql Version + +- 5.5/5.6/5.7/8.0 + ## Support Those Engines > Spark
diff --git a/docs/en/connector-v2/source/Mysql.md b/docs/en/connector-v2/source/Mysql.md index 32933f8c9a8..001ef1463da 100644 --- a/docs/en/connector-v2/source/Mysql.md +++ b/docs/en/connector-v2/source/Mysql.md @@ -2,6 +2,10 @@ > JDBC Mysql Source Connector +## Support Mysql Version + +- 5.5/5.6/5.7/8.0 + ## Support Those Engines > Spark
From dde3104f761f9e3f4a298f2ba9a61e2e0dcdbee1 Mon Sep 17 00:00:00 2001 From: He Wang Date: Wed, 6 Sep 2023 11:13:10 +0800 Subject: [PATCH 06/15] [Improve][Connector-v2][Jdbc] Refactor AbstractJdbcCatalog (#5096) * refactor jdbc connector AbstractJdbcCatalog * add testCatalog for pg --- .../jdbc/catalog/AbstractJdbcCatalog.java | 342 +++++++++++++++--- .../jdbc/catalog/mysql/MySqlCatalog.java | 309 ++++------------ .../mysql/MysqlCreateTableSqlBuilder.java | 2 +- .../jdbc/catalog/oracle/OracleCatalog.java | 224 +++--------- .../jdbc/catalog/psql/PostgresCatalog.java | 311 +++------------- .../psql/PostgresCreateTableSqlBuilder.java | 2 +- .../catalog/sqlserver/SqlServerCatalog.java | 295 +++------------ .../catalog/oracle/OracleCatalogTest.java | 19 +- .../catalog/psql/PostgresCatalogTest.java | 4 +- .../sql/MysqlCreateTableSqlBuilderTest.java | 2 +- .../seatunnel/jdbc/AbstractJdbcIT.java | 65 +++- .../connectors/seatunnel/jdbc/JdbcCase.java | 5 + .../seatunnel/jdbc/JdbcMysqlIT.java | 17 + .../seatunnel/jdbc/JdbcOracleIT.java | 47 ++- .../src/test/resources/sql/oracle_init.sql | 22 ++ .../seatunnel/jdbc/JdbcPostgresIT.java | 43 +++ .../seatunnel/jdbc/JdbcSqlServerIT.java | 34 +- 17 files changed, 732 insertions(+), 1011 deletions(-) create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/sql/oracle_init.sql diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/AbstractJdbcCatalog.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/AbstractJdbcCatalog.java index 22752ba5011..ddc327fbc30 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/AbstractJdbcCatalog.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/AbstractJdbcCatalog.java @@ -20,9 +20,12 @@ import org.apache.seatunnel.api.table.catalog.Catalog; import org.apache.seatunnel.api.table.catalog.CatalogTable; +import org.apache.seatunnel.api.table.catalog.Column; import org.apache.seatunnel.api.table.catalog.ConstraintKey; import org.apache.seatunnel.api.table.catalog.PrimaryKey; +import org.apache.seatunnel.api.table.catalog.TableIdentifier; import org.apache.seatunnel.api.table.catalog.TablePath; +import org.apache.seatunnel.api.table.catalog.TableSchema; import org.apache.seatunnel.api.table.catalog.exception.CatalogException; import org.apache.seatunnel.api.table.catalog.exception.DatabaseAlreadyExistException; import org.apache.seatunnel.api.table.catalog.exception.DatabaseNotExistException; @@ -40,14 +43,19 @@ import java.sql.Connection; import java.sql.DatabaseMetaData; import java.sql.DriverManager; +import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; +import java.util.Collections; import java.util.Comparator; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; import static org.apache.seatunnel.shade.com.google.common.base.Preconditions.checkArgument; @@ -56,6 +64,8 @@ public abstract class AbstractJdbcCatalog implements Catalog { private static final Logger LOG = LoggerFactory.getLogger(AbstractJdbcCatalog.class); + protected static final Set SYS_DATABASES = new HashSet<>(); + protected final String catalogName; protected final String defaultDatabase; protected final String username; @@ -66,7 +76,7 @@ public abstract class AbstractJdbcCatalog implements Catalog { protected final Optional defaultSchema; - protected Connection defaultConnection; + protected final Map connectionMap; public AbstractJdbcCatalog( String catalogName, @@ -88,6 +98,7 @@ public AbstractJdbcCatalog( this.defaultUrl = urlInfo.getOrigin(); this.suffix = urlInfo.getSuffix(); this.defaultSchema = Optional.ofNullable(defaultSchema); + this.connectionMap = new ConcurrentHashMap<>(); } @Override @@ -95,51 +106,101 @@ public String getDefaultDatabase() { return defaultDatabase; } - public String getCatalogName() { - return catalogName; + protected Connection getConnection(String url) { + if (connectionMap.containsKey(url)) { + return connectionMap.get(url); + } + try { + Connection connection = DriverManager.getConnection(url, username, pwd); + connectionMap.put(url, connection); + return connection; + } catch (SQLException e) { + throw new CatalogException(String.format("Failed connecting to %s via JDBC.", url), e); + } } - public String getUsername() { - return username; + @Override + public void open() throws CatalogException { + getConnection(defaultUrl); + LOG.info("Catalog {} established connection to {}", catalogName, defaultUrl); } - public String getPassword() { - return pwd; + @Override + public void close() throws CatalogException { + for (Map.Entry entry : connectionMap.entrySet()) { + try { + entry.getValue().close(); + } catch (SQLException e) { + throw new CatalogException( + String.format("Failed to close %s via JDBC.", entry.getKey()), e); + } + } + connectionMap.clear(); + LOG.info("Catalog {} closing", catalogName); } - public String getBaseUrl() { - return baseUrl; + protected String getSelectColumnsSql(TablePath tablePath) { + throw new UnsupportedOperationException(); } - @Override - public void open() throws CatalogException { - try { - defaultConnection = DriverManager.getConnection(defaultUrl, username, pwd); - } catch (SQLException e) { - throw new CatalogException( - String.format("Failed connecting to %s via JDBC.", defaultUrl), e); - } + protected Column buildColumn(ResultSet resultSet) throws SQLException { + throw new UnsupportedOperationException(); + } - LOG.info("Catalog {} established connection to {}", catalogName, defaultUrl); + protected TableIdentifier getTableIdentifier(TablePath tablePath) { + return TableIdentifier.of( + catalogName, + tablePath.getDatabaseName(), + tablePath.getSchemaName(), + tablePath.getTableName()); } - @Override - public void close() throws CatalogException { - if (defaultConnection == null) { - return; + public CatalogTable getTable(TablePath tablePath) + throws CatalogException, TableNotExistException { + if (!tableExists(tablePath)) { + throw new TableNotExistException(catalogName, tablePath); } + + String dbUrl = getUrlFromDatabaseName(tablePath.getDatabaseName()); + Connection conn = getConnection(dbUrl); try { - defaultConnection.close(); - } catch (SQLException e) { + DatabaseMetaData metaData = conn.getMetaData(); + Optional primaryKey = getPrimaryKey(metaData, tablePath); + List constraintKeys = getConstraintKeys(metaData, tablePath); + try (PreparedStatement ps = conn.prepareStatement(getSelectColumnsSql(tablePath)); + ResultSet resultSet = ps.executeQuery()) { + + TableSchema.Builder builder = TableSchema.builder(); + while (resultSet.next()) { + builder.column(buildColumn(resultSet)); + } + // add primary key + primaryKey.ifPresent(builder::primaryKey); + // add constraint key + constraintKeys.forEach(builder::constraintKey); + TableIdentifier tableIdentifier = getTableIdentifier(tablePath); + return CatalogTable.of( + tableIdentifier, + builder.build(), + buildConnectorOptions(tablePath), + Collections.emptyList(), + "", + catalogName); + } + + } catch (Exception e) { throw new CatalogException( - String.format("Failed to close %s via JDBC.", defaultUrl), e); + String.format("Failed getting table %s", tablePath.getFullName()), e); } - LOG.info("Catalog {} closing", catalogName); } - protected Optional getPrimaryKey( - DatabaseMetaData metaData, String database, String table) throws SQLException { - return getPrimaryKey(metaData, database, table, table); + protected Optional getPrimaryKey(DatabaseMetaData metaData, TablePath tablePath) + throws SQLException { + return getPrimaryKey( + metaData, + tablePath.getDatabaseName(), + tablePath.getSchemaName(), + tablePath.getTableName()); } protected Optional getPrimaryKey( @@ -174,9 +235,13 @@ protected Optional getPrimaryKey( return Optional.of(PrimaryKey.of(pkName, pkFields)); } - protected List getConstraintKeys( - DatabaseMetaData metaData, String database, String table) throws SQLException { - return getConstraintKeys(metaData, database, table, table); + protected List getConstraintKeys(DatabaseMetaData metaData, TablePath tablePath) + throws SQLException { + return getConstraintKeys( + metaData, + tablePath.getDatabaseName(), + tablePath.getSchemaName(), + tablePath.getTableName()); } protected List getConstraintKeys( @@ -217,16 +282,24 @@ protected List getConstraintKeys( return new ArrayList<>(constraintKeyMap.values()); } - protected Optional getColumnDefaultValue( - DatabaseMetaData metaData, String database, String schema, String table, String column) - throws SQLException { - try (ResultSet resultSet = metaData.getColumns(database, schema, table, column)) { - while (resultSet.next()) { - String defaultValue = resultSet.getString("COLUMN_DEF"); - return Optional.ofNullable(defaultValue); - } + protected String getListDatabaseSql() { + throw new UnsupportedOperationException(); + } + + @Override + public List listDatabases() throws CatalogException { + try { + return queryString( + defaultUrl, + getListDatabaseSql(), + rs -> { + String s = rs.getString(1); + return SYS_DATABASES.contains(s) ? null : s; + }); + } catch (Exception e) { + throw new CatalogException( + String.format("Failed listing database in catalog %s", this.catalogName), e); } - return Optional.empty(); } @Override @@ -236,11 +309,44 @@ public boolean databaseExists(String databaseName) throws CatalogException { return listDatabases().contains(databaseName); } + protected String getListTableSql(String databaseName) { + throw new UnsupportedOperationException(); + } + + protected String getTableName(ResultSet rs) throws SQLException { + String schemaName = rs.getString(1); + String tableName = rs.getString(2); + if (StringUtils.isNotBlank(schemaName) && !SYS_DATABASES.contains(schemaName)) { + return schemaName + "." + tableName; + } + return null; + } + + protected String getTableName(TablePath tablePath) { + return tablePath.getSchemaAndTableName(); + } + + @Override + public List listTables(String databaseName) + throws CatalogException, DatabaseNotExistException { + if (!databaseExists(databaseName)) { + throw new DatabaseNotExistException(this.catalogName, databaseName); + } + + String dbUrl = getUrlFromDatabaseName(databaseName); + try { + return queryString(dbUrl, getListTableSql(databaseName), this::getTableName); + } catch (Exception e) { + throw new CatalogException( + String.format("Failed listing database in catalog %s", catalogName), e); + } + } + @Override public boolean tableExists(TablePath tablePath) throws CatalogException { try { return databaseExists(tablePath.getDatabaseName()) - && listTables(tablePath.getDatabaseName()).contains(tablePath.getTableName()); + && listTables(tablePath.getDatabaseName()).contains(getTableName(tablePath)); } catch (DatabaseNotExistException e) { return false; } @@ -261,24 +367,61 @@ public void createTable(TablePath tablePath, CatalogTable table, boolean ignoreI defaultSchema.get(), tablePath.getTableName()); } - if (!createTableInternal(tablePath, table) && !ignoreIfExists) { + + if (tableExists(tablePath)) { + if (ignoreIfExists) { + return; + } throw new TableAlreadyExistException(catalogName, tablePath); } + + createTableInternal(tablePath, table); + } + + protected String getCreateTableSql(TablePath tablePath, CatalogTable table) { + throw new UnsupportedOperationException(); } - protected abstract boolean createTableInternal(TablePath tablePath, CatalogTable table) - throws CatalogException; + protected void createTableInternal(TablePath tablePath, CatalogTable table) + throws CatalogException { + String dbUrl = getUrlFromDatabaseName(tablePath.getDatabaseName()); + try { + executeInternal(dbUrl, getCreateTableSql(tablePath, table)); + } catch (Exception e) { + throw new CatalogException( + String.format("Failed creating table %s", tablePath.getFullName()), e); + } + } @Override public void dropTable(TablePath tablePath, boolean ignoreIfNotExists) throws TableNotExistException, CatalogException { checkNotNull(tablePath, "Table path cannot be null"); - if (!dropTableInternal(tablePath) && !ignoreIfNotExists) { + + if (!tableExists(tablePath)) { + if (ignoreIfNotExists) { + return; + } throw new TableNotExistException(catalogName, tablePath); } + + dropTableInternal(tablePath); } - protected abstract boolean dropTableInternal(TablePath tablePath) throws CatalogException; + protected String getDropTableSql(TablePath tablePath) { + throw new UnsupportedOperationException(); + } + + protected void dropTableInternal(TablePath tablePath) throws CatalogException { + String dbUrl = getUrlFromDatabaseName(tablePath.getDatabaseName()); + try { + // Will there exist concurrent drop for one table? + executeInternal(dbUrl, getDropTableSql(tablePath)); + } catch (SQLException e) { + throw new CatalogException( + String.format("Failed dropping table %s", tablePath.getFullName()), e); + } + } @Override public void createDatabase(TablePath tablePath, boolean ignoreIfExists) @@ -287,14 +430,42 @@ public void createDatabase(TablePath tablePath, boolean ignoreIfExists) checkNotNull(tablePath.getDatabaseName(), "Database name cannot be null"); if (databaseExists(tablePath.getDatabaseName())) { + if (ignoreIfExists) { + return; + } throw new DatabaseAlreadyExistException(catalogName, tablePath.getDatabaseName()); } - if (!createDatabaseInternal(tablePath.getDatabaseName()) && !ignoreIfExists) { - throw new DatabaseAlreadyExistException(catalogName, tablePath.getDatabaseName()); + + createDatabaseInternal(tablePath.getDatabaseName()); + } + + protected String getCreateDatabaseSql(String databaseName) { + throw new UnsupportedOperationException(); + } + + protected void createDatabaseInternal(String databaseName) { + try { + executeInternal(defaultUrl, getCreateDatabaseSql(databaseName)); + } catch (Exception e) { + throw new CatalogException( + String.format( + "Failed creating database %s in catalog %s", + databaseName, this.catalogName), + e); } } - protected abstract boolean createDatabaseInternal(String databaseName); + protected void closeDatabaseConnection(String databaseName) { + String dbUrl = getUrlFromDatabaseName(databaseName); + try { + Connection connection = connectionMap.remove(dbUrl); + if (connection != null) { + connection.close(); + } + } catch (SQLException e) { + throw new CatalogException(String.format("Failed to close %s via JDBC.", dbUrl), e); + } + } @Override public void dropDatabase(TablePath tablePath, boolean ignoreIfNotExists) @@ -302,10 +473,77 @@ public void dropDatabase(TablePath tablePath, boolean ignoreIfNotExists) checkNotNull(tablePath, "Table path cannot be null"); checkNotNull(tablePath.getDatabaseName(), "Database name cannot be null"); - if (!dropDatabaseInternal(tablePath.getDatabaseName()) && !ignoreIfNotExists) { + if (!databaseExists(tablePath.getDatabaseName())) { + if (ignoreIfNotExists) { + return; + } throw new DatabaseNotExistException(catalogName, tablePath.getDatabaseName()); } + + dropDatabaseInternal(tablePath.getDatabaseName()); + } + + protected String getDropDatabaseSql(String databaseName) { + throw new UnsupportedOperationException(); + } + + protected void dropDatabaseInternal(String databaseName) throws CatalogException { + try { + executeInternal(defaultUrl, getDropDatabaseSql(databaseName)); + } catch (Exception e) { + throw new CatalogException( + String.format( + "Failed dropping database %s in catalog %s", + databaseName, this.catalogName), + e); + } + } + + protected String getUrlFromDatabaseName(String databaseName) { + String url = baseUrl.endsWith("/") ? baseUrl : baseUrl + "/"; + return url + databaseName + suffix; + } + + protected String getOptionTableName(TablePath tablePath) { + return tablePath.getFullName(); + } + + @SuppressWarnings("MagicNumber") + protected Map buildConnectorOptions(TablePath tablePath) { + Map options = new HashMap<>(8); + options.put("connector", "jdbc"); + options.put("url", getUrlFromDatabaseName(tablePath.getDatabaseName())); + options.put("table-name", getOptionTableName(tablePath)); + options.put("username", username); + options.put("password", pwd); + return options; + } + + @FunctionalInterface + public interface ResultSetConsumer { + T apply(ResultSet rs) throws SQLException; + } + + protected List queryString(String url, String sql, ResultSetConsumer consumer) + throws SQLException { + try (PreparedStatement ps = getConnection(url).prepareStatement(sql)) { + List result = new ArrayList<>(); + ResultSet rs = ps.executeQuery(); + while (rs.next()) { + String value = consumer.apply(rs); + if (value != null) { + result.add(value); + } + } + return result; + } } - protected abstract boolean dropDatabaseInternal(String databaseName) throws CatalogException; + // If sql is DDL, the execute() method always returns false, so the return value + // should not be used to determine whether changes were made in database. + protected boolean executeInternal(String url, String sql) throws SQLException { + try (PreparedStatement ps = getConnection(url).prepareStatement(sql)) { + return ps.execute(); + } + } } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/mysql/MySqlCatalog.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/mysql/MySqlCatalog.java index 267a68f0eef..b558926e453 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/mysql/MySqlCatalog.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/mysql/MySqlCatalog.java @@ -19,47 +19,34 @@ package org.apache.seatunnel.connectors.seatunnel.jdbc.catalog.mysql; import org.apache.seatunnel.api.table.catalog.CatalogTable; +import org.apache.seatunnel.api.table.catalog.Column; import org.apache.seatunnel.api.table.catalog.ConstraintKey; import org.apache.seatunnel.api.table.catalog.PhysicalColumn; import org.apache.seatunnel.api.table.catalog.PrimaryKey; import org.apache.seatunnel.api.table.catalog.TableIdentifier; import org.apache.seatunnel.api.table.catalog.TablePath; -import org.apache.seatunnel.api.table.catalog.TableSchema; -import org.apache.seatunnel.api.table.catalog.exception.CatalogException; -import org.apache.seatunnel.api.table.catalog.exception.DatabaseNotExistException; -import org.apache.seatunnel.api.table.catalog.exception.TableNotExistException; import org.apache.seatunnel.api.table.type.SeaTunnelDataType; import org.apache.seatunnel.common.utils.JdbcUrlUtil; import org.apache.seatunnel.connectors.seatunnel.jdbc.catalog.AbstractJdbcCatalog; import com.mysql.cj.MysqlType; -import com.mysql.cj.jdbc.result.ResultSetImpl; -import com.mysql.cj.util.StringUtils; import lombok.extern.slf4j.Slf4j; -import java.sql.Connection; import java.sql.DatabaseMetaData; -import java.sql.DriverManager; -import java.sql.PreparedStatement; import java.sql.ResultSet; -import java.sql.ResultSetMetaData; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Optional; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; @Slf4j public class MySqlCatalog extends AbstractJdbcCatalog { - protected static final Set SYS_DATABASES = new HashSet<>(4); - private final String SELECT_COLUMNS = + private static final MysqlDataTypeConvertor DATA_TYPE_CONVERTOR = new MysqlDataTypeConvertor(); + + private static final String SELECT_COLUMNS_SQL_TEMPLATE = "SELECT * FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = '%s' AND TABLE_NAME ='%s'"; static { @@ -69,137 +56,65 @@ public class MySqlCatalog extends AbstractJdbcCatalog { SYS_DATABASES.add("sys"); } - protected final Map connectionMap; - public MySqlCatalog( String catalogName, String username, String pwd, JdbcUrlUtil.UrlInfo urlInfo) { super(catalogName, username, pwd, urlInfo, null); - this.connectionMap = new ConcurrentHashMap<>(); } - public Connection getConnection(String url) { - if (connectionMap.containsKey(url)) { - return connectionMap.get(url); - } - try { - Connection connection = DriverManager.getConnection(url, username, pwd); - connectionMap.put(url, connection); - return connection; - } catch (SQLException e) { - throw new CatalogException(String.format("Failed connecting to %s via JDBC.", url), e); - } + @Override + protected String getListDatabaseSql() { + return "SHOW DATABASES;"; } @Override - public void close() throws CatalogException { - for (Map.Entry entry : connectionMap.entrySet()) { - try { - entry.getValue().close(); - } catch (SQLException e) { - throw new CatalogException( - String.format("Failed to close %s via JDBC.", entry.getKey()), e); - } - } - super.close(); + protected String getListTableSql(String databaseName) { + return "SHOW TABLES;"; } @Override - public List listDatabases() throws CatalogException { - try (PreparedStatement ps = defaultConnection.prepareStatement("SHOW DATABASES;")) { - - List databases = new ArrayList<>(); - ResultSet rs = ps.executeQuery(); - - while (rs.next()) { - String databaseName = rs.getString(1); - if (!SYS_DATABASES.contains(databaseName)) { - databases.add(rs.getString(1)); - } - } - - return databases; - } catch (Exception e) { - throw new CatalogException( - String.format("Failed listing database in catalog %s", this.catalogName), e); - } + protected String getTableName(ResultSet rs) throws SQLException { + return rs.getString(1); } @Override - public List listTables(String databaseName) - throws CatalogException, DatabaseNotExistException { - if (!databaseExists(databaseName)) { - throw new DatabaseNotExistException(this.catalogName, databaseName); - } - - String dbUrl = getUrlFromDatabaseName(databaseName); - Connection connection = getConnection(dbUrl); - try (PreparedStatement ps = connection.prepareStatement("SHOW TABLES;")) { - - ResultSet rs = ps.executeQuery(); - - List tables = new ArrayList<>(); - - while (rs.next()) { - tables.add(rs.getString(1)); - } - - return tables; - } catch (Exception e) { - throw new CatalogException( - String.format("Failed listing database in catalog %s", catalogName), e); - } + protected String getTableName(TablePath tablePath) { + return tablePath.getTableName(); } @Override - public CatalogTable getTable(TablePath tablePath) - throws CatalogException, TableNotExistException { - if (!tableExists(tablePath)) { - throw new TableNotExistException(catalogName, tablePath); - } - - String dbUrl = getUrlFromDatabaseName(tablePath.getDatabaseName()); - Connection conn = getConnection(dbUrl); - try { - DatabaseMetaData metaData = conn.getMetaData(); + protected String getSelectColumnsSql(TablePath tablePath) { + return String.format( + SELECT_COLUMNS_SQL_TEMPLATE, tablePath.getDatabaseName(), tablePath.getTableName()); + } - Optional primaryKey = - getPrimaryKey(metaData, tablePath.getDatabaseName(), tablePath.getTableName()); - List constraintKeys = - getConstraintKeys( - metaData, tablePath.getDatabaseName(), tablePath.getTableName()); - String sql = - String.format( - SELECT_COLUMNS, tablePath.getDatabaseName(), tablePath.getTableName()); - try (PreparedStatement ps = conn.prepareStatement(sql); - ResultSet resultSet = ps.executeQuery(); ) { + @Override + protected TableIdentifier getTableIdentifier(TablePath tablePath) { + return TableIdentifier.of( + catalogName, tablePath.getDatabaseName(), tablePath.getTableName()); + } - TableSchema.Builder builder = TableSchema.builder(); - while (resultSet.next()) { - buildTable(resultSet, builder); - } - // add primary key - primaryKey.ifPresent(builder::primaryKey); - // add constraint key - constraintKeys.forEach(builder::constraintKey); - TableIdentifier tableIdentifier = - TableIdentifier.of( - catalogName, tablePath.getDatabaseName(), tablePath.getTableName()); - return CatalogTable.of( - tableIdentifier, - builder.build(), - buildConnectorOptions(tablePath), - Collections.emptyList(), - "", - "mysql"); - } + @Override + protected Optional getPrimaryKey(DatabaseMetaData metaData, TablePath tablePath) + throws SQLException { + return getPrimaryKey( + metaData, + tablePath.getDatabaseName(), + tablePath.getTableName(), + tablePath.getTableName()); + } - } catch (Exception e) { - throw new CatalogException( - String.format("Failed getting table %s", tablePath.getFullName()), e); - } + @Override + protected List getConstraintKeys(DatabaseMetaData metaData, TablePath tablePath) + throws SQLException { + return getConstraintKeys( + metaData, + tablePath.getDatabaseName(), + tablePath.getTableName(), + tablePath.getTableName()); } - private void buildTable(ResultSet resultSet, TableSchema.Builder builder) throws SQLException { + @Override + protected Column buildColumn(ResultSet resultSet) throws SQLException { String columnName = resultSet.getString("COLUMN_NAME"); String sourceType = resultSet.getString("COLUMN_TYPE"); String typeName = resultSet.getString("DATA_TYPE").toUpperCase(); @@ -243,121 +158,39 @@ private void buildTable(ResultSet resultSet, TableSchema.Builder builder) throws break; } - PhysicalColumn physicalColumn = - PhysicalColumn.of( - columnName, - type, - 0, - isNullable, - defaultValue, - comment, - sourceType, - sourceType.contains("unsigned"), - sourceType.contains("zerofill"), - bitLen, - null, - columnLength); - builder.column(physicalColumn); + return PhysicalColumn.of( + columnName, + type, + 0, + isNullable, + defaultValue, + comment, + sourceType, + sourceType.contains("unsigned"), + sourceType.contains("zerofill"), + bitLen, + null, + columnLength); } - public static Map getColumnsDefaultValue(TablePath tablePath, Connection conn) { - StringBuilder queryBuf = new StringBuilder("SHOW FULL COLUMNS FROM "); - queryBuf.append(StringUtils.quoteIdentifier(tablePath.getTableName(), "`", false)); - queryBuf.append(" FROM "); - queryBuf.append(StringUtils.quoteIdentifier(tablePath.getDatabaseName(), "`", false)); - try (PreparedStatement ps2 = conn.prepareStatement(queryBuf.toString())) { - ResultSet rs = ps2.executeQuery(); - Map result = new HashMap<>(); - while (rs.next()) { - String field = rs.getString("Field"); - Object defaultValue = rs.getObject("Default"); - result.put(field, defaultValue); - } - return result; - } catch (Exception e) { - throw new CatalogException( - String.format( - "Failed getting table(%s) columns default value", - tablePath.getFullName()), - e); - } - } - - // todo: If the origin source is mysql, we can directly use create table like to create the @Override - protected boolean createTableInternal(TablePath tablePath, CatalogTable table) - throws CatalogException { - String dbUrl = getUrlFromDatabaseName(tablePath.getDatabaseName()); - - String createTableSql = - MysqlCreateTableSqlBuilder.builder(tablePath, table).build(table.getCatalogName()); - Connection connection = getConnection(dbUrl); - log.info("create table sql: {}", createTableSql); - try (PreparedStatement ps = connection.prepareStatement(createTableSql)) { - return ps.execute(); - } catch (Exception e) { - throw new CatalogException( - String.format("Failed creating table %s", tablePath.getFullName()), e); - } + protected String getCreateTableSql(TablePath tablePath, CatalogTable table) { + return MysqlCreateTableSqlBuilder.builder(tablePath, table).build(table.getCatalogName()); } @Override - protected boolean dropTableInternal(TablePath tablePath) throws CatalogException { - String dbUrl = getUrlFromDatabaseName(tablePath.getDatabaseName()); - Connection connection = getConnection(dbUrl); - try (PreparedStatement ps = - connection.prepareStatement( - String.format("DROP TABLE IF EXISTS %s;", tablePath.getFullName()))) { - // Will there exist concurrent drop for one table? - return ps.execute(); - } catch (SQLException e) { - throw new CatalogException( - String.format("Failed dropping table %s", tablePath.getFullName()), e); - } + protected String getDropTableSql(TablePath tablePath) { + return String.format("DROP TABLE %s;", tablePath.getFullName()); } @Override - protected boolean createDatabaseInternal(String databaseName) throws CatalogException { - try (PreparedStatement ps = - defaultConnection.prepareStatement( - String.format("CREATE DATABASE `%s`;", databaseName))) { - return ps.execute(); - } catch (Exception e) { - throw new CatalogException( - String.format( - "Failed creating database %s in catalog %s", - databaseName, this.catalogName), - e); - } + protected String getCreateDatabaseSql(String databaseName) { + return String.format("CREATE DATABASE `%s`;", databaseName); } @Override - protected boolean dropDatabaseInternal(String databaseName) throws CatalogException { - try (PreparedStatement ps = - defaultConnection.prepareStatement( - String.format("DROP DATABASE `%s`;", databaseName))) { - return ps.execute(); - } catch (Exception e) { - throw new CatalogException( - String.format( - "Failed dropping database %s in catalog %s", - databaseName, this.catalogName), - e); - } - } - - /** - * @see com.mysql.cj.MysqlType - * @see ResultSetImpl#getObjectStoredProc(int, int) - */ - @SuppressWarnings("unchecked") - private SeaTunnelDataType fromJdbcType(ResultSetMetaData metadata, int colIndex) - throws SQLException { - MysqlType mysqlType = MysqlType.getByName(metadata.getColumnTypeName(colIndex)); - Map dataTypeProperties = new HashMap<>(); - dataTypeProperties.put(MysqlDataTypeConvertor.PRECISION, metadata.getPrecision(colIndex)); - dataTypeProperties.put(MysqlDataTypeConvertor.SCALE, metadata.getScale(colIndex)); - return new MysqlDataTypeConvertor().toSeaTunnelType(mysqlType, dataTypeProperties); + protected String getDropDatabaseSql(String databaseName) { + return String.format("DROP DATABASE `%s`;", databaseName); } private SeaTunnelDataType fromJdbcType(String typeName, int precision, int scale) { @@ -365,22 +198,6 @@ private SeaTunnelDataType fromJdbcType(String typeName, int precision, int sc Map dataTypeProperties = new HashMap<>(); dataTypeProperties.put(MysqlDataTypeConvertor.PRECISION, precision); dataTypeProperties.put(MysqlDataTypeConvertor.SCALE, scale); - return new MysqlDataTypeConvertor().toSeaTunnelType(mysqlType, dataTypeProperties); - } - - @SuppressWarnings("MagicNumber") - private Map buildConnectorOptions(TablePath tablePath) { - Map options = new HashMap<>(8); - options.put("connector", "jdbc"); - options.put("url", baseUrl + tablePath.getDatabaseName()); - options.put("table-name", tablePath.getFullName()); - options.put("username", username); - options.put("password", pwd); - return options; - } - - private String getUrlFromDatabaseName(String databaseName) { - String url = baseUrl.endsWith("/") ? baseUrl : baseUrl + "/"; - return url + databaseName + suffix; + return DATA_TYPE_CONVERTOR.toSeaTunnelType(mysqlType, dataTypeProperties); } } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/mysql/MysqlCreateTableSqlBuilder.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/mysql/MysqlCreateTableSqlBuilder.java index 608062fc999..490ecd30ff8 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/mysql/MysqlCreateTableSqlBuilder.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/mysql/MysqlCreateTableSqlBuilder.java @@ -119,7 +119,7 @@ public String build(String catalogName) { List sqls = new ArrayList<>(); sqls.add( String.format( - "CREATE TABLE IF NOT EXISTS %s (\n%s\n)", + "CREATE TABLE %s (\n%s\n)", tableName, buildColumnsIdentifySql(catalogName))); if (engine != null) { sqls.add("ENGINE = " + engine); diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/oracle/OracleCatalog.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/oracle/OracleCatalog.java index 261f4f7fb6f..b90a86a7abb 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/oracle/OracleCatalog.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/oracle/OracleCatalog.java @@ -18,33 +18,22 @@ package org.apache.seatunnel.connectors.seatunnel.jdbc.catalog.oracle; import org.apache.seatunnel.api.table.catalog.CatalogTable; -import org.apache.seatunnel.api.table.catalog.ConstraintKey; +import org.apache.seatunnel.api.table.catalog.Column; import org.apache.seatunnel.api.table.catalog.PhysicalColumn; -import org.apache.seatunnel.api.table.catalog.PrimaryKey; -import org.apache.seatunnel.api.table.catalog.TableIdentifier; import org.apache.seatunnel.api.table.catalog.TablePath; -import org.apache.seatunnel.api.table.catalog.TableSchema; -import org.apache.seatunnel.api.table.catalog.exception.CatalogException; -import org.apache.seatunnel.api.table.catalog.exception.DatabaseNotExistException; -import org.apache.seatunnel.api.table.catalog.exception.TableNotExistException; import org.apache.seatunnel.api.table.type.SeaTunnelDataType; import org.apache.seatunnel.common.utils.JdbcUrlUtil; import org.apache.seatunnel.connectors.seatunnel.jdbc.catalog.AbstractJdbcCatalog; import lombok.extern.slf4j.Slf4j; -import java.sql.DatabaseMetaData; -import java.sql.PreparedStatement; import java.sql.ResultSet; -import java.sql.ResultSetMetaData; import java.sql.SQLException; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import static org.apache.seatunnel.connectors.seatunnel.jdbc.catalog.oracle.OracleDataTypeConvertor.ORACLE_BFILE; import static org.apache.seatunnel.connectors.seatunnel.jdbc.catalog.oracle.OracleDataTypeConvertor.ORACLE_BLOB; @@ -61,8 +50,10 @@ @Slf4j public class OracleCatalog extends AbstractJdbcCatalog { + private static final OracleDataTypeConvertor DATA_TYPE_CONVERTOR = new OracleDataTypeConvertor(); + private static final List EXCLUDED_SCHEMAS = Collections.unmodifiableList( Arrays.asList( @@ -87,7 +78,7 @@ public class OracleCatalog extends AbstractJdbcCatalog { "EXFSYS", "SYSMAN")); - private static final String SELECT_COLUMNS_SQL = + private static final String SELECT_COLUMNS_SQL_TEMPLATE = "SELECT\n" + " cols.COLUMN_NAME,\n" + " CASE \n" @@ -127,158 +118,50 @@ public OracleCatalog( } @Override - public List listDatabases() throws CatalogException { - try (PreparedStatement ps = - defaultConnection.prepareStatement("SELECT name FROM v$database")) { - - List databases = new ArrayList<>(); - ResultSet rs = ps.executeQuery(); - - while (rs.next()) { - String databaseName = rs.getString(1); - databases.add(databaseName); - } - return databases; - } catch (Exception e) { - throw new CatalogException( - String.format("Failed listing database in catalog %s", this.catalogName), e); - } + protected String getListDatabaseSql() { + return "SELECT name FROM v$database"; } @Override - protected boolean createTableInternal(TablePath tablePath, CatalogTable table) - throws CatalogException { - String createTableSql = new OracleCreateTableSqlBuilder(table).build(tablePath); - String[] createTableSqls = createTableSql.split(";"); - for (String sql : createTableSqls) { - log.info("create table sql: {}", sql); - try (PreparedStatement ps = defaultConnection.prepareStatement(sql)) { - ps.execute(); - } catch (Exception e) { - throw new CatalogException( - String.format("Failed creating table %s", tablePath.getFullName()), e); - } - } - return true; + protected String getCreateTableSql(TablePath tablePath, CatalogTable table) { + return new OracleCreateTableSqlBuilder(table).build(tablePath); } @Override - protected boolean dropTableInternal(TablePath tablePath) throws CatalogException { - return false; + protected String getDropTableSql(TablePath tablePath) { + return String.format("DROP TABLE %s", getTableName(tablePath)); } @Override - protected boolean createDatabaseInternal(String databaseName) { - return false; + protected String getTableName(TablePath tablePath) { + return tablePath.getSchemaAndTableName().toUpperCase(); } @Override - protected boolean dropDatabaseInternal(String databaseName) throws CatalogException { - return false; + protected String getListTableSql(String databaseName) { + return "SELECT OWNER, TABLE_NAME FROM ALL_TABLES" + + " WHERE TABLE_NAME NOT LIKE 'MDRT_%'" + + " AND TABLE_NAME NOT LIKE 'MDRS_%'" + + " AND TABLE_NAME NOT LIKE 'MDXT_%'" + + " AND (TABLE_NAME NOT LIKE 'SYS_IOT_OVER_%' AND IOT_NAME IS NULL)"; } @Override - public boolean tableExists(TablePath tablePath) throws CatalogException { - try { - return databaseExists(tablePath.getDatabaseName()) - && listTables(tablePath.getDatabaseName()) - .contains(tablePath.getSchemaAndTableName().toUpperCase()); - } catch (DatabaseNotExistException e) { - return false; + protected String getTableName(ResultSet rs) throws SQLException { + if (EXCLUDED_SCHEMAS.contains(rs.getString(1))) { + return null; } + return rs.getString(1) + "." + rs.getString(2); } @Override - public List listTables(String databaseName) - throws CatalogException, DatabaseNotExistException { - if (!databaseExists(databaseName)) { - throw new DatabaseNotExistException(this.catalogName, databaseName); - } - - try (PreparedStatement ps = - defaultConnection.prepareStatement( - "SELECT OWNER, TABLE_NAME FROM ALL_TABLES\n" - + "WHERE TABLE_NAME NOT LIKE 'MDRT_%'\n" - + " AND TABLE_NAME NOT LIKE 'MDRS_%'\n" - + " AND TABLE_NAME NOT LIKE 'MDXT_%'\n" - + " AND (TABLE_NAME NOT LIKE 'SYS_IOT_OVER_%' AND IOT_NAME IS NULL)")) { - - ResultSet rs = ps.executeQuery(); - List tables = new ArrayList<>(); - while (rs.next()) { - if (EXCLUDED_SCHEMAS.contains(rs.getString(1))) { - continue; - } - tables.add(rs.getString(1) + "." + rs.getString(2)); - } - - return tables; - } catch (Exception e) { - throw new CatalogException( - String.format("Failed listing database in catalog %s", catalogName), e); - } + protected String getSelectColumnsSql(TablePath tablePath) { + return String.format( + SELECT_COLUMNS_SQL_TEMPLATE, tablePath.getSchemaName(), tablePath.getTableName()); } @Override - public CatalogTable getTable(TablePath tablePath) - throws CatalogException, TableNotExistException { - if (!tableExists(tablePath)) { - throw new TableNotExistException(catalogName, tablePath); - } - - try { - DatabaseMetaData metaData = defaultConnection.getMetaData(); - Optional primaryKey = - getPrimaryKey( - metaData, - tablePath.getDatabaseName(), - tablePath.getSchemaName(), - tablePath.getTableName()); - List constraintKeys = - getConstraintKeys( - metaData, - tablePath.getDatabaseName(), - tablePath.getSchemaName(), - tablePath.getTableName()); - - String sql = - String.format( - SELECT_COLUMNS_SQL, - tablePath.getSchemaName(), - tablePath.getTableName()); - try (PreparedStatement ps = defaultConnection.prepareStatement(sql); - ResultSet resultSet = ps.executeQuery()) { - TableSchema.Builder builder = TableSchema.builder(); - // add column - while (resultSet.next()) { - buildColumn(resultSet, builder); - } - - // add primary key - primaryKey.ifPresent(builder::primaryKey); - // add constraint key - constraintKeys.forEach(builder::constraintKey); - TableIdentifier tableIdentifier = - TableIdentifier.of( - catalogName, - tablePath.getDatabaseName(), - tablePath.getSchemaName(), - tablePath.getTableName()); - return CatalogTable.of( - tableIdentifier, - builder.build(), - buildConnectorOptions(tablePath), - Collections.emptyList(), - ""); - } - - } catch (Exception e) { - throw new CatalogException( - String.format("Failed getting table %s", tablePath.getFullName()), e); - } - } - - private void buildColumn(ResultSet resultSet, TableSchema.Builder builder) throws SQLException { + protected Column buildColumn(ResultSet resultSet) throws SQLException { String columnName = resultSet.getString("COLUMN_NAME"); String typeName = resultSet.getString("TYPE_NAME"); String fullTypeName = resultSet.getString("FULL_TYPE_NAME"); @@ -314,31 +197,19 @@ private void buildColumn(ResultSet resultSet, TableSchema.Builder builder) throw break; } - PhysicalColumn physicalColumn = - PhysicalColumn.of( - columnName, - type, - 0, - isNullable, - defaultValue, - columnComment, - fullTypeName, - false, - false, - bitLen, - null, - columnLength); - builder.column(physicalColumn); - } - - @SuppressWarnings("unchecked") - private SeaTunnelDataType fromJdbcType(ResultSetMetaData metadata, int colIndex) - throws SQLException { - String columnType = metadata.getColumnTypeName(colIndex); - Map dataTypeProperties = new HashMap<>(); - dataTypeProperties.put(OracleDataTypeConvertor.PRECISION, metadata.getPrecision(colIndex)); - dataTypeProperties.put(OracleDataTypeConvertor.SCALE, metadata.getScale(colIndex)); - return DATA_TYPE_CONVERTOR.toSeaTunnelType(columnType, dataTypeProperties); + return PhysicalColumn.of( + columnName, + type, + 0, + isNullable, + defaultValue, + columnComment, + fullTypeName, + false, + false, + bitLen, + null, + columnLength); } private SeaTunnelDataType fromJdbcType(String typeName, long precision, long scale) { @@ -348,14 +219,13 @@ private SeaTunnelDataType fromJdbcType(String typeName, long precision, long return DATA_TYPE_CONVERTOR.toSeaTunnelType(typeName, dataTypeProperties); } - @SuppressWarnings("MagicNumber") - private Map buildConnectorOptions(TablePath tablePath) { - Map options = new HashMap<>(8); - options.put("connector", "jdbc"); - options.put("url", baseUrl); - options.put("table-name", tablePath.getSchemaAndTableName()); - options.put("username", username); - options.put("password", pwd); - return options; + @Override + protected String getUrlFromDatabaseName(String databaseName) { + return defaultUrl; + } + + @Override + protected String getOptionTableName(TablePath tablePath) { + return tablePath.getSchemaAndTableName(); } } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/psql/PostgresCatalog.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/psql/PostgresCatalog.java index e3507666d08..2769d09ebb7 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/psql/PostgresCatalog.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/psql/PostgresCatalog.java @@ -18,39 +18,20 @@ package org.apache.seatunnel.connectors.seatunnel.jdbc.catalog.psql; import org.apache.seatunnel.api.table.catalog.CatalogTable; -import org.apache.seatunnel.api.table.catalog.ConstraintKey; +import org.apache.seatunnel.api.table.catalog.Column; import org.apache.seatunnel.api.table.catalog.PhysicalColumn; -import org.apache.seatunnel.api.table.catalog.PrimaryKey; -import org.apache.seatunnel.api.table.catalog.TableIdentifier; import org.apache.seatunnel.api.table.catalog.TablePath; -import org.apache.seatunnel.api.table.catalog.TableSchema; import org.apache.seatunnel.api.table.catalog.exception.CatalogException; -import org.apache.seatunnel.api.table.catalog.exception.DatabaseNotExistException; -import org.apache.seatunnel.api.table.catalog.exception.TableNotExistException; import org.apache.seatunnel.api.table.type.SeaTunnelDataType; import org.apache.seatunnel.common.utils.JdbcUrlUtil; import org.apache.seatunnel.connectors.seatunnel.jdbc.catalog.AbstractJdbcCatalog; -import com.mysql.cj.MysqlType; -import com.mysql.cj.jdbc.result.ResultSetImpl; import lombok.extern.slf4j.Slf4j; -import java.sql.Connection; -import java.sql.DatabaseMetaData; -import java.sql.DriverManager; -import java.sql.PreparedStatement; import java.sql.ResultSet; -import java.sql.ResultSetMetaData; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; -import java.util.List; import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; import static org.apache.seatunnel.connectors.seatunnel.jdbc.catalog.psql.PostgresDataTypeConvertor.PG_BIT; import static org.apache.seatunnel.connectors.seatunnel.jdbc.catalog.psql.PostgresDataTypeConvertor.PG_BYTEA; @@ -65,7 +46,10 @@ @Slf4j public class PostgresCatalog extends AbstractJdbcCatalog { - private static final String SELECT_COLUMNS_SQL = + private static final PostgresDataTypeConvertor DATA_TYPE_CONVERTOR = + new PostgresDataTypeConvertor(); + + private static final String SELECT_COLUMNS_SQL_TEMPLATE = "SELECT \n" + " a.attname AS column_name, \n" + "\t\tt.typname as type_name,\n" @@ -102,8 +86,6 @@ public class PostgresCatalog extends AbstractJdbcCatalog { + "ORDER BY \n" + " a.attnum;"; - protected static final Set SYS_DATABASES = new HashSet<>(9); - static { SYS_DATABASES.add("information_schema"); SYS_DATABASES.add("pg_catalog"); @@ -116,8 +98,6 @@ public class PostgresCatalog extends AbstractJdbcCatalog { SYS_DATABASES.add("template1"); } - protected final Map connectionMap; - public PostgresCatalog( String catalogName, String username, @@ -125,154 +105,26 @@ public PostgresCatalog( JdbcUrlUtil.UrlInfo urlInfo, String defaultSchema) { super(catalogName, username, pwd, urlInfo, defaultSchema); - this.connectionMap = new ConcurrentHashMap<>(); - } - - public Connection getConnection(String url) { - if (connectionMap.containsKey(url)) { - return connectionMap.get(url); - } - try { - Connection connection = DriverManager.getConnection(url, username, pwd); - connectionMap.put(url, connection); - return connection; - } catch (SQLException e) { - throw new CatalogException(String.format("Failed connecting to %s via JDBC.", url), e); - } } @Override - public void close() throws CatalogException { - for (Map.Entry entry : connectionMap.entrySet()) { - try { - entry.getValue().close(); - } catch (SQLException e) { - throw new CatalogException( - String.format("Failed to close %s via JDBC.", entry.getKey()), e); - } - } - super.close(); + protected String getListDatabaseSql() { + return "select datname from pg_database;"; } @Override - public List listDatabases() throws CatalogException { - try (PreparedStatement ps = - defaultConnection.prepareStatement("select datname from pg_database;")) { - - List databases = new ArrayList<>(); - ResultSet rs = ps.executeQuery(); - - while (rs.next()) { - String databaseName = rs.getString(1); - if (!SYS_DATABASES.contains(databaseName)) { - databases.add(rs.getString(1)); - } - } - - return databases; - } catch (Exception e) { - throw new CatalogException( - String.format("Failed listing database in catalog %s", this.catalogName), e); - } + protected String getListTableSql(String databaseName) { + return "SELECT table_schema, table_name FROM information_schema.tables;"; } @Override - public List listTables(String databaseName) - throws CatalogException, DatabaseNotExistException { - if (!databaseExists(databaseName)) { - throw new DatabaseNotExistException(this.catalogName, databaseName); - } - - String dbUrl = getUrlFromDatabaseName(databaseName); - Connection connection = getConnection(dbUrl); - try (PreparedStatement ps = - connection.prepareStatement( - "SELECT table_schema, table_name FROM information_schema.tables;")) { - - ResultSet rs = ps.executeQuery(); - - List tables = new ArrayList<>(); - - while (rs.next()) { - String schemaName = rs.getString("table_schema"); - String tableName = rs.getString("table_name"); - if (org.apache.commons.lang3.StringUtils.isNotBlank(schemaName) - && !SYS_DATABASES.contains(schemaName)) { - tables.add(schemaName + "." + tableName); - } - } - - return tables; - } catch (Exception e) { - throw new CatalogException( - String.format("Failed listing database in catalog %s", catalogName), e); - } + protected String getSelectColumnsSql(TablePath tablePath) { + return String.format( + SELECT_COLUMNS_SQL_TEMPLATE, tablePath.getSchemaName(), tablePath.getTableName()); } @Override - public CatalogTable getTable(TablePath tablePath) - throws CatalogException, TableNotExistException { - if (!tableExists(tablePath)) { - throw new TableNotExistException(catalogName, tablePath); - } - - String dbUrl = getUrlFromDatabaseName(tablePath.getDatabaseName()); - Connection conn = getConnection(dbUrl); - try { - DatabaseMetaData metaData = conn.getMetaData(); - Optional primaryKey = - getPrimaryKey( - metaData, - tablePath.getDatabaseName(), - tablePath.getSchemaName(), - tablePath.getTableName()); - List constraintKeys = - getConstraintKeys( - metaData, - tablePath.getDatabaseName(), - tablePath.getSchemaName(), - tablePath.getTableName()); - - String sql = - String.format( - SELECT_COLUMNS_SQL, - tablePath.getSchemaName(), - tablePath.getTableName()); - try (PreparedStatement ps = conn.prepareStatement(sql); - ResultSet resultSet = ps.executeQuery()) { - TableSchema.Builder builder = TableSchema.builder(); - - // add column - while (resultSet.next()) { - buildColumn(resultSet, builder); - } - - // add primary key - primaryKey.ifPresent(builder::primaryKey); - // add constraint key - constraintKeys.forEach(builder::constraintKey); - TableIdentifier tableIdentifier = - TableIdentifier.of( - catalogName, - tablePath.getDatabaseName(), - tablePath.getSchemaName(), - tablePath.getTableName()); - return CatalogTable.of( - tableIdentifier, - builder.build(), - buildConnectorOptions(tablePath), - Collections.emptyList(), - "", - "postgres"); - } - - } catch (Exception e) { - throw new CatalogException( - String.format("Failed getting table %s", tablePath.getFullName()), e); - } - } - - private void buildColumn(ResultSet resultSet, TableSchema.Builder builder) throws SQLException { + protected Column buildColumn(ResultSet resultSet) throws SQLException { String columnName = resultSet.getString("column_name"); String typeName = resultSet.getString("type_name"); String fullTypeName = resultSet.getString("full_type_name"); @@ -282,8 +134,9 @@ private void buildColumn(ResultSet resultSet, TableSchema.Builder builder) throw Object defaultValue = resultSet.getObject("default_value"); boolean isNullable = resultSet.getString("is_nullable").equals("YES"); - if (defaultValue != null && defaultValue.toString().contains("regclass")) + if (defaultValue != null && defaultValue.toString().contains("regclass")) { defaultValue = null; + } SeaTunnelDataType type = fromJdbcType(typeName, columnLength, columnScale); long bitLen = 0; @@ -311,131 +164,55 @@ private void buildColumn(ResultSet resultSet, TableSchema.Builder builder) throw break; } - PhysicalColumn physicalColumn = - PhysicalColumn.of( - columnName, - type, - 0, - isNullable, - defaultValue, - columnComment, - fullTypeName, - false, - false, - bitLen, - null, - columnLength); - builder.column(physicalColumn); + return PhysicalColumn.of( + columnName, + type, + 0, + isNullable, + defaultValue, + columnComment, + fullTypeName, + false, + false, + bitLen, + null, + columnLength); } @Override - protected boolean createTableInternal(TablePath tablePath, CatalogTable table) - throws CatalogException { - String createTableSql = new PostgresCreateTableSqlBuilder(table).build(tablePath); - String dbUrl = getUrlFromDatabaseName(tablePath.getDatabaseName()); - Connection conn = getConnection(dbUrl); - log.info("create table sql: {}", createTableSql); - try (PreparedStatement ps = conn.prepareStatement(createTableSql)) { - ps.execute(); - } catch (Exception e) { - throw new CatalogException( - String.format("Failed creating table %s", tablePath.getFullName()), e); - } - return true; + protected String getCreateTableSql(TablePath tablePath, CatalogTable table) { + return new PostgresCreateTableSqlBuilder(table).build(tablePath); } @Override - protected boolean dropTableInternal(TablePath tablePath) throws CatalogException { - String dbUrl = getUrlFromDatabaseName(tablePath.getDatabaseName()); - - String schemaName = tablePath.getSchemaName(); - String tableName = tablePath.getTableName(); - - String sql = "DROP TABLE IF EXISTS \"" + schemaName + "\".\"" + tableName + "\""; - Connection connection = getConnection(dbUrl); - try (PreparedStatement ps = connection.prepareStatement(sql)) { - // Will there exist concurrent drop for one table? - return ps.execute(); - } catch (SQLException e) { - throw new CatalogException( - String.format("Failed dropping table %s", tablePath.getFullName()), e); - } + protected String getDropTableSql(TablePath tablePath) { + return "DROP TABLE \"" + + tablePath.getSchemaName() + + "\".\"" + + tablePath.getTableName() + + "\""; } @Override - protected boolean createDatabaseInternal(String databaseName) throws CatalogException { - String sql = "CREATE DATABASE \"" + databaseName + "\""; - try (PreparedStatement ps = defaultConnection.prepareStatement(sql)) { - return ps.execute(); - } catch (Exception e) { - throw new CatalogException( - String.format( - "Failed creating database %s in catalog %s", - databaseName, this.catalogName), - e); - } + protected String getCreateDatabaseSql(String databaseName) { + return "CREATE DATABASE \"" + databaseName + "\""; } @Override - public boolean tableExists(TablePath tablePath) throws CatalogException { - try { - return databaseExists(tablePath.getDatabaseName()) - && listTables(tablePath.getDatabaseName()) - .contains(tablePath.getSchemaAndTableName()); - } catch (DatabaseNotExistException e) { - return false; - } + protected String getDropDatabaseSql(String databaseName) { + return "DROP DATABASE \"" + databaseName + "\""; } @Override - protected boolean dropDatabaseInternal(String databaseName) throws CatalogException { - String sql = "DROP DATABASE IF EXISTS \"" + databaseName + "\""; - try (PreparedStatement ps = defaultConnection.prepareStatement(sql)) { - return ps.execute(); - } catch (Exception e) { - throw new CatalogException( - String.format( - "Failed dropping database %s in catalog %s", - databaseName, this.catalogName), - e); - } - } - - /** - * @see MysqlType - * @see ResultSetImpl#getObjectStoredProc(int, int) - */ - @SuppressWarnings("unchecked") - private SeaTunnelDataType fromJdbcType(ResultSetMetaData metadata, int colIndex) - throws SQLException { - String columnTypeName = metadata.getColumnTypeName(colIndex); - Map dataTypeProperties = new HashMap<>(); - dataTypeProperties.put( - PostgresDataTypeConvertor.PRECISION, metadata.getPrecision(colIndex)); - dataTypeProperties.put(PostgresDataTypeConvertor.SCALE, metadata.getScale(colIndex)); - return new PostgresDataTypeConvertor().toSeaTunnelType(columnTypeName, dataTypeProperties); + protected void dropDatabaseInternal(String databaseName) throws CatalogException { + closeDatabaseConnection(databaseName); + super.dropDatabaseInternal(databaseName); } private SeaTunnelDataType fromJdbcType(String typeName, long precision, long scale) { Map dataTypeProperties = new HashMap<>(); dataTypeProperties.put(PostgresDataTypeConvertor.PRECISION, precision); dataTypeProperties.put(PostgresDataTypeConvertor.SCALE, scale); - return new PostgresDataTypeConvertor().toSeaTunnelType(typeName, dataTypeProperties); - } - - @SuppressWarnings("MagicNumber") - private Map buildConnectorOptions(TablePath tablePath) { - Map options = new HashMap<>(8); - options.put("connector", "jdbc"); - options.put("url", baseUrl + tablePath.getDatabaseName()); - options.put("table-name", tablePath.getFullName()); - options.put("username", username); - options.put("password", pwd); - return options; - } - - private String getUrlFromDatabaseName(String databaseName) { - String url = baseUrl.endsWith("/") ? baseUrl : baseUrl + "/"; - return url + databaseName + suffix; + return DATA_TYPE_CONVERTOR.toSeaTunnelType(typeName, dataTypeProperties); } } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/psql/PostgresCreateTableSqlBuilder.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/psql/PostgresCreateTableSqlBuilder.java index 85f4468bef9..d423f183010 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/psql/PostgresCreateTableSqlBuilder.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/psql/PostgresCreateTableSqlBuilder.java @@ -48,7 +48,7 @@ public PostgresCreateTableSqlBuilder(CatalogTable catalogTable) { public String build(TablePath tablePath) { StringBuilder createTableSql = new StringBuilder(); createTableSql - .append("CREATE TABLE IF NOT EXISTS ") + .append("CREATE TABLE ") .append(tablePath.getSchemaAndTableName()) .append(" (\n"); diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/sqlserver/SqlServerCatalog.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/sqlserver/SqlServerCatalog.java index ea04c60bff5..7d18ed2d905 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/sqlserver/SqlServerCatalog.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/sqlserver/SqlServerCatalog.java @@ -19,15 +19,10 @@ package org.apache.seatunnel.connectors.seatunnel.jdbc.catalog.sqlserver; import org.apache.seatunnel.api.table.catalog.CatalogTable; -import org.apache.seatunnel.api.table.catalog.ConstraintKey; +import org.apache.seatunnel.api.table.catalog.Column; import org.apache.seatunnel.api.table.catalog.PhysicalColumn; -import org.apache.seatunnel.api.table.catalog.PrimaryKey; -import org.apache.seatunnel.api.table.catalog.TableIdentifier; import org.apache.seatunnel.api.table.catalog.TablePath; -import org.apache.seatunnel.api.table.catalog.TableSchema; import org.apache.seatunnel.api.table.catalog.exception.CatalogException; -import org.apache.seatunnel.api.table.catalog.exception.DatabaseNotExistException; -import org.apache.seatunnel.api.table.catalog.exception.TableNotExistException; import org.apache.seatunnel.api.table.type.SeaTunnelDataType; import org.apache.seatunnel.common.utils.JdbcUrlUtil; import org.apache.seatunnel.connectors.seatunnel.jdbc.catalog.AbstractJdbcCatalog; @@ -37,33 +32,35 @@ import lombok.extern.slf4j.Slf4j; -import java.sql.Connection; -import java.sql.DatabaseMetaData; -import java.sql.DriverManager; -import java.sql.PreparedStatement; import java.sql.ResultSet; -import java.sql.ResultSetMetaData; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; -import java.util.List; import java.util.Map; -import java.util.Optional; -import java.util.Set; @Slf4j public class SqlServerCatalog extends AbstractJdbcCatalog { - private static final Set SYS_DATABASES = new HashSet<>(4); - - static { - SYS_DATABASES.add("master"); - SYS_DATABASES.add("tempdb"); - SYS_DATABASES.add("model"); - SYS_DATABASES.add("msdb"); - } + private static final SqlServerDataTypeConvertor DATA_TYPE_CONVERTOR = + new SqlServerDataTypeConvertor(); + + private static final String SELECT_COLUMNS_SQL_TEMPLATE = + "SELECT tbl.name AS table_name,\n" + + " col.name AS column_name,\n" + + " ext.value AS comment,\n" + + " col.column_id AS column_id,\n" + + " types.name AS type,\n" + + " col.max_length AS max_length,\n" + + " col.precision AS precision,\n" + + " col.scale AS scale,\n" + + " col.is_nullable AS is_nullable,\n" + + " def.definition AS default_value\n" + + "FROM sys.tables tbl\n" + + " INNER JOIN sys.columns col ON tbl.object_id = col.object_id\n" + + " LEFT JOIN sys.types types ON col.user_type_id = types.user_type_id\n" + + " LEFT JOIN sys.extended_properties ext ON ext.major_id = col.object_id AND ext.minor_id = col.column_id\n" + + " LEFT JOIN sys.default_constraints def ON col.default_object_id = def.object_id AND ext.minor_id = col.column_id AND ext.name = 'MS_Description'\n" + + "WHERE schema_name(tbl.schema_id) = '%s' %s\n" + + "ORDER BY tbl.name, col.column_id"; public SqlServerCatalog( String catalogName, @@ -75,133 +72,29 @@ public SqlServerCatalog( } @Override - public List listDatabases() throws CatalogException { - try (Connection conn = DriverManager.getConnection(defaultUrl, username, pwd); - PreparedStatement ps = conn.prepareStatement("SELECT NAME FROM sys.databases")) { - - List databases = new ArrayList<>(); - ResultSet rs = ps.executeQuery(); - - while (rs.next()) { - String databaseName = rs.getString(1); - if (!SYS_DATABASES.contains(databaseName)) { - databases.add(databaseName); - } - } - - return databases; - } catch (Exception e) { - throw new CatalogException( - String.format("Failed listing database in catalog %s", this.catalogName), e); - } - } - - @Override - public List listTables(String databaseName) - throws CatalogException, DatabaseNotExistException { - if (!databaseExists(databaseName)) { - throw new DatabaseNotExistException(this.catalogName, databaseName); - } - - String dbUrl = getUrlFromDatabaseName(databaseName); - try (Connection conn = DriverManager.getConnection(dbUrl, username, pwd); - PreparedStatement ps = - conn.prepareStatement( - "SELECT TABLE_SCHEMA, TABLE_NAME FROM " - + databaseName - + ".INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE = 'BASE TABLE'")) { - - ResultSet rs = ps.executeQuery(); - - List tables = new ArrayList<>(); - - while (rs.next()) { - tables.add(rs.getString(1) + "." + rs.getString(2)); - } - - return tables; - } catch (Exception e) { - throw new CatalogException( - String.format("Failed listing database in catalog %s", catalogName), e); - } + protected String getListDatabaseSql() { + return "SELECT NAME FROM sys.databases"; } @Override - public boolean tableExists(TablePath tablePath) throws CatalogException { - try { - return databaseExists(tablePath.getDatabaseName()) - && listTables(tablePath.getDatabaseName()) - .contains(tablePath.getSchemaAndTableName()); - } catch (DatabaseNotExistException e) { - return false; - } + protected String getListTableSql(String databaseName) { + return "SELECT TABLE_SCHEMA, TABLE_NAME FROM " + + databaseName + + ".INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE = 'BASE TABLE'"; } @Override - public CatalogTable getTable(TablePath tablePath) - throws CatalogException, TableNotExistException { - if (!tableExists(tablePath)) { - throw new TableNotExistException(catalogName, tablePath); - } + protected String getSelectColumnsSql(TablePath tablePath) { String tableSql = StringUtils.isNotEmpty(tablePath.getTableName()) ? "AND tbl.name = '" + tablePath.getTableName() + "'" : ""; - String columnSql = - String.format( - " SELECT tbl.name AS table_name, \n col.name AS column_name, \n ext.value AS comment, \n col.column_id AS column_id, \n types.name AS type, \n col.max_length AS max_length, \n col.precision AS precision, \n col.scale AS scale, \n col.is_nullable AS is_nullable, \n def.definition AS default_value\n FROM sys.tables tbl \nINNER JOIN sys.columns col \n ON tbl.object_id = col.object_id \n LEFT JOIN sys.types types \n ON col.user_type_id = types.user_type_id \n LEFT JOIN sys.extended_properties ext \n ON ext.major_id = col.object_id and ext.minor_id = col.column_id \n LEFT JOIN sys.default_constraints def ON col.default_object_id = def.object_id \n AND ext.minor_id = col.column_id \n AND ext.name = 'MS_Description' \n WHERE schema_name(tbl.schema_id) = '%s' \n %s \n ORDER BY tbl.name, col.column_id", - tablePath.getSchemaName(), tableSql); - - String dbUrl = getUrlFromDatabaseName(tablePath.getDatabaseName()); - try (Connection conn = DriverManager.getConnection(dbUrl, username, pwd)) { - DatabaseMetaData metaData = conn.getMetaData(); - Optional primaryKey = - getPrimaryKey( - metaData, - tablePath.getDatabaseName(), - tablePath.getSchemaName(), - tablePath.getTableName()); - List constraintKeys = - getConstraintKeys( - metaData, - tablePath.getDatabaseName(), - tablePath.getSchemaName(), - tablePath.getTableName()); - - try (PreparedStatement ps = conn.prepareStatement(columnSql); - ResultSet resultSet = ps.executeQuery(); ) { - TableSchema.Builder builder = TableSchema.builder(); - while (resultSet.next()) { - buildTable(resultSet, builder); - } - - // add primary key - primaryKey.ifPresent(builder::primaryKey); - // add constraint key - constraintKeys.forEach(builder::constraintKey); - TableIdentifier tableIdentifier = - TableIdentifier.of( - catalogName, - tablePath.getDatabaseName(), - tablePath.getSchemaName(), - tablePath.getTableName()); - return CatalogTable.of( - tableIdentifier, - builder.build(), - buildConnectorOptions(tablePath), - Collections.emptyList(), - "", - "sqlserver"); - } - - } catch (Exception e) { - throw new CatalogException( - String.format("Failed getting table %s", tablePath.getFullName()), e); - } + return String.format(SELECT_COLUMNS_SQL_TEMPLATE, tablePath.getSchemaName(), tableSql); } - private void buildTable(ResultSet resultSet, TableSchema.Builder builder) throws SQLException { + @Override + protected Column buildColumn(ResultSet resultSet) throws SQLException { String columnName = resultSet.getString("column_name"); String sourceType = resultSet.getString("type"); // String typeName = resultSet.getString("DATA_TYPE").toUpperCase(); @@ -266,21 +159,19 @@ private void buildTable(ResultSet resultSet, TableSchema.Builder builder) throws default: break; } - PhysicalColumn physicalColumn = - PhysicalColumn.of( - columnName, - type, - 0, - isNullable, - defaultValue, - comment, - sourceType, - false, - false, - bitLen, - null, - columnLength); - builder.column(physicalColumn); + return PhysicalColumn.of( + columnName, + type, + 0, + isNullable, + defaultValue, + comment, + sourceType, + false, + false, + bitLen, + null, + columnLength); } private SeaTunnelDataType fromJdbcType(String typeName, int precision, int scale) { @@ -288,103 +179,37 @@ private SeaTunnelDataType fromJdbcType(String typeName, int precision, int sc Map dataTypeProperties = new HashMap<>(); dataTypeProperties.put(SqlServerDataTypeConvertor.PRECISION, precision); dataTypeProperties.put(SqlServerDataTypeConvertor.SCALE, scale); - return new SqlServerDataTypeConvertor().toSeaTunnelType(pair.getLeft(), dataTypeProperties); + return DATA_TYPE_CONVERTOR.toSeaTunnelType(pair.getLeft(), dataTypeProperties); } @Override - protected boolean createTableInternal(TablePath tablePath, CatalogTable table) - throws CatalogException { - - String createTableSql = - SqlServerCreateTableSqlBuilder.builder(tablePath, table).build(tablePath, table); - log.info("create table sql: {}", createTableSql); - try (Connection conn = DriverManager.getConnection(defaultUrl, username, pwd); - PreparedStatement ps = conn.prepareStatement(createTableSql)) { - System.out.println(createTableSql); - return ps.execute(); - } catch (Exception e) { - throw new CatalogException( - String.format("Failed creating table %s", tablePath.getFullName()), e); - } + protected String getCreateTableSql(TablePath tablePath, CatalogTable table) { + return SqlServerCreateTableSqlBuilder.builder(tablePath, table).build(tablePath, table); } @Override - protected boolean dropTableInternal(TablePath tablePath) throws CatalogException { - String dbUrl = getUrlFromDatabaseName(tablePath.getDatabaseName()); - try (Connection conn = DriverManager.getConnection(dbUrl, username, pwd); - PreparedStatement ps = - conn.prepareStatement( - String.format( - "DROP TABLE IF EXISTS %s", tablePath.getFullName()))) { - // Will there exist concurrent drop for one table? - return ps.execute(); - } catch (SQLException e) { - throw new CatalogException( - String.format("Failed dropping table %s", tablePath.getFullName()), e); - } + protected String getDropTableSql(TablePath tablePath) { + return String.format("DROP TABLE %s", tablePath.getFullName()); } @Override - protected boolean createDatabaseInternal(String databaseName) throws CatalogException { - try (Connection conn = DriverManager.getConnection(defaultUrl, username, pwd); - PreparedStatement ps = - conn.prepareStatement( - String.format("CREATE DATABASE `%s`", databaseName))) { - return ps.execute(); - } catch (Exception e) { - throw new CatalogException( - String.format( - "Failed creating database %s in catalog %s", - databaseName, this.catalogName), - e); - } + protected String getCreateDatabaseSql(String databaseName) { + return String.format("CREATE DATABASE %s", databaseName); } @Override - protected boolean dropDatabaseInternal(String databaseName) throws CatalogException { - try (Connection conn = DriverManager.getConnection(defaultUrl, username, pwd); - PreparedStatement ps = - conn.prepareStatement( - String.format("DROP DATABASE IF EXISTS `%s`;", databaseName))) { - return ps.execute(); - } catch (Exception e) { - throw new CatalogException( - String.format( - "Failed dropping database %s in catalog %s", - databaseName, this.catalogName), - e); - } + protected String getDropDatabaseSql(String databaseName) { + return String.format("DROP DATABASE %s;", databaseName); } - @SuppressWarnings("unchecked") - private SeaTunnelDataType fromJdbcType(ResultSetMetaData metadata, int colIndex) - throws SQLException { - Pair> pair = - SqlServerType.parse(metadata.getColumnTypeName(colIndex)); - Map dataTypeProperties = new HashMap<>(); - dataTypeProperties.put( - SqlServerDataTypeConvertor.PRECISION, metadata.getPrecision(colIndex)); - dataTypeProperties.put(SqlServerDataTypeConvertor.SCALE, metadata.getScale(colIndex)); - return new SqlServerDataTypeConvertor().toSeaTunnelType(pair.getLeft(), dataTypeProperties); - } - - @SuppressWarnings("MagicNumber") - private Map buildConnectorOptions(TablePath tablePath) { - Map options = new HashMap<>(8); - options.put("connector", "jdbc"); - options.put("url", getUrlFromDatabaseName(tablePath.getDatabaseName())); - options.put("table-name", tablePath.getFullName()); - options.put("username", username); - options.put("password", pwd); - return options; + @Override + protected void dropDatabaseInternal(String databaseName) throws CatalogException { + closeDatabaseConnection(databaseName); + super.dropDatabaseInternal(databaseName); } - private String getUrlFromDatabaseName(String databaseName) { + @Override + protected String getUrlFromDatabaseName(String databaseName) { return baseUrl + ";databaseName=" + databaseName + ";" + suffix; } - - private String getCreateTableSql(TablePath tablePath, CatalogTable table) { - - return ""; - } } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/oracle/OracleCatalogTest.java b/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/oracle/OracleCatalogTest.java index 6b8c49bc0ab..1c5fb5a2b22 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/oracle/OracleCatalogTest.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/oracle/OracleCatalogTest.java @@ -19,8 +19,6 @@ import org.apache.seatunnel.api.table.catalog.CatalogTable; import org.apache.seatunnel.api.table.catalog.TablePath; -import org.apache.seatunnel.common.utils.JdbcUrlUtil; -import org.apache.seatunnel.connectors.seatunnel.jdbc.catalog.mysql.MySqlCatalog; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; @@ -41,25 +39,10 @@ void testCatalog() { catalog.open(); - MySqlCatalog mySqlCatalog = - new MySqlCatalog( - "mysql", - "root", - "root@123", - JdbcUrlUtil.getUrlInfo("jdbc:mysql://127.0.0.1:33062/mingdongtest")); - - mySqlCatalog.open(); - - CatalogTable table1 = - mySqlCatalog.getTable(TablePath.of("mingdongtest", "all_types_table_02")); - List strings = catalog.listDatabases(); - System.out.println(strings); - - List strings1 = catalog.listTables("XE"); CatalogTable table = catalog.getTable(TablePath.of("XE", "TEST", "PG_TYPES_TABLE_CP1")); - catalog.createTableInternal(new TablePath("XE", "TEST", "TEST003"), table); + catalog.createTable(new TablePath("XE", "TEST", "TEST003"), table, false); } } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/psql/PostgresCatalogTest.java b/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/psql/PostgresCatalogTest.java index badab864fc3..6ef4d9e6548 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/psql/PostgresCatalogTest.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/psql/PostgresCatalogTest.java @@ -53,7 +53,7 @@ void testCatalog() { catalog.getTable(TablePath.of("st_test", "public", "all_types_table_02")); System.out.println("find table: " + table); - catalog.createTableInternal( - new TablePath("liulitest", "public", "all_types_table_02"), table); + catalog.createTable( + new TablePath("liulitest", "public", "all_types_table_02"), table, false); } } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/sql/MysqlCreateTableSqlBuilderTest.java b/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/sql/MysqlCreateTableSqlBuilderTest.java index 3de5c65bf8d..b75ac68223b 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/sql/MysqlCreateTableSqlBuilderTest.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/catalog/sql/MysqlCreateTableSqlBuilderTest.java @@ -93,7 +93,7 @@ public void testBuild() { MysqlCreateTableSqlBuilder.builder(tablePath, catalogTable).build("mysql"); // create table sql is change; The old unit tests are no longer applicable String expect = - "CREATE TABLE IF NOT EXISTS test_table (\n" + "CREATE TABLE test_table (\n" + "\tid null NOT NULL COMMENT 'id', \n" + "\tname null NOT NULL COMMENT 'name', \n" + "\tage null NULL COMMENT 'age', \n" diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-common/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/AbstractJdbcIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-common/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/AbstractJdbcIT.java index 6528be0e1fc..a38fb2217f2 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-common/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/AbstractJdbcIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-common/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/AbstractJdbcIT.java @@ -17,6 +17,9 @@ package org.apache.seatunnel.connectors.seatunnel.jdbc; +import org.apache.seatunnel.api.table.catalog.Catalog; +import org.apache.seatunnel.api.table.catalog.CatalogTable; +import org.apache.seatunnel.api.table.catalog.TablePath; import org.apache.seatunnel.api.table.type.SeaTunnelRow; import org.apache.seatunnel.common.exception.SeaTunnelRuntimeException; import org.apache.seatunnel.common.utils.ExceptionUtils; @@ -31,6 +34,7 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestTemplate; import org.testcontainers.containers.Container; import org.testcontainers.containers.GenericContainer; @@ -76,6 +80,7 @@ public abstract class AbstractJdbcIT extends TestSuiteBase implements TestResour protected GenericContainer dbServer; protected JdbcCase jdbcCase; protected Connection connection; + protected Catalog catalog; abstract JdbcCase getJdbcCase(); @@ -141,12 +146,16 @@ protected void createNeededTables() { String.format( createTemplate, buildTableInfoWithSchema( - jdbcCase.getDatabase(), jdbcCase.getSourceTable())); + jdbcCase.getDatabase(), + jdbcCase.getSchema(), + jdbcCase.getSourceTable())); String createSink = String.format( createTemplate, buildTableInfoWithSchema( - jdbcCase.getDatabase(), jdbcCase.getSinkTable())); + jdbcCase.getDatabase(), + jdbcCase.getSchema(), + jdbcCase.getSinkTable())); statement.execute(createSource); statement.execute(createSink); @@ -173,6 +182,14 @@ public String insertTable(String schema, String table, String... fields) { + ")"; } + protected void clearTable(String database, String schema, String table) { + clearTable(database, table); + } + + protected String buildTableInfoWithSchema(String database, String schema, String table) { + return buildTableInfoWithSchema(database, table); + } + public void clearTable(String schema, String table) { try (Statement statement = connection.createStatement()) { statement.execute("TRUNCATE TABLE " + buildTableInfoWithSchema(schema, table)); @@ -215,6 +232,7 @@ public void startUp() { createSchemaIfNeeded(); createNeededTables(); insertTestData(); + initCatalog(); } @Override @@ -226,6 +244,10 @@ public void tearDown() throws SQLException { if (connection != null) { connection.close(); } + + if (catalog != null) { + catalog.close(); + } } @TestTemplate @@ -238,6 +260,43 @@ public void testJdbcDb(TestContainer container) } compareResult(); - clearTable(jdbcCase.getDatabase(), jdbcCase.getSinkTable()); + clearTable(jdbcCase.getDatabase(), jdbcCase.getSchema(), jdbcCase.getSinkTable()); + } + + protected void initCatalog() {} + + @Test + public void testCatalog() { + if (catalog == null) { + return; + } + + TablePath sourceTablePath = + new TablePath( + jdbcCase.getDatabase(), jdbcCase.getSchema(), jdbcCase.getSourceTable()); + TablePath targetTablePath = + new TablePath( + jdbcCase.getCatalogDatabase(), + jdbcCase.getCatalogSchema(), + jdbcCase.getCatalogTable()); + boolean createdDb = false; + + if (!catalog.databaseExists(targetTablePath.getDatabaseName())) { + catalog.createDatabase(targetTablePath, false); + Assertions.assertTrue(catalog.databaseExists(targetTablePath.getDatabaseName())); + createdDb = true; + } + + CatalogTable catalogTable = catalog.getTable(sourceTablePath); + catalog.createTable(targetTablePath, catalogTable, false); + Assertions.assertTrue(catalog.tableExists(targetTablePath)); + + catalog.dropTable(targetTablePath, false); + Assertions.assertFalse(catalog.tableExists(targetTablePath)); + + if (createdDb) { + catalog.dropDatabase(targetTablePath, false); + Assertions.assertFalse(catalog.databaseExists(targetTablePath.getDatabaseName())); + } } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-common/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcCase.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-common/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcCase.java index 805fcbd16bb..5f17eacc51a 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-common/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcCase.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-common/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcCase.java @@ -41,6 +41,7 @@ public class JdbcCase { private int port; private int localPort; private String database; + private String schema; private String sourceTable; private String sinkTable; private String jdbcTemplate; @@ -50,4 +51,8 @@ public class JdbcCase { private List configFile; private Pair> testData; private Map containerEnv; + + private String catalogDatabase; + private String catalogSchema; + private String catalogTable; } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcMysqlIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcMysqlIT.java index f4b1338b15b..b10aa0c2225 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcMysqlIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcMysqlIT.java @@ -19,6 +19,8 @@ package org.apache.seatunnel.connectors.seatunnel.jdbc; import org.apache.seatunnel.api.table.type.SeaTunnelRow; +import org.apache.seatunnel.common.utils.JdbcUrlUtil; +import org.apache.seatunnel.connectors.seatunnel.jdbc.catalog.mysql.MySqlCatalog; import org.apache.commons.lang3.tuple.Pair; @@ -48,6 +50,7 @@ public class JdbcMysqlIT extends AbstractJdbcIT { private static final String MYSQL_DATABASE = "seatunnel"; private static final String MYSQL_SOURCE = "source"; private static final String MYSQL_SINK = "sink"; + private static final String CATALOG_DATABASE = "catalog_database"; private static final String MYSQL_USERNAME = "root"; private static final String MYSQL_PASSWORD = "Abc!@#135_seatunnel"; @@ -138,6 +141,8 @@ JdbcCase getJdbcCase() { .configFile(CONFIG_FILE) .insertSql(insertSql) .testData(testDataSet) + .catalogDatabase(CATALOG_DATABASE) + .catalogTable(MYSQL_SINK) .build(); } @@ -282,4 +287,16 @@ protected GenericContainer initContainer() { return container; } + + @Override + protected void initCatalog() { + catalog = + new MySqlCatalog( + "mysql", + jdbcCase.getUserName(), + jdbcCase.getPassword(), + JdbcUrlUtil.getUrlInfo( + jdbcCase.getJdbcUrl().replace(HOST, dbServer.getHost()))); + catalog.open(); + } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOracleIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOracleIT.java index d0f8ce3b687..75bdffbd6ca 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOracleIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcOracleIT.java @@ -19,6 +19,8 @@ package org.apache.seatunnel.connectors.seatunnel.jdbc; import org.apache.seatunnel.api.table.type.SeaTunnelRow; +import org.apache.seatunnel.connectors.seatunnel.jdbc.catalog.oracle.OracleCatalog; +import org.apache.seatunnel.connectors.seatunnel.jdbc.catalog.oracle.OracleURLParser; import org.apache.commons.lang3.tuple.Pair; @@ -27,6 +29,7 @@ import org.testcontainers.containers.output.Slf4jLogConsumer; import org.testcontainers.utility.DockerImageName; import org.testcontainers.utility.DockerLoggerFactory; +import org.testcontainers.utility.MountableFile; import com.google.common.collect.Lists; @@ -47,11 +50,13 @@ public class JdbcOracleIT extends AbstractJdbcIT { private static final String DRIVER_CLASS = "oracle.jdbc.OracleDriver"; private static final int ORACLE_PORT = 1521; private static final String ORACLE_URL = "jdbc:oracle:thin:@" + HOST + ":%s/%s"; - private static final String USERNAME = "testUser"; + private static final String USERNAME = "TESTUSER"; private static final String PASSWORD = "testPassword"; - private static final String DATABASE = "TESTUSER"; + private static final String DATABASE = "XE"; + private static final String SCHEMA = USERNAME; private static final String SOURCE_TABLE = "E2E_TABLE_SOURCE"; private static final String SINK_TABLE = "E2E_TABLE_SINK"; + private static final String CATALOG_TABLE = "E2E_TABLE_CATALOG"; private static final List CONFIG_FILE = Lists.newArrayList("/jdbc_oracle_source_to_sink.conf"); @@ -78,11 +83,11 @@ JdbcCase getJdbcCase() { containerEnv.put("ORACLE_PASSWORD", PASSWORD); containerEnv.put("APP_USER", USERNAME); containerEnv.put("APP_USER_PASSWORD", PASSWORD); - String jdbcUrl = String.format(ORACLE_URL, ORACLE_PORT, DATABASE); + String jdbcUrl = String.format(ORACLE_URL, ORACLE_PORT, SCHEMA); Pair> testDataSet = initTestData(); String[] fieldNames = testDataSet.getKey(); - String insertSql = insertTable(DATABASE, SOURCE_TABLE, fieldNames); + String insertSql = insertTable(SCHEMA, SOURCE_TABLE, fieldNames); return JdbcCase.builder() .dockerImage(ORACLE_IMAGE) @@ -97,8 +102,12 @@ JdbcCase getJdbcCase() { .userName(USERNAME) .password(PASSWORD) .database(DATABASE) + .schema(SCHEMA) .sourceTable(SOURCE_TABLE) .sinkTable(SINK_TABLE) + .catalogDatabase(DATABASE) + .catalogSchema(SCHEMA) + .catalogTable(CATALOG_TABLE) .createSql(CREATE_SQL) .configFile(CONFIG_FILE) .insertSql(insertSql) @@ -162,9 +171,10 @@ GenericContainer initContainer() { GenericContainer container = new OracleContainer(imageName) - .withDatabaseName(DATABASE) - .withUsername(USERNAME) - .withPassword(PASSWORD) + .withDatabaseName(SCHEMA) + .withCopyFileToContainer( + MountableFile.forClasspathResource("sql/oracle_init.sql"), + "/container-entrypoint-startdb.d/init.sql") .withNetwork(NETWORK) .withNetworkAliases(ORACLE_NETWORK_ALIASES) .withExposedPorts(ORACLE_PORT) @@ -181,4 +191,27 @@ GenericContainer initContainer() { public String quoteIdentifier(String field) { return "\"" + field + "\""; } + + @Override + protected void clearTable(String database, String schema, String table) { + clearTable(schema, table); + } + + @Override + protected String buildTableInfoWithSchema(String database, String schema, String table) { + return buildTableInfoWithSchema(schema, table); + } + + @Override + protected void initCatalog() { + String jdbcUrl = jdbcCase.getJdbcUrl().replace(HOST, dbServer.getHost()); + catalog = + new OracleCatalog( + "oracle", + jdbcCase.getUserName(), + jdbcCase.getPassword(), + OracleURLParser.parse(jdbcUrl), + SCHEMA); + catalog.open(); + } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/sql/oracle_init.sql b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/sql/oracle_init.sql new file mode 100644 index 00000000000..ba77de271ea --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/sql/oracle_init.sql @@ -0,0 +1,22 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +ALTER SESSION SET CONTAINER = TESTUSER; + +CREATE USER TESTUSER IDENTIFIED BY testPassword; + +GRANT DBA TO TESTUSER; \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-3/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcPostgresIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-3/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcPostgresIT.java index f66ef615d7b..a5796c1aaac 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-3/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcPostgresIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-3/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcPostgresIT.java @@ -17,6 +17,11 @@ package org.apache.seatunnel.connectors.seatunnel.jdbc; +import org.apache.seatunnel.api.table.catalog.Catalog; +import org.apache.seatunnel.api.table.catalog.CatalogTable; +import org.apache.seatunnel.api.table.catalog.TablePath; +import org.apache.seatunnel.common.utils.JdbcUrlUtil; +import org.apache.seatunnel.connectors.seatunnel.jdbc.catalog.psql.PostgresCatalog; import org.apache.seatunnel.e2e.common.TestResource; import org.apache.seatunnel.e2e.common.TestSuiteBase; import org.apache.seatunnel.e2e.common.container.ContainerExtendedFactory; @@ -26,6 +31,7 @@ import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestTemplate; import org.testcontainers.containers.Container; import org.testcontainers.containers.PostgreSQLContainer; @@ -252,6 +258,43 @@ public void testAutoGenerateSQL(TestContainer container) } } + @Test + public void testCatalog() { + String schema = "public"; + String databaseName = POSTGRESQL_CONTAINER.getDatabaseName(); + String tableName = "pg_e2e_sink_table"; + String catalogDatabaseName = "pg_e2e_catalog_database"; + String catalogTableName = "pg_e2e_catalog_table"; + + Catalog catalog = + new PostgresCatalog( + "postgres", + POSTGRESQL_CONTAINER.getUsername(), + POSTGRESQL_CONTAINER.getPassword(), + JdbcUrlUtil.getUrlInfo(POSTGRESQL_CONTAINER.getJdbcUrl()), + schema); + catalog.open(); + + TablePath tablePath = new TablePath(databaseName, schema, tableName); + TablePath catalogTablePath = new TablePath(catalogDatabaseName, schema, catalogTableName); + + Assertions.assertFalse(catalog.databaseExists(catalogTablePath.getDatabaseName())); + catalog.createDatabase(catalogTablePath, false); + Assertions.assertTrue(catalog.databaseExists(catalogTablePath.getDatabaseName())); + + CatalogTable catalogTable = catalog.getTable(tablePath); + catalog.createTable(catalogTablePath, catalogTable, false); + Assertions.assertTrue(catalog.tableExists(catalogTablePath)); + + catalog.dropTable(catalogTablePath, false); + Assertions.assertFalse(catalog.tableExists(catalogTablePath)); + + catalog.dropDatabase(catalogTablePath, false); + Assertions.assertFalse(catalog.databaseExists(catalogTablePath.getDatabaseName())); + + catalog.close(); + } + private void initializeJdbcTable() { try (Connection connection = getJdbcConnection()) { Statement statement = connection.createStatement(); diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-3/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcSqlServerIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-3/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcSqlServerIT.java index f615b6656ea..0a170ff4bed 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-3/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcSqlServerIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-3/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/JdbcSqlServerIT.java @@ -19,6 +19,8 @@ import org.apache.seatunnel.api.table.type.SeaTunnelRow; import org.apache.seatunnel.common.exception.SeaTunnelRuntimeException; +import org.apache.seatunnel.connectors.seatunnel.jdbc.catalog.sqlserver.SqlServerCatalog; +import org.apache.seatunnel.connectors.seatunnel.jdbc.catalog.sqlserver.SqlServerURLParser; import org.apache.seatunnel.e2e.common.TestSuiteBase; import org.apache.commons.lang3.tuple.Pair; @@ -44,9 +46,16 @@ public class JdbcSqlServerIT extends AbstractJdbcIT { private static final String SQLSERVER_CONTAINER_HOST = "sqlserver"; private static final String SQLSERVER_SOURCE = "source"; private static final String SQLSERVER_SINK = "sink"; + private static final String SQLSERVER_DATABASE = "master"; + private static final String SQLSERVER_SCHEMA = "dbo"; + private static final String SQLSERVER_CATALOG_DATABASE = "catalog_test"; + private static final int SQLSERVER_CONTAINER_PORT = 1433; private static final String SQLSERVER_URL = - "jdbc:sqlserver://" + AbstractJdbcIT.HOST + ":%s;encrypt=false;"; + "jdbc:sqlserver://" + + AbstractJdbcIT.HOST + + ":%s;encrypt=false;databaseName=" + + SQLSERVER_DATABASE; private static final String DRIVER_CLASS = "com.microsoft.sqlserver.jdbc.SQLServerDriver"; private static final List CONFIG_FILE = Lists.newArrayList("/jdbc_sqlserver_source_to_sink.conf"); @@ -81,8 +90,13 @@ JdbcCase getJdbcCase() { .jdbcUrl(jdbcUrl) .userName(username) .password(password) + .database(SQLSERVER_DATABASE) + .schema(SQLSERVER_SCHEMA) .sourceTable(SQLSERVER_SOURCE) .sinkTable(SQLSERVER_SINK) + .catalogDatabase(SQLSERVER_CATALOG_DATABASE) + .catalogSchema(SQLSERVER_SCHEMA) + .catalogTable(SQLSERVER_SINK) .createSql(CREATE_SQL) .configFile(CONFIG_FILE) .insertSql(insertSql) @@ -158,4 +172,22 @@ public String quoteIdentifier(String field) { public void clearTable(String schema, String table) { // do nothing. } + + @Override + protected String buildTableInfoWithSchema(String database, String schema, String table) { + return buildTableInfoWithSchema(schema, table); + } + + @Override + protected void initCatalog() { + catalog = + new SqlServerCatalog( + "sqlserver", + jdbcCase.getUserName(), + jdbcCase.getPassword(), + SqlServerURLParser.parse( + jdbcCase.getJdbcUrl().replace(HOST, dbServer.getHost())), + SQLSERVER_SCHEMA); + catalog.open(); + } } From d1e9673f76d3692beee3bef0adef338dd521d822 Mon Sep 17 00:00:00 2001 From: Jia Fan Date: Wed, 6 Sep 2023 14:42:42 +0800 Subject: [PATCH 07/15] [Fix][Zeta] Fix Zeta will close task twice error (#5422) --- .../engine/server/TaskExecutionService.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/TaskExecutionService.java b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/TaskExecutionService.java index 2725836f875..009225347c5 100644 --- a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/TaskExecutionService.java +++ b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/TaskExecutionService.java @@ -603,10 +603,10 @@ public void run() { ClassLoader oldClassLoader = Thread.currentThread().getContextClassLoader(); Thread.currentThread().setContextClassLoader(classLoader); final Task t = tracker.task; + ProgressState result = null; try { startedLatch.countDown(); t.init(); - ProgressState result; do { result = t.call(); } while (!result.isDone() @@ -623,10 +623,12 @@ public void run() { taskGroupExecutionTracker.exception(e); } finally { taskGroupExecutionTracker.taskDone(t); - try { - tracker.task.close(); - } catch (IOException e) { - logger.severe("Close task error", e); + if (result == null || !result.isDone()) { + try { + tracker.task.close(); + } catch (IOException e) { + logger.severe("Close task error", e); + } } } Thread.currentThread().setContextClassLoader(oldClassLoader); From 8a3f261b0c895a4b78523cad321d1dc3f2a84753 Mon Sep 17 00:00:00 2001 From: ic4y <83933160+ic4y@users.noreply.github.com> Date: Thu, 7 Sep 2023 17:50:48 +0800 Subject: [PATCH 08/15] [bugfix][zeta] Disable CheckpointTimeOutTest (#5438) --- .../engine/server/checkpoint/CheckpointTimeOutTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/checkpoint/CheckpointTimeOutTest.java b/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/checkpoint/CheckpointTimeOutTest.java index 3cf7636adf8..bb9c0149025 100644 --- a/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/checkpoint/CheckpointTimeOutTest.java +++ b/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/checkpoint/CheckpointTimeOutTest.java @@ -25,6 +25,7 @@ import org.apache.seatunnel.engine.server.TestUtils; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import com.hazelcast.internal.serialization.Data; @@ -42,6 +43,7 @@ public class CheckpointTimeOutTest extends AbstractSeaTunnelServerTest { public static long JOB_ID = System.currentTimeMillis(); @Test + @Disabled("Currently unstable tests, waiting for @EricJoy2048 to refactor state handling logic") public void testJobLevelCheckpointTimeOut() { startJob(JOB_ID, CONF_PATH); From ee4040bbd448c76f7113e36f14fea97b1c443fb1 Mon Sep 17 00:00:00 2001 From: FlechazoW <35768015+FlechazoW@users.noreply.github.com> Date: Mon, 11 Sep 2023 11:04:58 +0800 Subject: [PATCH 09/15] [Feature-#5258][Bin] Add .bat script for all shell script. (#5445) --- .gitignore | 3 +- bin/install-plugin.cmd | 60 ++++++++++ config/seatunnel-env.cmd | 21 ++++ .../start-seatunnel-flink-13-connector-v2.cmd | 71 ++++++++++++ .../start-seatunnel-flink-15-connector-v2.cmd | 71 ++++++++++++ .../start-seatunnel-spark-2-connector-v2.cmd | 71 ++++++++++++ .../start-seatunnel-spark-3-connector-v2.cmd | 71 ++++++++++++ .../src/main/bin/seatunnel-cluster.cmd | 86 ++++++++++++++ .../src/main/bin/seatunnel-cluster.sh | 2 +- .../src/main/bin/seatunnel.cmd | 108 ++++++++++++++++++ .../src/main/bin/seatunnel.sh | 6 + .../src/main/bin/stop-seatunnel-cluster.cmd | 58 ++++++++++ 12 files changed, 626 insertions(+), 2 deletions(-) create mode 100644 bin/install-plugin.cmd create mode 100644 config/seatunnel-env.cmd create mode 100644 seatunnel-core/seatunnel-flink-starter/seatunnel-flink-13-starter/src/main/bin/start-seatunnel-flink-13-connector-v2.cmd create mode 100644 seatunnel-core/seatunnel-flink-starter/seatunnel-flink-15-starter/src/main/bin/start-seatunnel-flink-15-connector-v2.cmd create mode 100644 seatunnel-core/seatunnel-spark-starter/seatunnel-spark-2-starter/src/main/bin/start-seatunnel-spark-2-connector-v2.cmd create mode 100644 seatunnel-core/seatunnel-spark-starter/seatunnel-spark-3-starter/src/main/bin/start-seatunnel-spark-3-connector-v2.cmd create mode 100644 seatunnel-core/seatunnel-starter/src/main/bin/seatunnel-cluster.cmd create mode 100644 seatunnel-core/seatunnel-starter/src/main/bin/seatunnel.cmd create mode 100644 seatunnel-core/seatunnel-starter/src/main/bin/stop-seatunnel-cluster.cmd diff --git a/.gitignore b/.gitignore index 25977068e4f..74311a0fa05 100644 --- a/.gitignore +++ b/.gitignore @@ -48,4 +48,5 @@ test.conf spark-warehouse *.flattened-pom.xml -seatunnel-examples \ No newline at end of file +seatunnel-examples +/lib/* \ No newline at end of file diff --git a/bin/install-plugin.cmd b/bin/install-plugin.cmd new file mode 100644 index 00000000000..4df77b968ca --- /dev/null +++ b/bin/install-plugin.cmd @@ -0,0 +1,60 @@ +@echo off +REM Licensed to the Apache Software Foundation (ASF) under one or more +REM contributor license agreements. See the NOTICE file distributed with +REM this work for additional information regarding copyright ownership. +REM The ASF licenses this file to You under the Apache License, Version 2.0 +REM (the "License"); you may not use this file except in compliance with +REM the License. You may obtain a copy of the License at +REM +REM http://www.apache.org/licenses/LICENSE-2.0 +REM +REM Unless required by applicable law or agreed to in writing, software +REM distributed under the License is distributed on an "AS IS" BASIS, +REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +REM See the License for the specific language governing permissions and +REM limitations under the License. + +REM This script is used to download the connector plug-ins required during the running process. +REM All are downloaded by default. You can also choose what you need. +REM You only need to configure the plug-in name in config\plugin_config.txt. + +REM Get seatunnel home +set "SEATUNNEL_HOME=%~dp0..\" +echo Set SEATUNNEL_HOME to [%SEATUNNEL_HOME%] + +REM Connector default version is 2.3.3, you can also choose a custom version. eg: 2.1.2: install-plugin.bat 2.1.2 +set "version=2.3.3" +if not "%~1"=="" set "version=%~1" +echo Install hadoop shade jar, usage version is %version% + +REM Create the lib directory +if not exist "%SEATUNNEL_HOME%\lib" ( + mkdir "%SEATUNNEL_HOME%\lib" + echo create lib directory +) + +call "%SEATUNNEL_HOME%\mvnw.cmd" dependency:get -DgroupId="org.apache.seatunnel" -Dclassifier="optional" -DartifactId="seatunnel-hadoop3-3.1.4-uber" -Dversion="%version%" -Ddest="%SEATUNNEL_HOME%\lib" + +echo Install SeaTunnel connectors plugins, usage version is %version% + +REM Create the connectors directory +if not exist "%SEATUNNEL_HOME%\connectors" ( + mkdir "%SEATUNNEL_HOME%\connectors" + echo create connectors directory +) + +REM Create the seatunnel connectors directory (for v2) +if not exist "%SEATUNNEL_HOME%\connectors\seatunnel" ( + mkdir "%SEATUNNEL_HOME%\connectors\seatunnel" + echo create seatunnel connectors directory +) + +for /f "usebackq delims=" %%a in ("%SEATUNNEL_HOME%\config\plugin_config") do ( + set "line=%%a" + setlocal enabledelayedexpansion + if "!line:~0,1!" neq "-" if "!line:~0,1!" neq "#" ( + echo install connector : !line! + call "%SEATUNNEL_HOME%\mvnw.cmd" dependency:get -DgroupId="org.apache.seatunnel" -DartifactId="!line!" -Dversion="%version%" -Ddest="%SEATUNNEL_HOME%\connectors\seatunnel" + ) + endlocal +) diff --git a/config/seatunnel-env.cmd b/config/seatunnel-env.cmd new file mode 100644 index 00000000000..79c2d3c117c --- /dev/null +++ b/config/seatunnel-env.cmd @@ -0,0 +1,21 @@ +@echo off +REM Licensed to the Apache Software Foundation (ASF) under one or more +REM contributor license agreements. See the NOTICE file distributed with +REM this work for additional information regarding copyright ownership. +REM The ASF licenses this file to You under the Apache License, Version 2.0 +REM (the "License"); you may not use this file except in compliance with +REM the License. You may obtain a copy of the License at +REM +REM http://www.apache.org/licenses/LICENSE-2.0 +REM +REM Unless required by applicable law or agreed to in writing, software +REM distributed under the License is distributed on an "AS IS" BASIS, +REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +REM See the License for the specific language governing permissions and +REM limitations under the License. + +REM Home directory of spark distribution. +if "%SPARK_HOME%" == "" set "SPARK_HOME=C:\Program Files\spark" + +REM Home directory of flink distribution. +if "%FLINK_HOME%" == "" set "FLINK_HOME=C:\Program Files\flink" \ No newline at end of file diff --git a/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-13-starter/src/main/bin/start-seatunnel-flink-13-connector-v2.cmd b/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-13-starter/src/main/bin/start-seatunnel-flink-13-connector-v2.cmd new file mode 100644 index 00000000000..c1cbc1d9556 --- /dev/null +++ b/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-13-starter/src/main/bin/start-seatunnel-flink-13-connector-v2.cmd @@ -0,0 +1,71 @@ +@echo off +rem Licensed to the Apache Software Foundation (ASF) under one or more +rem contributor license agreements. See the NOTICE file distributed with +rem this work for additional information regarding copyright ownership. +rem The ASF licenses this file to You under the Apache License, Version 2.0 +rem (the "License"); you may not use this file except in compliance with +rem the License. You may obtain a copy of the License at +rem +rem http://www.apache.org/licenses/LICENSE-2.0 +rem +rem Unless required by applicable law or agreed to in writing, software +rem distributed under the License is distributed on an "AS IS" BASIS, +rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +rem See the License for the specific language governing permissions and +rem limitations under the License. + +setlocal enabledelayedexpansion + +rem resolve links - %0 may be a softlink +set "PRG=%~f0" +:resolve_loop +rem Get the parent directory of the script +set "PRG_DIR=%~dp0" +rem Change current drive and directory to %PRG_DIR% and execute the 'dir' command, which will fail if %PRG% is not a valid file. +cd /d "%PRG_DIR%" || ( + echo Cannot determine the script's current directory. + exit /b 1 +) + +set "APP_DIR=%~dp0" +set "CONF_DIR=%APP_DIR%\config" +set "APP_JAR=%APP_DIR%\starter\seatunnel-flink-13-starter.jar" +set "APP_MAIN=org.apache.seatunnel.core.starter.flink.FlinkStarter" + +if exist "%CONF_DIR%\seatunnel-env.cmd" ( + call "%CONF_DIR%\seatunnel-env.cmd" +) + +if "%~1"=="" ( + set "args=-h" +) else ( + set "args=%*" +) + +set "JAVA_OPTS=" +rem Log4j2 Config +if exist "%CONF_DIR%\log4j2.properties" ( + set "JAVA_OPTS=!JAVA_OPTS! -Dlog4j2.configurationFile=%CONF_DIR%\log4j2.properties" + set "JAVA_OPTS=!JAVA_OPTS! -Dseatunnel.logs.path=%APP_DIR%\logs" + set "JAVA_OPTS=!JAVA_OPTS! -Dseatunnel.logs.file_name=seatunnel-flink-starter" +) + +set "CLASS_PATH=%APP_DIR%\starter\logging\*;%APP_JAR%" + +for /f "delims=" %%i in ('java %JAVA_OPTS% -cp %CLASS_PATH% %APP_MAIN% %args%') do ( + set "CMD=%%i" + setlocal disabledelayedexpansion + if !errorlevel! equ 234 ( + echo !CMD! + endlocal + exit /b 0 + ) else if !errorlevel! equ 0 ( + echo Execute SeaTunnel Flink Job: !CMD! + endlocal + call !CMD! + ) else ( + echo !CMD! + endlocal + exit /b !errorlevel! + ) +) diff --git a/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-15-starter/src/main/bin/start-seatunnel-flink-15-connector-v2.cmd b/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-15-starter/src/main/bin/start-seatunnel-flink-15-connector-v2.cmd new file mode 100644 index 00000000000..ed4c1f6979e --- /dev/null +++ b/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-15-starter/src/main/bin/start-seatunnel-flink-15-connector-v2.cmd @@ -0,0 +1,71 @@ +@echo off +rem Licensed to the Apache Software Foundation (ASF) under one or more +rem contributor license agreements. See the NOTICE file distributed with +rem this work for additional information regarding copyright ownership. +rem The ASF licenses this file to You under the Apache License, Version 2.0 +rem (the "License"); you may not use this file except in compliance with +rem the License. You may obtain a copy of the License at +rem +rem http://www.apache.org/licenses/LICENSE-2.0 +rem +rem Unless required by applicable law or agreed to in writing, software +rem distributed under the License is distributed on an "AS IS" BASIS, +rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +rem See the License for the specific language governing permissions and +rem limitations under the License. + +setlocal enabledelayedexpansion + +rem resolve links - %0 may be a softlink +set "PRG=%~f0" +:resolve_loop +rem Get the parent directory of the script +set "PRG_DIR=%~dp0" +rem Change current drive and directory to %PRG_DIR% and execute the 'dir' command, which will fail if %PRG% is not a valid file. +cd /d "%PRG_DIR%" || ( + echo Cannot determine the script's current directory. + exit /b 1 +) + +set "APP_DIR=%~dp0" +set "CONF_DIR=%APP_DIR%\config" +set "APP_JAR=%APP_DIR%\starter\seatunnel-flink-15-starter.jar" +set "APP_MAIN=org.apache.seatunnel.core.starter.flink.FlinkStarter" + +if exist "%CONF_DIR%\seatunnel-env.cmd" ( + call "%CONF_DIR%\seatunnel-env.cmd" +) + +if "%~1"=="" ( + set "args=-h" +) else ( + set "args=%*" +) + +set "JAVA_OPTS=" +rem Log4j2 Config +if exist "%CONF_DIR%\log4j2.properties" ( + set "JAVA_OPTS=!JAVA_OPTS! -Dlog4j2.configurationFile=%CONF_DIR%\log4j2.properties" + set "JAVA_OPTS=!JAVA_OPTS! -Dseatunnel.logs.path=%APP_DIR%\logs" + set "JAVA_OPTS=!JAVA_OPTS! -Dseatunnel.logs.file_name=seatunnel-flink-starter" +) + +set "CLASS_PATH=%APP_DIR%\starter\logging\*;%APP_JAR%" + +for /f "delims=" %%i in ('java %JAVA_OPTS% -cp %CLASS_PATH% %APP_MAIN% %args%') do ( + set "CMD=%%i" + setlocal disabledelayedexpansion + if !errorlevel! equ 234 ( + echo !CMD! + endlocal + exit /b 0 + ) else if !errorlevel! equ 0 ( + echo Execute SeaTunnel Flink Job: !CMD! + endlocal + call !CMD! + ) else ( + echo !CMD! + endlocal + exit /b !errorlevel! + ) +) diff --git a/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-2-starter/src/main/bin/start-seatunnel-spark-2-connector-v2.cmd b/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-2-starter/src/main/bin/start-seatunnel-spark-2-connector-v2.cmd new file mode 100644 index 00000000000..b2671671383 --- /dev/null +++ b/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-2-starter/src/main/bin/start-seatunnel-spark-2-connector-v2.cmd @@ -0,0 +1,71 @@ +@echo off +rem Licensed to the Apache Software Foundation (ASF) under one or more +rem contributor license agreements. See the NOTICE file distributed with +rem this work for additional information regarding copyright ownership. +rem The ASF licenses this file to You under the Apache License, Version 2.0 +rem (the "License"); you may not use this file except in compliance with +rem the License. You may obtain a copy of the License at +rem +rem http://www.apache.org/licenses/LICENSE-2.0 +rem +rem Unless required by applicable law or agreed to in writing, software +rem distributed under the License is distributed on an "AS IS" BASIS, +rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +rem See the License for the specific language governing permissions and +rem limitations under the License. + +setlocal enabledelayedexpansion + +rem resolve links - %0 may be a softlink +set "PRG=%~f0" +:resolve_loop +rem Get the parent directory of the script +set "PRG_DIR=%~dp0" +rem Change current drive and directory to %PRG_DIR% and execute the 'dir' command, which will fail if %PRG% is not a valid file. +cd /d "%PRG_DIR%" || ( + echo Cannot determine the script's current directory. + exit /b 1 +) + +set "APP_DIR=%~dp0" +set "CONF_DIR=%APP_DIR%\config" +set "APP_JAR=%APP_DIR%\starter\seatunnel-spark-2-starter.jar" +set "APP_MAIN=org.apache.seatunnel.core.starter.spark.SparkStarter" + +if exist "%CONF_DIR%\seatunnel-env.cmd" ( + call "%CONF_DIR%\seatunnel-env.cmd" +) + +if "%~1"=="" ( + set "args=-h" +) else ( + set "args=%*" +) + +set "JAVA_OPTS=" +rem Log4j2 Config +if exist "%CONF_DIR%\log4j2.properties" ( + set "JAVA_OPTS=!JAVA_OPTS! -Dlog4j2.configurationFile=%CONF_DIR%\log4j2.properties" + set "JAVA_OPTS=!JAVA_OPTS! -Dseatunnel.logs.path=%APP_DIR%\logs" + set "JAVA_OPTS=!JAVA_OPTS! -Dseatunnel.logs.file_name=seatunnel-spark-starter" +) + +set "CLASS_PATH=%APP_DIR%\starter\logging\*;%APP_JAR%" + +for /f "delims=" %%i in ('java %JAVA_OPTS% -cp %CLASS_PATH% %APP_MAIN% %args%') do ( + set "CMD=%%i" + setlocal disabledelayedexpansion + if !errorlevel! equ 234 ( + echo !CMD! + endlocal + exit /b 0 + ) else if !errorlevel! equ 0 ( + echo Execute SeaTunnel Spark Job: !CMD! + endlocal + call !CMD! + ) else ( + echo !CMD! + endlocal + exit /b !errorlevel! + ) +) diff --git a/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-3-starter/src/main/bin/start-seatunnel-spark-3-connector-v2.cmd b/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-3-starter/src/main/bin/start-seatunnel-spark-3-connector-v2.cmd new file mode 100644 index 00000000000..433fe23c6d1 --- /dev/null +++ b/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-3-starter/src/main/bin/start-seatunnel-spark-3-connector-v2.cmd @@ -0,0 +1,71 @@ +@echo off +rem Licensed to the Apache Software Foundation (ASF) under one or more +rem contributor license agreements. See the NOTICE file distributed with +rem this work for additional information regarding copyright ownership. +rem The ASF licenses this file to You under the Apache License, Version 2.0 +rem (the "License"); you may not use this file except in compliance with +rem the License. You may obtain a copy of the License at +rem +rem http://www.apache.org/licenses/LICENSE-2.0 +rem +rem Unless required by applicable law or agreed to in writing, software +rem distributed under the License is distributed on an "AS IS" BASIS, +rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +rem See the License for the specific language governing permissions and +rem limitations under the License. + +setlocal enabledelayedexpansion + +rem resolve links - %0 may be a softlink +set "PRG=%~f0" +:resolve_loop +rem Get the parent directory of the script +set "PRG_DIR=%~dp0" +rem Change current drive and directory to %PRG_DIR% and execute the 'dir' command, which will fail if %PRG% is not a valid file. +cd /d "%PRG_DIR%" || ( + echo Cannot determine the script's current directory. + exit /b 1 +) + +set "APP_DIR=%~dp0" +set "CONF_DIR=%APP_DIR%\config" +set "APP_JAR=%APP_DIR%\starter\seatunnel-spark-3-starter.jar" +set "APP_MAIN=org.apache.seatunnel.core.starter.spark.SparkStarter" + +if exist "%CONF_DIR%\seatunnel-env.cmd" ( + call "%CONF_DIR%\seatunnel-env.cmd" +) + +if "%~1"=="" ( + set "args=-h" +) else ( + set "args=%*" +) + +set "JAVA_OPTS=" +rem Log4j2 Config +if exist "%CONF_DIR%\log4j2.properties" ( + set "JAVA_OPTS=!JAVA_OPTS! -Dlog4j2.configurationFile=%CONF_DIR%\log4j2.properties" + set "JAVA_OPTS=!JAVA_OPTS! -Dseatunnel.logs.path=%APP_DIR%\logs" + set "JAVA_OPTS=!JAVA_OPTS! -Dseatunnel.logs.file_name=seatunnel-spark-starter" +) + +set "CLASS_PATH=%APP_DIR%\starter\logging\*;%APP_JAR%" + +for /f "delims=" %%i in ('java %JAVA_OPTS% -cp %CLASS_PATH% %APP_MAIN% %args%') do ( + set "CMD=%%i" + setlocal disabledelayedexpansion + if !errorlevel! equ 234 ( + echo !CMD! + endlocal + exit /b 0 + ) else if !errorlevel! equ 0 ( + echo Execute SeaTunnel Spark Job: !CMD! + endlocal + call !CMD! + ) else ( + echo !CMD! + endlocal + exit /b !errorlevel! + ) +) diff --git a/seatunnel-core/seatunnel-starter/src/main/bin/seatunnel-cluster.cmd b/seatunnel-core/seatunnel-starter/src/main/bin/seatunnel-cluster.cmd new file mode 100644 index 00000000000..e94a4bb482f --- /dev/null +++ b/seatunnel-core/seatunnel-starter/src/main/bin/seatunnel-cluster.cmd @@ -0,0 +1,86 @@ +@echo off +REM Licensed to the Apache Software Foundation (ASF) under one or more +REM contributor license agreements. See the NOTICE file distributed with +REM this work for additional information regarding copyright ownership. +REM The ASF licenses this file to You under the Apache License, Version 2.0 +REM (the "License"); you may not use this file except in compliance with +REM the License. You may obtain a copy of the License at +REM +REM http://www.apache.org/licenses/LICENSE-2.0 +REM +REM Unless required by applicable law or agreed to in writing, software +REM distributed under the License is distributed on an "AS IS" BASIS, +REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +REM See the License for the specific language governing permissions and +REM limitations under the License. + +setlocal enabledelayedexpansion + +REM resolve links - %0 may be a softlink +for %%F in ("%~f0") do ( + set "PRG=%%~fF" + set "PRG_DIR=%%~dpF" + set "APP_DIR=%%~dpF.." +) + +set "CONF_DIR=%APP_DIR%\config" +set "APP_JAR=%APP_DIR%\starter\seatunnel-starter.jar" +set "APP_MAIN=org.apache.seatunnel.core.starter.seatunnel.SeaTunnelServer" +set "OUT=%APP_DIR%\logs\seatunnel-server.out" + +set "HELP=false" +set "args=" + +for %%I in (%*) do ( + set "args=!args! %%I" + if "%%I"=="-d" set "DAEMON=true" + if "%%I"=="--daemon" set "DAEMON=true" + if "%%I"=="-h" set "HELP=true" + if "%%I"=="--help" set "HELP=true" +) + +REM SeaTunnel Engine Config +set "HAZELCAST_CONFIG=%CONF_DIR%\hazelcast.yaml" +set "SEATUNNEL_CONFIG=%CONF_DIR%\seatunnel.yaml" +set "JAVA_OPTS=%JvmOption%" + +for %%I in (%*) do ( + set "arg=%%I" + if "!arg:~0,10!"=="JvmOption=" ( + set "JAVA_OPTS=%JAVA_OPTS% !arg:~10!" + ) +) + +set "JAVA_OPTS=%JAVA_OPTS% -Dseatunnel.config=%SEATUNNEL_CONFIG%" +set "JAVA_OPTS=%JAVA_OPTS% -Dhazelcast.config=%HAZELCAST_CONFIG%" +set "JAVA_OPTS=%JAVA_OPTS% -Dlog4j2.contextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector" + +REM Server Debug Config +REM Usage instructions: +REM If you need to debug your code in cluster mode, please enable this configuration option and listen to the specified +REM port in your IDE. After that, you can happily debug your code. +REM set "JAVA_OPTS=%JAVA_OPTS% -Xdebug -Xrunjdwp:server=y,transport=dt_socket,address=5001,suspend=y" + +if exist "%CONF_DIR%\log4j2.properties" ( + set "JAVA_OPTS=%JAVA_OPTS% -Dlog4j2.configurationFile=%CONF_DIR%\log4j2.properties" + set "JAVA_OPTS=%JAVA_OPTS% -Dseatunnel.logs.path=%APP_DIR%\logs" + set "JAVA_OPTS=%JAVA_OPTS% -Dseatunnel.logs.file_name=seatunnel-engine-server" +) + +set "CLASS_PATH=%APP_DIR%\lib\*;%APP_JAR%" + +for /f "usebackq delims=" %%I in ("%APP_DIR%\config\jvm_options") do ( + set "line=%%I" + if not "!line:~0,1!"=="#" if "!line!" NEQ "" ( + set "JAVA_OPTS=!JAVA_OPTS! !line!" + ) +) + +if "%HELP%"=="false" ( + if not exist "%APP_DIR%\logs\" mkdir "%APP_DIR%\logs" + start "SeaTunnel Server" java %JAVA_OPTS% -cp "%CLASS_PATH%" %APP_MAIN% %args% > "%OUT%" 2>&1 +) else ( + java %JAVA_OPTS% -cp "%CLASS_PATH%" %APP_MAIN% %args% +) + +endlocal \ No newline at end of file diff --git a/seatunnel-core/seatunnel-starter/src/main/bin/seatunnel-cluster.sh b/seatunnel-core/seatunnel-starter/src/main/bin/seatunnel-cluster.sh index d4644f2d4da..e85a97e67a4 100755 --- a/seatunnel-core/seatunnel-starter/src/main/bin/seatunnel-cluster.sh +++ b/seatunnel-core/seatunnel-starter/src/main/bin/seatunnel-cluster.sh @@ -93,7 +93,7 @@ fi # Usage instructions: # If you need to debug your code in cluster mode, please enable this configuration option and listen to the specified # port in your IDE. After that, you can happily debug your code. -# JAVA_OPTS="${JAVA_OPTS} -Xrunjdwp:server=y,transport=dt_socket,address=8000,suspend=n" +# JAVA_OPTS="${JAVA_OPTS} -Xdebug -Xrunjdwp:server=y,transport=dt_socket,address=5001,suspend=y" CLASS_PATH=${APP_DIR}/lib/*:${APP_JAR} diff --git a/seatunnel-core/seatunnel-starter/src/main/bin/seatunnel.cmd b/seatunnel-core/seatunnel-starter/src/main/bin/seatunnel.cmd new file mode 100644 index 00000000000..cf9258e9d50 --- /dev/null +++ b/seatunnel-core/seatunnel-starter/src/main/bin/seatunnel.cmd @@ -0,0 +1,108 @@ +@echo off +REM Licensed to the Apache Software Foundation (ASF) under one or more +REM contributor license agreements. See the NOTICE file distributed with +REM this work for additional information regarding copyright ownership. +REM The ASF licenses this file to You under the Apache License, Version 2.0 +REM (the "License"); you may not use this file except in compliance with +REM the License. You may obtain a copy of the License at +REM +REM http://www.apache.org/licenses/LICENSE-2.0 +REM +REM Unless required by applicable law or agreed to in writing, software +REM distributed under the License is distributed on an "AS IS" BASIS, +REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +REM See the License for the specific language governing permissions and +REM limitations under the License. + +setlocal enabledelayedexpansion +REM resolve links - %0 may be a softlink +set "PRG=%~0" + +:resolveLoop +for %%F in ("%PRG%") do ( + set "PRG_DIR=%%~dpF" + set "PRG_NAME=%%~nxF" +) +set "PRG=%PRG_DIR%%PRG_NAME%" + +REM Get application directory +cd "%PRG_DIR%\.." +set "APP_DIR=%CD%" + +set "CONF_DIR=%APP_DIR%\config" +set "APP_JAR=%APP_DIR%\starter\seatunnel-starter.jar" +set "APP_MAIN=org.apache.seatunnel.core.starter.seatunnel.SeaTunnelClient" + +if exist "%CONF_DIR%\seatunnel-env.cmd" call "%CONF_DIR%\seatunnel-env.cmd" + +if "%~1"=="" ( + set "args=-h" +) else ( + set "args=%*" +) + +REM SeaTunnel Engine Config +if not defined HAZELCAST_CLIENT_CONFIG ( + set "HAZELCAST_CLIENT_CONFIG=%CONF_DIR%\hazelcast-client.yaml" +) + +if not defined HAZELCAST_CONFIG ( + set "HAZELCAST_CONFIG=%CONF_DIR%\hazelcast.yaml" +) + +if not defined SEATUNNEL_CONFIG ( + set "SEATUNNEL_CONFIG=%CONF_DIR%\seatunnel.yaml" +) + +if defined JvmOption ( + set "JAVA_OPTS=%JAVA_OPTS% %JvmOption%" +) + +for %%i in (%*) do ( + set "arg=%%i" + if "!arg:~0,9!"=="JvmOption" ( + set "JVM_OPTION=!arg:~9!" + set "JAVA_OPTS=!JAVA_OPTS! !JVM_OPTION!" + goto :break_loop + ) +) +:break_loop + +set "JAVA_OPTS=%JAVA_OPTS% -Dhazelcast.client.config=%HAZELCAST_CLIENT_CONFIG%" +set "JAVA_OPTS=%JAVA_OPTS% -Dseatunnel.config=%SEATUNNEL_CONFIG%" +set "JAVA_OPTS=%JAVA_OPTS% -Dhazelcast.config=%HAZELCAST_CONFIG%" + +REM if you want to debug, please +REM set "JAVA_OPTS=%JAVA_OPTS% -Xdebug -Xrunjdwp:transport=dt_socket,server=y,address=5000,suspend=y" + +REM Log4j2 Config +if exist "%CONF_DIR%\log4j2_client.properties" ( + set "JAVA_OPTS=%JAVA_OPTS% -Dlog4j2.configurationFile=%CONF_DIR%\log4j2_client.properties" + set "JAVA_OPTS=%JAVA_OPTS% -Dseatunnel.logs.path=%APP_DIR%\logs" + for %%i in (%args%) do ( + set "arg=%%i" + if "!arg!"=="-m" set "is_local_mode=true" + if "!arg!"=="--master" set "is_local_mode=true" + if "!arg!"=="-e" set "is_local_mode=true" + if "!arg!"=="--deploy-mode" set "is_local_mode=true" + ) + if defined is_local_mode ( + for /f "tokens=1-3 delims=:" %%A in ('echo %time%') do ( + set "ntime=%%A%%B%%C" + ) + set "JAVA_OPTS=%JAVA_OPTS% -Dseatunnel.logs.file_name=seatunnel-starter-client-!date:~0,4!!date:~5,2!!date:~8,2!-!time:~0,2!!time:~3,2!!time:~6,2!!ntime!" + ) else ( + set "JAVA_OPTS=%JAVA_OPTS% -Dseatunnel.logs.file_name=seatunnel-starter-client" + ) +) + +set "CLASS_PATH=%APP_DIR%\lib\*;%APP_JAR%" + +for /f "usebackq delims=" %%a in ("%APP_DIR%\config\jvm_client_options") do ( + set "line=%%a" + if not "!line:~0,1!"=="#" if "!line!" neq "" ( + set "JAVA_OPTS=!JAVA_OPTS! !line!" + ) +) + +java %JAVA_OPTS% -cp %CLASS_PATH% %APP_MAIN% %args% diff --git a/seatunnel-core/seatunnel-starter/src/main/bin/seatunnel.sh b/seatunnel-core/seatunnel-starter/src/main/bin/seatunnel.sh index 7c25ec126c0..b95800f1c2c 100755 --- a/seatunnel-core/seatunnel-starter/src/main/bin/seatunnel.sh +++ b/seatunnel-core/seatunnel-starter/src/main/bin/seatunnel.sh @@ -81,6 +81,12 @@ JAVA_OPTS="${JAVA_OPTS} -Dhazelcast.client.config=${HAZELCAST_CLIENT_CONFIG}" JAVA_OPTS="${JAVA_OPTS} -Dseatunnel.config=${SEATUNNEL_CONFIG}" JAVA_OPTS="${JAVA_OPTS} -Dhazelcast.config=${HAZELCAST_CONFIG}" +# Client Debug Config +# Usage instructions: +# If you need to debug your code in cluster mode, please enable this configuration option and listen to the specified +# port in your IDE. After that, you can happily debug your code. +# JAVA_OPTS="${JAVA_OPTS} -Xdebug -Xrunjdwp:transport=dt_socket,server=y,address=5000,suspend=y" + # Log4j2 Config if [ -e "${CONF_DIR}/log4j2_client.properties" ]; then JAVA_OPTS="${JAVA_OPTS} -Dlog4j2.configurationFile=${CONF_DIR}/log4j2_client.properties" diff --git a/seatunnel-core/seatunnel-starter/src/main/bin/stop-seatunnel-cluster.cmd b/seatunnel-core/seatunnel-starter/src/main/bin/stop-seatunnel-cluster.cmd new file mode 100644 index 00000000000..0c0cb72b014 --- /dev/null +++ b/seatunnel-core/seatunnel-starter/src/main/bin/stop-seatunnel-cluster.cmd @@ -0,0 +1,58 @@ +@echo off +rem Licensed to the Apache Software Foundation (ASF) under one or more +rem contributor license agreements. See the NOTICE file distributed with +rem this work for additional information regarding copyright ownership. +rem The ASF licenses this file to You under the Apache License, Version 2.0 +rem (the "License"); you may not use this file except in compliance with +rem the License. You may obtain a copy of the License at +rem +rem http://www.apache.org/licenses/LICENSE-2.0 +rem +rem Unless required by applicable law or agreed to in writing, software +rem distributed under the License is distributed on an "AS IS" BASIS, +rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +rem See the License for the specific language governing permissions and +rem limitations under the License. + +setlocal enabledelayedexpansion + +set "SEATUNNEL_DEFAULT_CLUSTER_NAME=seatunnel_default_cluster" +set "SHOW_USAGE=Usage: stop-seatunnel-cluster.bat \n Options: \n -cn, --cluster The name of the cluster to shut down (default: $SEATUNNEL_DEFAULT_CLUSTER_NAME) \n -h, --help Show the usage message" +set "APP_MAIN=org.apache.seatunnel.core.starter.seatunnel.SeaTunnelServer" +set "CLUSTER_NAME=" + +if "%~1"=="" ( + echo !SHOW_USAGE! + exit /B 1 +) + +:parse_args +if "%~1"=="-cn" ( + shift + set "CLUSTER_NAME=%~1" + shift + goto :parse_args +) else if "%~1"=="--cluster" ( + shift + set "CLUSTER_NAME=%~1" + shift + goto :parse_args +) else if "%~1"=="-h" ( + echo !SHOW_USAGE! + exit /B 0 +) else if "%~1"=="--help" ( + echo !SHOW_USAGE! + exit /B 0 +) + +if not defined CLUSTER_NAME ( + for /f %%i in ('tasklist /fi "imagename eq java.exe" ^| find "!APP_MAIN!"') do ( + taskkill /F /PID %%i + ) +) else ( + for /f %%i in ('tasklist /fi "imagename eq java.exe" ^| find "!APP_MAIN!" ^| find "!CLUSTER_NAME!"') do ( + taskkill /F /PID %%i + ) +) + +exit /B 0 \ No newline at end of file From cdd8e0a65ee10d0646f6f5eb786cd0ee1ff9aa6b Mon Sep 17 00:00:00 2001 From: kk <127465317+jackyyyyyssss@users.noreply.github.com> Date: Mon, 11 Sep 2023 14:31:57 +0800 Subject: [PATCH 10/15] [BUG][Connector-V2][http] fix httpheader cover (#5446) --- .../seatunnel/http/client/HttpClientProvider.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/seatunnel-connectors-v2/connector-http/connector-http-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/http/client/HttpClientProvider.java b/seatunnel-connectors-v2/connector-http/connector-http-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/http/client/HttpClientProvider.java index ef5b1f77d6b..2c6fe67b797 100644 --- a/seatunnel-connectors-v2/connector-http/connector-http-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/http/client/HttpClientProvider.java +++ b/seatunnel-connectors-v2/connector-http/connector-http-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/http/client/HttpClientProvider.java @@ -404,7 +404,17 @@ private void addHeaders(HttpRequestBase request, Map headers) { headers.forEach(request::addHeader); } + private boolean checkAlreadyHaveContentType(HttpEntityEnclosingRequestBase request) { + if (request.getEntity() != null && request.getEntity().getContentType() != null) { + return HTTP.CONTENT_TYPE.equals(request.getEntity().getContentType().getName()); + } + return false; + } + private void addBody(HttpEntityEnclosingRequestBase request, String body) { + if (checkAlreadyHaveContentType(request)) { + return; + } request.addHeader(HTTP.CONTENT_TYPE, APPLICATION_JSON); if (StringUtils.isBlank(body)) { From 23d79b0d1777916e542540eed1e27ceda10866e2 Mon Sep 17 00:00:00 2001 From: sunjane <7359662+aijing-sun@users.noreply.github.com> Date: Mon, 11 Sep 2023 14:36:11 +0800 Subject: [PATCH 11/15] [Hotfix][Connector-V2][StarRocks] fix starrocks template sql parser #5071 (#5332) * [Hotfix][Connector-V2][StarRocks] fix starrocks template sql parser offset && add primary key for default template --- .../config/StarRocksSinkOptions.java | 1 + .../starrocks/sink/StarRocksSaveModeUtil.java | 12 ++- .../starrocks/StarRocksCreateTableTest.java | 82 +++++++++++++++++++ 3 files changed, 93 insertions(+), 2 deletions(-) diff --git a/seatunnel-connectors-v2/connector-starrocks/src/main/java/org/apache/seatunnel/connectors/seatunnel/starrocks/config/StarRocksSinkOptions.java b/seatunnel-connectors-v2/connector-starrocks/src/main/java/org/apache/seatunnel/connectors/seatunnel/starrocks/config/StarRocksSinkOptions.java index eed2afc3605..1129d447162 100644 --- a/seatunnel-connectors-v2/connector-starrocks/src/main/java/org/apache/seatunnel/connectors/seatunnel/starrocks/config/StarRocksSinkOptions.java +++ b/seatunnel-connectors-v2/connector-starrocks/src/main/java/org/apache/seatunnel/connectors/seatunnel/starrocks/config/StarRocksSinkOptions.java @@ -60,6 +60,7 @@ public interface StarRocksSinkOptions { .stringType() .defaultValue( "CREATE TABLE IF NOT EXISTS `${database}`.`${table_name}` (\n" + + "${rowtype_primary_key},\n" + "${rowtype_fields}\n" + ") ENGINE=OLAP\n" + " PRIMARY KEY (${rowtype_primary_key})\n" diff --git a/seatunnel-connectors-v2/connector-starrocks/src/main/java/org/apache/seatunnel/connectors/seatunnel/starrocks/sink/StarRocksSaveModeUtil.java b/seatunnel-connectors-v2/connector-starrocks/src/main/java/org/apache/seatunnel/connectors/seatunnel/starrocks/sink/StarRocksSaveModeUtil.java index cb0d086859b..bbbc04eb20e 100644 --- a/seatunnel-connectors-v2/connector-starrocks/src/main/java/org/apache/seatunnel/connectors/seatunnel/starrocks/sink/StarRocksSaveModeUtil.java +++ b/seatunnel-connectors-v2/connector-starrocks/src/main/java/org/apache/seatunnel/connectors/seatunnel/starrocks/sink/StarRocksSaveModeUtil.java @@ -27,6 +27,8 @@ import org.apache.commons.lang3.StringUtils; +import java.util.Comparator; +import java.util.List; import java.util.Map; import java.util.function.Function; import java.util.stream.Collectors; @@ -82,8 +84,14 @@ private static String mergeColumnInTemplate( Map columnMap = tableSchema.getColumns().stream() .collect(Collectors.toMap(Column::getName, Function.identity())); - for (String col : columnInTemplate.keySet()) { - CreateTableParser.ColumnInfo columnInfo = columnInTemplate.get(col); + List columnInfosInSeq = + columnInTemplate.values().stream() + .sorted( + Comparator.comparingInt( + CreateTableParser.ColumnInfo::getStartIndex)) + .collect(Collectors.toList()); + for (CreateTableParser.ColumnInfo columnInfo : columnInfosInSeq) { + String col = columnInfo.getName(); if (StringUtils.isEmpty(columnInfo.getInfo())) { if (columnMap.containsKey(col)) { Column column = columnMap.get(col); diff --git a/seatunnel-connectors-v2/connector-starrocks/src/test/java/org/apache/seatunnel/connectors/seatunnel/starrocks/StarRocksCreateTableTest.java b/seatunnel-connectors-v2/connector-starrocks/src/test/java/org/apache/seatunnel/connectors/seatunnel/starrocks/StarRocksCreateTableTest.java index 22536ffd684..b571deb68ad 100644 --- a/seatunnel-connectors-v2/connector-starrocks/src/test/java/org/apache/seatunnel/connectors/seatunnel/starrocks/StarRocksCreateTableTest.java +++ b/seatunnel-connectors-v2/connector-starrocks/src/test/java/org/apache/seatunnel/connectors/seatunnel/starrocks/StarRocksCreateTableTest.java @@ -22,8 +22,11 @@ import org.apache.seatunnel.api.table.catalog.PrimaryKey; import org.apache.seatunnel.api.table.catalog.TableSchema; import org.apache.seatunnel.api.table.type.BasicType; +import org.apache.seatunnel.api.table.type.DecimalType; +import org.apache.seatunnel.api.table.type.LocalTimeType; import org.apache.seatunnel.connectors.seatunnel.starrocks.sink.StarRocksSaveModeUtil; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import java.util.ArrayList; @@ -70,4 +73,83 @@ public void test() { System.out.println(result); } + + @Test + public void testInSeq() { + + List columns = new ArrayList<>(); + + columns.add(PhysicalColumn.of("L_ORDERKEY", BasicType.INT_TYPE, null, false, null, "")); + columns.add(PhysicalColumn.of("L_PARTKEY", BasicType.INT_TYPE, null, false, null, "")); + columns.add(PhysicalColumn.of("L_SUPPKEY", BasicType.INT_TYPE, null, false, null, "")); + columns.add(PhysicalColumn.of("L_LINENUMBER", BasicType.INT_TYPE, null, false, null, "")); + columns.add(PhysicalColumn.of("L_QUANTITY", new DecimalType(15, 2), null, false, null, "")); + columns.add( + PhysicalColumn.of( + "L_EXTENDEDPRICE", new DecimalType(15, 2), null, false, null, "")); + columns.add(PhysicalColumn.of("L_DISCOUNT", new DecimalType(15, 2), null, false, null, "")); + columns.add(PhysicalColumn.of("L_TAX", new DecimalType(15, 2), null, false, null, "")); + columns.add( + PhysicalColumn.of("L_RETURNFLAG", BasicType.STRING_TYPE, null, false, null, "")); + columns.add( + PhysicalColumn.of("L_LINESTATUS", BasicType.STRING_TYPE, null, false, null, "")); + columns.add( + PhysicalColumn.of( + "L_SHIPDATE", LocalTimeType.LOCAL_DATE_TYPE, null, false, null, "")); + columns.add( + PhysicalColumn.of( + "L_COMMITDATE", LocalTimeType.LOCAL_DATE_TYPE, null, false, null, "")); + columns.add( + PhysicalColumn.of( + "L_RECEIPTDATE", LocalTimeType.LOCAL_DATE_TYPE, null, false, null, "")); + columns.add( + PhysicalColumn.of("L_SHIPINSTRUCT", BasicType.STRING_TYPE, null, false, null, "")); + columns.add(PhysicalColumn.of("L_SHIPMODE", BasicType.STRING_TYPE, null, false, null, "")); + columns.add(PhysicalColumn.of("L_COMMENT", BasicType.STRING_TYPE, null, false, null, "")); + + String result = + StarRocksSaveModeUtil.fillingCreateSql( + "CREATE TABLE IF NOT EXISTS `${database}`.`${table_name}` (\n" + + "`L_COMMITDATE`,\n" + + "${rowtype_primary_key},\n" + + "L_SUPPKEY BIGINT NOT NULL,\n" + + "${rowtype_fields}\n" + + ") ENGINE=OLAP\n" + + " PRIMARY KEY (L_COMMITDATE, ${rowtype_primary_key}, L_SUPPKEY)\n" + + "DISTRIBUTED BY HASH (${rowtype_primary_key})" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\" \n" + + ")", + "tpch", + "lineitem", + TableSchema.builder() + .primaryKey( + PrimaryKey.of( + "", Arrays.asList("L_ORDERKEY", "L_LINENUMBER"))) + .columns(columns) + .build()); + String expected = + "CREATE TABLE IF NOT EXISTS `tpch`.`lineitem` (\n" + + "`L_COMMITDATE` DATE NOT NULL ,\n" + + "`L_ORDERKEY` INT NOT NULL ,`L_LINENUMBER` INT NOT NULL ,\n" + + "L_SUPPKEY BIGINT NOT NULL,\n" + + "`L_PARTKEY` INT NOT NULL ,\n" + + "`L_QUANTITY` Decimal(15, 2) NOT NULL ,\n" + + "`L_EXTENDEDPRICE` Decimal(15, 2) NOT NULL ,\n" + + "`L_DISCOUNT` Decimal(15, 2) NOT NULL ,\n" + + "`L_TAX` Decimal(15, 2) NOT NULL ,\n" + + "`L_RETURNFLAG` STRING NOT NULL ,\n" + + "`L_LINESTATUS` STRING NOT NULL ,\n" + + "`L_SHIPDATE` DATE NOT NULL ,\n" + + "`L_RECEIPTDATE` DATE NOT NULL ,\n" + + "`L_SHIPINSTRUCT` STRING NOT NULL ,\n" + + "`L_SHIPMODE` STRING NOT NULL ,\n" + + "`L_COMMENT` STRING NOT NULL \n" + + ") ENGINE=OLAP\n" + + " PRIMARY KEY (L_COMMITDATE, `L_ORDERKEY`,`L_LINENUMBER`, L_SUPPKEY)\n" + + "DISTRIBUTED BY HASH (`L_ORDERKEY`,`L_LINENUMBER`)PROPERTIES (\n" + + " \"replication_num\" = \"1\" \n" + + ")"; + Assertions.assertEquals(result, expected); + } } From 04ce22ac1eb34f1bb976764e97bed5f15edaed5a Mon Sep 17 00:00:00 2001 From: Volodymyr Date: Mon, 11 Sep 2023 14:42:59 +0800 Subject: [PATCH 12/15] [Hotfix][Connector-V2][Hive] fix the bug that hive-site.xml can not be injected in HiveConf (#5261) --- .../hive/utils/HiveMetaStoreProxy.java | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/utils/HiveMetaStoreProxy.java b/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/utils/HiveMetaStoreProxy.java index f6ba5cfb12c..788fe38dc1c 100644 --- a/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/utils/HiveMetaStoreProxy.java +++ b/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/utils/HiveMetaStoreProxy.java @@ -35,6 +35,8 @@ import lombok.NonNull; import lombok.extern.slf4j.Slf4j; +import java.io.File; +import java.net.MalformedURLException; import java.util.List; import java.util.Objects; @@ -54,10 +56,11 @@ private HiveMetaStoreProxy(Config config) { Configuration configuration = new Configuration(); FileSystemUtils.doKerberosAuthentication(configuration, principal, keytabPath); } - if (config.hasPath(HiveConfig.HIVE_SITE_PATH.key())) { - hiveConf.addResource(config.getString(HiveConfig.HIVE_SITE_PATH.key())); - } try { + if (config.hasPath(HiveConfig.HIVE_SITE_PATH.key())) { + String hiveSitePath = config.getString(HiveConfig.HIVE_SITE_PATH.key()); + hiveConf.addResource(new File(hiveSitePath).toURI().toURL()); + } hiveMetaStoreClient = new HiveMetaStoreClient(hiveConf); } catch (MetaException e) { String errorMsg = @@ -67,6 +70,14 @@ private HiveMetaStoreProxy(Config config) { metastoreUri); throw new HiveConnectorException( HiveConnectorErrorCode.INITIALIZE_HIVE_METASTORE_CLIENT_FAILED, errorMsg, e); + } catch (MalformedURLException e) { + String errorMsg = + String.format( + "Using this hive uris [%s], hive conf [%s] to initialize " + + "hive metastore client instance failed", + metastoreUri, config.getString(HiveConfig.HIVE_SITE_PATH.key())); + throw new HiveConnectorException( + HiveConnectorErrorCode.INITIALIZE_HIVE_METASTORE_CLIENT_FAILED, errorMsg, e); } } From 07105a5c274c606c62ab8c10623aaca592ebc1d8 Mon Sep 17 00:00:00 2001 From: Nick Young <72905543+NickYoungPeng@users.noreply.github.com> Date: Mon, 11 Sep 2023 14:53:05 +0800 Subject: [PATCH 13/15] [Bug] [seatunnel-formats] Allow the entry in the map to be null and allow the key in the entry to be null (#5277) * :bug: "Add null check in text deserialization This commit adds a null check while splitting the kv in the TextDeserializationSchema. Not having this check could potentially lead to a crash if 'kvs' has less than two elements. Now, the function will return null in this case, significantly improving reliability." * :bug: Refactor text deserialization, allow null value assignments Modified the TextDeserializationSchema to allow key-value pairs with null values. This adjustment helps to handle data records with missing values more accurately. The code was also reformatted for improved readability. * :art: Refactor Builder constructor in TextDeserializationSchema Tidied up the Builder constructor in TextDeserializationSchema by removing unnecessary empty lines. This change improves code readability and consistency with the rest of the class? * :white_check_mark: Add test case where the value is empty and the key is empty in the map --------- Co-authored-by: yangpeng --- .../format/text/TextDeserializationSchema.java | 10 +++++++--- .../seatunnel/format/text/TextFormatSchemaTest.java | 9 ++++++++- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/seatunnel-formats/seatunnel-format-text/src/main/java/org/apache/seatunnel/format/text/TextDeserializationSchema.java b/seatunnel-formats/seatunnel-format-text/src/main/java/org/apache/seatunnel/format/text/TextDeserializationSchema.java index 86f25a69b51..1ec0499fb5e 100644 --- a/seatunnel-formats/seatunnel-format-text/src/main/java/org/apache/seatunnel/format/text/TextDeserializationSchema.java +++ b/seatunnel-formats/seatunnel-format-text/src/main/java/org/apache/seatunnel/format/text/TextDeserializationSchema.java @@ -187,9 +187,13 @@ private Object convert(String field, SeaTunnelDataType fieldType, int level) String[] kvs = field.split(separators[level + 1]); for (String kv : kvs) { String[] splits = kv.split(separators[level + 2]); - objectMap.put( - convert(splits[0], keyType, level + 1), - convert(splits[1], valueType, level + 1)); + if (splits.length < 2) { + objectMap.put(convert(splits[0], keyType, level + 1), null); + } else { + objectMap.put( + convert(splits[0], keyType, level + 1), + convert(splits[1], valueType, level + 1)); + } } return objectMap; case STRING: diff --git a/seatunnel-formats/seatunnel-format-text/src/test/java/org/apache/seatunnel/format/text/TextFormatSchemaTest.java b/seatunnel-formats/seatunnel-format-text/src/test/java/org/apache/seatunnel/format/text/TextFormatSchemaTest.java index f099103efc6..57e99d49b69 100644 --- a/seatunnel-formats/seatunnel-format-text/src/test/java/org/apache/seatunnel/format/text/TextFormatSchemaTest.java +++ b/seatunnel-formats/seatunnel-format-text/src/test/java/org/apache/seatunnel/format/text/TextFormatSchemaTest.java @@ -45,7 +45,14 @@ public class TextFormatSchemaTest { + '\002' + "Kris" + '\003' - + "21\001" + + "21" + + '\002' + + "nullValueKey" + + '\003' + + '\002' + + '\003' + + "1231" + + "\001" + "tyrantlucifer\001" + "true\001" + "1\001" From cde42a3983fc1e8d5a70570ba22e99b946f82c9c Mon Sep 17 00:00:00 2001 From: ZhilinLi Date: Mon, 11 Sep 2023 14:58:26 +0800 Subject: [PATCH 14/15] [Docs][Connector][Source][doc]add Parallel parallelism (#5310) --- docs/en/connector-v2/sink/Mysql.md | 8 +++---- docs/en/connector-v2/source/Clickhouse.md | 2 +- docs/en/connector-v2/source/Jdbc.md | 28 +++++++++++++++-------- docs/en/connector-v2/source/MongoDB.md | 7 ++++++ docs/en/connector-v2/source/Mysql.md | 13 ++++++++--- docs/en/connector-v2/source/OceanBase.md | 7 ++++++ docs/en/connector-v2/source/Oracle.md | 7 ++++++ docs/en/connector-v2/source/PostgreSQL.md | 7 ++++++ 8 files changed, 62 insertions(+), 17 deletions(-) diff --git a/docs/en/connector-v2/sink/Mysql.md b/docs/en/connector-v2/sink/Mysql.md index 6c01c35ee8c..f453a60c4e9 100644 --- a/docs/en/connector-v2/sink/Mysql.md +++ b/docs/en/connector-v2/sink/Mysql.md @@ -122,7 +122,7 @@ transform { sink { jdbc { - url = "jdbc:mysql://localhost:3306/test" + url = "jdbc:mysql://localhost:3306/test?useUnicode=true&characterEncoding=UTF-8&rewriteBatchedStatements=true" driver = "com.mysql.cj.jdbc.Driver" user = "root" password = "123456" @@ -140,7 +140,7 @@ sink { ``` sink { jdbc { - url = "jdbc:mysql://localhost:3306/test" + url = "jdbc:mysql://localhost:3306/test?useUnicode=true&characterEncoding=UTF-8&rewriteBatchedStatements=true" driver = "com.mysql.cj.jdbc.Driver" user = "root" password = "123456" @@ -159,7 +159,7 @@ sink { ``` sink { jdbc { - url = "jdbc:mysql://localhost:3306/test" + url = "jdbc:mysql://localhost:3306/test?useUnicode=true&characterEncoding=UTF-8&rewriteBatchedStatements=true" driver = "com.mysql.cj.jdbc.Driver" max_retries = 0 @@ -181,7 +181,7 @@ sink { ``` sink { jdbc { - url = "jdbc:mysql://localhost:3306/test" + url = "jdbc:mysql://localhost:3306/test?useUnicode=true&characterEncoding=UTF-8&rewriteBatchedStatements=true" driver = "com.mysql.cj.jdbc.Driver" user = "root" password = "123456" diff --git a/docs/en/connector-v2/source/Clickhouse.md b/docs/en/connector-v2/source/Clickhouse.md index 7596bf72a8f..d70a8f0e33f 100644 --- a/docs/en/connector-v2/source/Clickhouse.md +++ b/docs/en/connector-v2/source/Clickhouse.md @@ -66,7 +66,7 @@ The following example demonstrates how to create a data synchronization job that ```bash # Set the basic configuration of the task to be performed env { - execution.parallelism = 1 + execution.parallelism = 10 job.mode = "BATCH" } diff --git a/docs/en/connector-v2/source/Jdbc.md b/docs/en/connector-v2/source/Jdbc.md index d82df87a02e..585c2bc0024 100644 --- a/docs/en/connector-v2/source/Jdbc.md +++ b/docs/en/connector-v2/source/Jdbc.md @@ -145,15 +145,25 @@ Jdbc { parallel: ``` -Jdbc { - url = "jdbc:mysql://localhost/test?serverTimezone=GMT%2b8" - driver = "com.mysql.cj.jdbc.Driver" - connection_check_timeout_sec = 100 - user = "root" - password = "123456" - query = "select * from type_bin" - partition_column = "id" - partition_num = 10 +env { + execution.parallelism = 10 + job.mode = "BATCH" +} +source { + Jdbc { + url = "jdbc:mysql://localhost/test?serverTimezone=GMT%2b8" + driver = "com.mysql.cj.jdbc.Driver" + connection_check_timeout_sec = 100 + user = "root" + password = "123456" + query = "select * from type_bin" + partition_column = "id" + partition_num = 10 + } +} + +sink { + Console {} } ``` diff --git a/docs/en/connector-v2/source/MongoDB.md b/docs/en/connector-v2/source/MongoDB.md index 137fb205b8c..d63d303fa24 100644 --- a/docs/en/connector-v2/source/MongoDB.md +++ b/docs/en/connector-v2/source/MongoDB.md @@ -283,6 +283,10 @@ By utilizing `flat.sync-string`, only one field attribute value can be set, and This operation will perform a string mapping on a single MongoDB data entry. ```bash +env { + execution.parallelism = 10 + job.mode = "BATCH" +} source { MongoDB { uri = "mongodb://user:password@127.0.0.1:27017" @@ -296,6 +300,9 @@ source { } } } +sink { + Console {} +} ``` Use the data samples synchronized with modified parameters, such as the following: diff --git a/docs/en/connector-v2/source/Mysql.md b/docs/en/connector-v2/source/Mysql.md index 001ef1463da..bdac5c0aec6 100644 --- a/docs/en/connector-v2/source/Mysql.md +++ b/docs/en/connector-v2/source/Mysql.md @@ -94,7 +94,7 @@ env { } source{ Jdbc { - url = "jdbc:mysql://localhost:3306/test?serverTimezone=GMT%2b8" + url = "jdbc:mysql://localhost:3306/test?serverTimezone=GMT%2b8&useUnicode=true&characterEncoding=UTF-8&rewriteBatchedStatements=true" driver = "com.mysql.cj.jdbc.Driver" connection_check_timeout_sec = 100 user = "root" @@ -118,9 +118,13 @@ sink { > Read your query table in parallel with the shard field you configured and the shard data You can do this if you want to read the whole table ``` +env { + execution.parallelism = 10 + job.mode = "BATCH" +} source { Jdbc { - url = "jdbc:mysql://localhost:3306/test?serverTimezone=GMT%2b8" + url = "jdbc:mysql://localhost:3306/test?serverTimezone=GMT%2b8&useUnicode=true&characterEncoding=UTF-8&rewriteBatchedStatements=true" driver = "com.mysql.cj.jdbc.Driver" connection_check_timeout_sec = 100 user = "root" @@ -133,6 +137,9 @@ source { partition_num = 10 } } +sink { + Console {} +} ``` ### Parallel Boundary: @@ -142,7 +149,7 @@ source { ``` source { Jdbc { - url = "jdbc:mysql://localhost:3306/test?serverTimezone=GMT%2b8" + url = "jdbc:mysql://localhost:3306/test?serverTimezone=GMT%2b8&useUnicode=true&characterEncoding=UTF-8&rewriteBatchedStatements=true" driver = "com.mysql.cj.jdbc.Driver" connection_check_timeout_sec = 100 user = "root" diff --git a/docs/en/connector-v2/source/OceanBase.md b/docs/en/connector-v2/source/OceanBase.md index bd035793eee..434e25284dd 100644 --- a/docs/en/connector-v2/source/OceanBase.md +++ b/docs/en/connector-v2/source/OceanBase.md @@ -127,6 +127,10 @@ sink { > Read your query table in parallel with the shard field you configured and the shard data. You can do this if you want to read the whole table ``` +env { + execution.parallelism = 10 + job.mode = "BATCH" +} source { Jdbc { driver = "com.oceanbase.jdbc.Driver" @@ -141,6 +145,9 @@ source { partition_num = 10 } } +sink { + Console {} +} ``` ### Parallel Boundary: diff --git a/docs/en/connector-v2/source/Oracle.md b/docs/en/connector-v2/source/Oracle.md index 385d55ca9e5..f191cda9d99 100644 --- a/docs/en/connector-v2/source/Oracle.md +++ b/docs/en/connector-v2/source/Oracle.md @@ -111,6 +111,10 @@ sink { > Read your query table in parallel with the shard field you configured and the shard data You can do this if you want to read the whole table ``` +env { + execution.parallelism = 10 + job.mode = "BATCH" +} source { Jdbc { url = "jdbc:oracle:thin:@datasource01:1523:xe" @@ -126,6 +130,9 @@ source { partition_num = 10 } } +sink { + Console {} +} ``` ### Parallel Boundary: diff --git a/docs/en/connector-v2/source/PostgreSQL.md b/docs/en/connector-v2/source/PostgreSQL.md index 50839780726..63ddbc25ecf 100644 --- a/docs/en/connector-v2/source/PostgreSQL.md +++ b/docs/en/connector-v2/source/PostgreSQL.md @@ -120,6 +120,10 @@ sink { > Read your query table in parallel with the shard field you configured and the shard data You can do this if you want to read the whole table ``` +env { + execution.parallelism = 10 + job.mode = "BATCH" +} source{ jdbc{ url = "jdbc:postgresql://localhost:5432/test" @@ -131,6 +135,9 @@ source{ partition_num = 5 } } +sink { + Console {} +} ``` ### Parallel Boundary: From 5e378831ee16414619e0db0be0776ef305512de8 Mon Sep 17 00:00:00 2001 From: Carl-Zhou-CN <67902676+Carl-Zhou-CN@users.noreply.github.com> Date: Tue, 12 Sep 2023 13:39:43 +0800 Subject: [PATCH 15/15] [Feature][Connector-V2][CDC] Support flink running cdc job (#4918) * [feature][Connector-V2][cdc] Support flink running cdc job * [feature][Connector-V2][cdc] Support flink running cdc job * [feature][Connector-V2][cdc] mongo e2e delete --------- Co-authored-by: zhouyao --- .../cdc/base/source/IncrementalSource.java | 3 +- .../execution/FlinkRuntimeEnvironment.java | 17 +++++---- .../FlinkAbstractPluginExecuteProcessor.java | 37 +++++++++++++++++-- .../execution/FlinkRuntimeEnvironment.java | 17 +++++---- .../execution/SourceExecuteProcessor.java | 3 +- .../core/starter/flink/utils/TableUtil.java | 8 ++-- .../seatunnel/cdc/mysql/MysqlCDCIT.java | 8 +++- .../cdc/sqlserver/SqlServerCDCIT.java | 2 +- 8 files changed, 69 insertions(+), 26 deletions(-) diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/main/java/org/apache/seatunnel/connectors/cdc/base/source/IncrementalSource.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/main/java/org/apache/seatunnel/connectors/cdc/base/source/IncrementalSource.java index c10ab3e0613..ed04fb0f5d7 100644 --- a/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/main/java/org/apache/seatunnel/connectors/cdc/base/source/IncrementalSource.java +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/main/java/org/apache/seatunnel/connectors/cdc/base/source/IncrementalSource.java @@ -27,6 +27,7 @@ import org.apache.seatunnel.api.source.SeaTunnelSource; import org.apache.seatunnel.api.source.SourceReader; import org.apache.seatunnel.api.source.SourceSplitEnumerator; +import org.apache.seatunnel.api.source.SupportCoordinate; import org.apache.seatunnel.api.table.type.SeaTunnelDataType; import org.apache.seatunnel.api.table.type.SeaTunnelRow; import org.apache.seatunnel.connectors.cdc.base.config.SourceConfig; @@ -76,7 +77,7 @@ @NoArgsConstructor public abstract class IncrementalSource - implements SeaTunnelSource { + implements SeaTunnelSource, SupportCoordinate { protected ReadonlyConfig readonlyConfig; protected SourceConfig.Factory configFactory; diff --git a/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-13-starter/src/main/java/org/apache/seatunnel/core/starter/flink/execution/FlinkRuntimeEnvironment.java b/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-13-starter/src/main/java/org/apache/seatunnel/core/starter/flink/execution/FlinkRuntimeEnvironment.java index 34aa7ee4f2d..996c9698fb0 100644 --- a/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-13-starter/src/main/java/org/apache/seatunnel/core/starter/flink/execution/FlinkRuntimeEnvironment.java +++ b/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-13-starter/src/main/java/org/apache/seatunnel/core/starter/flink/execution/FlinkRuntimeEnvironment.java @@ -316,19 +316,22 @@ private void setCheckpoint() { } } - public void registerResultTable(Config config, DataStream dataStream) { - if (config.hasPath(RESULT_TABLE_NAME)) { - String name = config.getString(RESULT_TABLE_NAME); - StreamTableEnvironment tableEnvironment = this.getStreamTableEnvironment(); - if (!TableUtil.tableExists(tableEnvironment, name)) { + public void registerResultTable( + Config config, DataStream dataStream, String name, Boolean isAppend) { + StreamTableEnvironment tableEnvironment = this.getStreamTableEnvironment(); + if (!TableUtil.tableExists(tableEnvironment, name)) { + if (isAppend) { if (config.hasPath("field_name")) { String fieldName = config.getString("field_name"); tableEnvironment.registerDataStream(name, dataStream, fieldName); - } else { - tableEnvironment.registerDataStream(name, dataStream); + return; } + tableEnvironment.registerDataStream(name, dataStream); + return; } } + tableEnvironment.createTemporaryView( + name, tableEnvironment.fromChangelogStream(dataStream)); } public static FlinkRuntimeEnvironment getInstance(Config config) { diff --git a/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-starter-common/src/main/java/org/apache/seatunnel/core/starter/flink/execution/FlinkAbstractPluginExecuteProcessor.java b/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-starter-common/src/main/java/org/apache/seatunnel/core/starter/flink/execution/FlinkAbstractPluginExecuteProcessor.java index e9d36ba068e..6c61f61b957 100644 --- a/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-starter-common/src/main/java/org/apache/seatunnel/core/starter/flink/execution/FlinkAbstractPluginExecuteProcessor.java +++ b/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-starter-common/src/main/java/org/apache/seatunnel/core/starter/flink/execution/FlinkAbstractPluginExecuteProcessor.java @@ -31,15 +31,19 @@ import java.net.URL; import java.net.URLClassLoader; +import java.util.HashMap; import java.util.List; import java.util.Optional; import java.util.function.BiConsumer; +import static org.apache.seatunnel.api.common.CommonOptions.RESULT_TABLE_NAME; + public abstract class FlinkAbstractPluginExecuteProcessor implements PluginExecuteProcessor, FlinkRuntimeEnvironment> { protected static final String ENGINE_TYPE = "seatunnel"; protected static final String PLUGIN_NAME = "plugin_name"; protected static final String SOURCE_TABLE_NAME = "source_table_name"; + protected static HashMap isAppendMap = new HashMap<>(); protected static final BiConsumer ADD_URL_TO_CLASSLOADER = (classLoader, url) -> { @@ -76,14 +80,41 @@ protected Optional> fromSourceTable(Config pluginConfig) { if (pluginConfig.hasPath(SOURCE_TABLE_NAME)) { StreamTableEnvironment tableEnvironment = flinkRuntimeEnvironment.getStreamTableEnvironment(); - Table table = tableEnvironment.from(pluginConfig.getString(SOURCE_TABLE_NAME)); - return Optional.ofNullable(TableUtil.tableToDataStream(tableEnvironment, table, true)); + String tableName = pluginConfig.getString(SOURCE_TABLE_NAME); + Table table = tableEnvironment.from(tableName); + return Optional.ofNullable( + TableUtil.tableToDataStream( + tableEnvironment, table, isAppendMap.getOrDefault(tableName, true))); } return Optional.empty(); } protected void registerResultTable(Config pluginConfig, DataStream dataStream) { - flinkRuntimeEnvironment.registerResultTable(pluginConfig, dataStream); + if (pluginConfig.hasPath(RESULT_TABLE_NAME.key())) { + String resultTable = pluginConfig.getString(RESULT_TABLE_NAME.key()); + if (pluginConfig.hasPath(SOURCE_TABLE_NAME)) { + String sourceTable = pluginConfig.getString(SOURCE_TABLE_NAME); + flinkRuntimeEnvironment.registerResultTable( + pluginConfig, + dataStream, + resultTable, + isAppendMap.getOrDefault(sourceTable, true)); + registerAppendStream(pluginConfig); + return; + } + flinkRuntimeEnvironment.registerResultTable( + pluginConfig, + dataStream, + resultTable, + isAppendMap.getOrDefault(resultTable, true)); + } + } + + protected void registerAppendStream(Config pluginConfig) { + if (pluginConfig.hasPath(RESULT_TABLE_NAME.key())) { + String tableName = pluginConfig.getString(RESULT_TABLE_NAME.key()); + isAppendMap.put(tableName, false); + } } protected abstract List initializePlugins( diff --git a/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-starter-common/src/main/java/org/apache/seatunnel/core/starter/flink/execution/FlinkRuntimeEnvironment.java b/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-starter-common/src/main/java/org/apache/seatunnel/core/starter/flink/execution/FlinkRuntimeEnvironment.java index 583a1cf3e5c..12168921d8c 100644 --- a/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-starter-common/src/main/java/org/apache/seatunnel/core/starter/flink/execution/FlinkRuntimeEnvironment.java +++ b/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-starter-common/src/main/java/org/apache/seatunnel/core/starter/flink/execution/FlinkRuntimeEnvironment.java @@ -316,19 +316,22 @@ private void setCheckpoint() { } } - public void registerResultTable(Config config, DataStream dataStream) { - if (config.hasPath(RESULT_TABLE_NAME)) { - String name = config.getString(RESULT_TABLE_NAME); - StreamTableEnvironment tableEnvironment = this.getStreamTableEnvironment(); - if (!TableUtil.tableExists(tableEnvironment, name)) { + public void registerResultTable( + Config config, DataStream dataStream, String name, Boolean isAppend) { + StreamTableEnvironment tableEnvironment = this.getStreamTableEnvironment(); + if (!TableUtil.tableExists(tableEnvironment, name)) { + if (isAppend) { if (config.hasPath("field_name")) { String fieldName = config.getString("field_name"); tableEnvironment.registerDataStream(name, dataStream, fieldName); - } else { - tableEnvironment.registerDataStream(name, dataStream); + return; } + tableEnvironment.registerDataStream(name, dataStream); + return; } } + tableEnvironment.createTemporaryView( + name, tableEnvironment.fromChangelogStream(dataStream)); } public static FlinkRuntimeEnvironment getInstance(Config config) { diff --git a/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-starter-common/src/main/java/org/apache/seatunnel/core/starter/flink/execution/SourceExecuteProcessor.java b/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-starter-common/src/main/java/org/apache/seatunnel/core/starter/flink/execution/SourceExecuteProcessor.java index 6bcc5fe8939..f3ebdd04378 100644 --- a/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-starter-common/src/main/java/org/apache/seatunnel/core/starter/flink/execution/SourceExecuteProcessor.java +++ b/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-starter-common/src/main/java/org/apache/seatunnel/core/starter/flink/execution/SourceExecuteProcessor.java @@ -65,9 +65,11 @@ public List> execute(List> upstreamDataStreams) List> sources = new ArrayList<>(); for (int i = 0; i < plugins.size(); i++) { SeaTunnelSource internalSource = plugins.get(i); + Config pluginConfig = pluginConfigs.get(i); BaseSeaTunnelSourceFunction sourceFunction; if (internalSource instanceof SupportCoordinate) { sourceFunction = new SeaTunnelCoordinatedSource(internalSource); + registerAppendStream(pluginConfig); } else { sourceFunction = new SeaTunnelParallelSource(internalSource); } @@ -80,7 +82,6 @@ public List> execute(List> upstreamDataStreams) sourceFunction, "SeaTunnel " + internalSource.getClass().getSimpleName(), bounded); - Config pluginConfig = pluginConfigs.get(i); if (pluginConfig.hasPath(CommonOptions.PARALLELISM.key())) { int parallelism = pluginConfig.getInt(CommonOptions.PARALLELISM.key()); sourceStream.setParallelism(parallelism); diff --git a/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-starter-common/src/main/java/org/apache/seatunnel/core/starter/flink/utils/TableUtil.java b/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-starter-common/src/main/java/org/apache/seatunnel/core/starter/flink/utils/TableUtil.java index ca1603cdf99..aad97518f4b 100644 --- a/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-starter-common/src/main/java/org/apache/seatunnel/core/starter/flink/utils/TableUtil.java +++ b/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-starter-common/src/main/java/org/apache/seatunnel/core/starter/flink/utils/TableUtil.java @@ -37,11 +37,9 @@ public static DataStream tableToDataStream( if (isAppend) { return tableEnvironment.toAppendStream(table, typeInfo); } - return tableEnvironment - .toRetractStream(table, typeInfo) - .filter(row -> row.f0) - .map(row -> row.f1) - .returns(typeInfo); + DataStream dataStream = tableEnvironment.toChangelogStream(table); + dataStream.getTransformation().setOutputType(typeInfo); + return dataStream; } public static boolean tableExists(TableEnvironment tableEnvironment, String name) { diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/MysqlCDCIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/MysqlCDCIT.java index 1d0d90853fc..b648febd7d9 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/MysqlCDCIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/MysqlCDCIT.java @@ -55,7 +55,7 @@ @Slf4j @DisabledOnContainer( value = {}, - type = {EngineType.SPARK, EngineType.FLINK}, + type = {EngineType.SPARK}, disabledReason = "Currently SPARK and FLINK do not support cdc") public class MysqlCDCIT extends TestSuiteBase implements TestResource { @@ -88,6 +88,9 @@ public class MysqlCDCIT extends TestSuiteBase implements TestResource { + " f_enum, cast(f_mediumblob as char) as f_mediumblob, f_long_varchar, f_real, f_time, f_tinyint, f_tinyint_unsigned," + " f_json, cast(f_year as year) from mysql_cdc_e2e_sink_table"; + private static final String CLEAN_SOURCE = "truncate table mysql_cdc_e2e_source_table"; + private static final String CLEAN_SINK = "truncate table mysql_cdc_e2e_sink_table"; + private static MySqlContainer createMySqlContainer(MySqlVersion version) { MySqlContainer mySqlContainer = new MySqlContainer(version) @@ -134,6 +137,9 @@ public void startUp() throws ClassNotFoundException, InterruptedException { @TestTemplate public void testMysqlCdcCheckDataE2e(TestContainer container) throws IOException, InterruptedException { + // Clear related content to ensure that multiple operations are not affected + executeSql(CLEAN_SOURCE); + executeSql(CLEAN_SINK); CompletableFuture executeJobFuture = CompletableFuture.supplyAsync( diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-sqlserver-e2e/src/test/java/org/apache/seatunnel/e2e/connector/cdc/sqlserver/SqlServerCDCIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-sqlserver-e2e/src/test/java/org/apache/seatunnel/e2e/connector/cdc/sqlserver/SqlServerCDCIT.java index 8bca3e3b036..bfe2a358889 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-sqlserver-e2e/src/test/java/org/apache/seatunnel/e2e/connector/cdc/sqlserver/SqlServerCDCIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-sqlserver-e2e/src/test/java/org/apache/seatunnel/e2e/connector/cdc/sqlserver/SqlServerCDCIT.java @@ -65,7 +65,7 @@ @Slf4j @DisabledOnContainer( value = {}, - type = {EngineType.SPARK, EngineType.FLINK}, + type = {EngineType.SPARK}, disabledReason = "Currently SPARK and FLINK do not support cdc") public class SqlServerCDCIT extends TestSuiteBase implements TestResource {