diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index 88a2d59e3f1..a9f2b3cd0a7 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -274,7 +274,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '17' ] os: [ 'ubuntu-latest', 'windows-latest' ] timeout-minutes: 60 steps: @@ -297,7 +297,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '17' ] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -328,7 +328,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '17' ] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -359,7 +359,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '17' ] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -390,7 +390,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '17' ] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -420,7 +420,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '17' ] os: [ 'ubuntu-latest' ] timeout-minutes: 120 steps: @@ -450,7 +450,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '17' ] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -480,7 +480,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '17' ] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -511,7 +511,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '17' ] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -542,9 +542,9 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '17' ] os: [ 'ubuntu-latest' ] - timeout-minutes: 45 + timeout-minutes: 60 steps: - uses: actions/checkout@v2 - name: Set up JDK ${{ matrix.java }} @@ -565,7 +565,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '17' ] os: [ 'ubuntu-latest' ] timeout-minutes: 45 steps: @@ -598,7 +598,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '17' ] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -622,7 +622,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '17' ] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -646,7 +646,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '17' ] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -673,7 +673,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '17' ] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -700,7 +700,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '17' ] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -727,7 +727,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '17' ] os: [ 'ubuntu-latest' ] timeout-minutes: 150 steps: @@ -754,7 +754,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '17' ] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -781,7 +781,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '17' ] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -808,7 +808,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '17' ] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -835,7 +835,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '17' ] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -861,7 +861,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '17' ] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -887,7 +887,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '17' ] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -911,7 +911,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '17' ] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -935,7 +935,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '17' ] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -959,7 +959,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '17' ] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -985,7 +985,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '17' ] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -1009,7 +1009,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '17' ] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -1032,7 +1032,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '17' ] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -1055,7 +1055,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '17' ] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -1078,7 +1078,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '17' ] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -1102,7 +1102,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '17' ] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: @@ -1128,7 +1128,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '17' ] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: diff --git a/.github/workflows/schedule_backend.yml b/.github/workflows/schedule_backend.yml index 503a3866df9..b41059b4e36 100644 --- a/.github/workflows/schedule_backend.yml +++ b/.github/workflows/schedule_backend.yml @@ -104,7 +104,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '11' ,'17'] os: [ 'ubuntu-latest', 'windows-latest' ] timeout-minutes: 90 steps: @@ -126,7 +126,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - java: [ '8', '11' ] + java: [ '8', '11' ,'17'] os: [ 'ubuntu-latest' ] timeout-minutes: 90 steps: diff --git a/.mvn/wrapper/maven-wrapper.properties b/.mvn/wrapper/maven-wrapper.properties index aa1e4df95dd..0700da07892 100644 --- a/.mvn/wrapper/maven-wrapper.properties +++ b/.mvn/wrapper/maven-wrapper.properties @@ -15,5 +15,5 @@ # specific language governing permissions and limitations # under the License. -distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.8.4/apache-maven-3.8.4-bin.zip -wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.1.0/maven-wrapper-3.1.0.jar +distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.8.6/apache-maven-3.8.6-bin.zip +wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar diff --git a/config/seatunnel-env.cmd b/config/seatunnel-env.cmd index 79c2d3c117c..5390c0d7641 100644 --- a/config/seatunnel-env.cmd +++ b/config/seatunnel-env.cmd @@ -18,4 +18,29 @@ REM Home directory of spark distribution. if "%SPARK_HOME%" == "" set "SPARK_HOME=C:\Program Files\spark" REM Home directory of flink distribution. -if "%FLINK_HOME%" == "" set "FLINK_HOME=C:\Program Files\flink" \ No newline at end of file +if "%FLINK_HOME%" == "" set "FLINK_HOME=C:\Program Files\flink" + +REM Initialize JAVA_OPTS to avoid unbound variable error. +if not defined JAVA_OPTS ( + set "JAVA_OPTS=" +) + +REM Function to check if the current Java version is 17 or higher. +for /f "tokens=2 delims==" %%v in ('java -XshowSettings:properties -version 2>&1 ^| findstr "java.version"') do ( + set "java_version=%%v" +) +for /f "tokens=1 delims=." %%v in ("%java_version%") do ( + set "java_major_version=%%v" +) + +REM Set JAVA_OPTS if Java version is 17 or newer. +if %java_major_version% geq 17 ( + if defined JAVA_OPTS ( + set "JAVA_OPTS=%JAVA_OPTS% --add-exports=java.base/sun.net.util=ALL-UNNAMED --add-exports=java.rmi/sun.rmi.registry=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED --add-exports=java.security.jgss/sun.security.krb5=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.text=ALL-UNNAMED --add-opens=java.base/java.time=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.locks=ALL-UNNAMED" + ) else ( + set "JAVA_OPTS=--add-exports=java.base/sun.net.util=ALL-UNNAMED --add-exports=java.rmi/sun.rmi.registry=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED --add-exports=java.security.jgss/sun.security.krb5=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.text=ALL-UNNAMED --add-opens=java.base/java.time=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.locks=ALL-UNNAMED" + ) +) + +REM Export JAVA_OPTS +set JAVA_OPTS \ No newline at end of file diff --git a/config/seatunnel-env.sh b/config/seatunnel-env.sh index 1bae8c76254..20583f7dcb8 100644 --- a/config/seatunnel-env.sh +++ b/config/seatunnel-env.sh @@ -20,3 +20,26 @@ SPARK_HOME=${SPARK_HOME:-/opt/spark} # Home directory of flink distribution. FLINK_HOME=${FLINK_HOME:-/opt/flink} + +# Initialize JAVA_OPTS to avoid unbound variable error. +JAVA_OPTS=${JAVA_OPTS:-} + +# Function to check if the current Java version is 17 or higher. +is_java_17_or_newer() { + java_version_output=$(java -version 2>&1) + if [[ $java_version_output == *"version \""* ]]; then + java_version=$(echo $java_version_output | awk -F '"' '/version/ {print $2}') + java_major_version=$(echo $java_version | cut -d'.' -f1) + if [[ "$java_major_version" -ge 17 ]]; then + return 0 + fi + fi + return 1 +} + +# Set JAVA_OPTS if Java version is 17 or newer. +if is_java_17_or_newer; then + JAVA_OPTS="$JAVA_OPTS --add-exports=java.base/sun.net.util=ALL-UNNAMED --add-exports=java.rmi/sun.rmi.registry=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED --add-exports=java.security.jgss/sun.security.krb5=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.text=ALL-UNNAMED --add-opens=java.base/java.time=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.locks=ALL-UNNAMED" +fi + +export JAVA_OPTS \ No newline at end of file diff --git a/docs/en/connector-v2/sink/Hive.md b/docs/en/connector-v2/sink/Hive.md index e3c62294ee6..99bbf739fd4 100644 --- a/docs/en/connector-v2/sink/Hive.md +++ b/docs/en/connector-v2/sink/Hive.md @@ -10,7 +10,7 @@ Write data to Hive. In order to use this connector, You must ensure your spark/flink cluster already integrated hive. The tested hive version is 2.3.9. -If you use SeaTunnel Engine, You need put seatunnel-hadoop3-3.1.4-uber.jar and hive-exec-3.1.3.jar and libfb303-0.9.3.jar in $SEATUNNEL_HOME/lib/ dir. +If you use SeaTunnel Engine, You need put seatunnel-hadoop3-3.3.4-uber.jar and hive-exec-3.1.3.jar and libfb303-0.9.3.jar in $SEATUNNEL_HOME/lib/ dir. ::: ## Key features diff --git a/docs/en/connector-v2/sink/OssFile.md b/docs/en/connector-v2/sink/OssFile.md index f83fdcf4997..a1b94576206 100644 --- a/docs/en/connector-v2/sink/OssFile.md +++ b/docs/en/connector-v2/sink/OssFile.md @@ -17,7 +17,7 @@ ### For SeaTunnel Zeta Engine -1. You must ensure `seatunnel-hadoop3-3.1.4-uber.jar`, `aliyun-sdk-oss-3.4.1.jar`, `hadoop-aliyun-3.1.4.jar` and `jdom-1.1.jar` in `${SEATUNNEL_HOME}/lib/` dir. +1. You must ensure `seatunnel-hadoop3-3.3.4-uber.jar`, `aliyun-sdk-oss-3.4.1.jar`, `hadoop-aliyun-3.1.4.jar` and `jdom-1.1.jar` in `${SEATUNNEL_HOME}/lib/` dir. ## Key features diff --git a/docs/en/connector-v2/source/Hive.md b/docs/en/connector-v2/source/Hive.md index da70cf7aa34..0fb6dbc5211 100644 --- a/docs/en/connector-v2/source/Hive.md +++ b/docs/en/connector-v2/source/Hive.md @@ -10,7 +10,7 @@ Read data from Hive. In order to use this connector, You must ensure your spark/flink cluster already integrated hive. The tested hive version is 2.3.9. -If you use SeaTunnel Engine, You need put seatunnel-hadoop3-3.1.4-uber.jar and hive-exec-3.1.3.jar and libfb303-0.9.3.jar in $SEATUNNEL_HOME/lib/ dir. +If you use SeaTunnel Engine, You need put seatunnel-hadoop3-3.3.4-uber.jar and hive-exec-3.1.3.jar and libfb303-0.9.3.jar in $SEATUNNEL_HOME/lib/ dir. ::: ## Key features diff --git a/docs/en/connector-v2/source/OssFile.md b/docs/en/connector-v2/source/OssFile.md index 3f781eb11a9..3a9b7661d80 100644 --- a/docs/en/connector-v2/source/OssFile.md +++ b/docs/en/connector-v2/source/OssFile.md @@ -17,7 +17,7 @@ ### For SeaTunnel Zeta Engine -1. You must ensure `seatunnel-hadoop3-3.1.4-uber.jar`, `aliyun-sdk-oss-3.4.1.jar`, `hadoop-aliyun-3.1.4.jar` and `jdom-1.1.jar` in `${SEATUNNEL_HOME}/lib/` dir. +1. You must ensure `seatunnel-hadoop3-3.3.4-uber.jar`, `aliyun-sdk-oss-3.4.1.jar`, `hadoop-aliyun-3.1.4.jar` and `jdom-1.1.jar` in `${SEATUNNEL_HOME}/lib/` dir. ## Key features diff --git a/docs/en/faq.md b/docs/en/faq.md index 2e50c9d4618..8892f30c812 100644 --- a/docs/en/faq.md +++ b/docs/en/faq.md @@ -351,3 +351,7 @@ The plugin can be completely independent from SeaTunnel project, so you can writ ## When I import a project, the compiler has the exception "class not found `org.apache.seatunnel.shade.com.typesafe.config.Config`" Run `mvn install` first. In the `seatunnel-config/seatunnel-config-base` subproject, the package `com.typesafe.config` has been relocated to `org.apache.seatunnel.shade.com.typesafe.config` and installed to the maven local repository in the subproject `seatunnel-config/seatunnel-config-shade`. + +## How to solve java.lang.NoClassDefFoundError: org/apache/hadoop/shaded/com/ctc/wstx/io/InputBootstrapper in jdk17 + +Upgrade hadoop version to 3.3.4 diff --git a/docs/zh/faq.md b/docs/zh/faq.md index 3be6ce38e56..8f0a3a83fef 100644 --- a/docs/zh/faq.md +++ b/docs/zh/faq.md @@ -352,3 +352,7 @@ SeaTunnel 拥有完全抽象、结构化的代码实现，很多人都选择 Sea ## 当我导入项目时，编译器出现异常“找不到类`org.apache.seatunnel.shade.com.typesafe.config.Config`” 首先运行“mvn install”。 在 `seatunnel-config/seatunnel-config-base` 子项目中，包 `com.typesafe.config` 已重新定位到 `org.apache.seatunnel.shade.com.typesafe.config` 并安装到 maven 本地存储库 在子项目 `seatunnel-config/seatunnel-config-shade` 中。 + +## JDK17环境下，提示java.lang.NoClassDefFoundError: org/apache/hadoop/shaded/com/ctc/wstx/io/InputBootstrapper怎么处理？ + +升级hadoop客户端版本为3.3.4。 diff --git a/pom.xml b/pom.xml index 41854d78fce..d12d47bf234 100644 --- a/pom.xml +++ b/pom.xml @@ -65,7 +65,7 @@ ${java.version} ${java.version} - 1.7.25 + 1.7.36 2.17.1 3.4.4 1.2.17 @@ -91,7 +91,7 @@ false false false - 2.22.2 + 3.3.1 2.22.2 1.6.8 3.0.1 @@ -133,7 +133,9 @@ 2.29.0 4.5 2.7.0 + 4.12.0 4.0.16 + false true @@ -471,7 +473,7 @@ org.apache.seatunnel - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber ${project.version} optional provided @@ -586,6 +588,7 @@ ch.qos.logback:logback-core org.apache.logging.log4j:log4j-to-slf4j + ${surefire.argLine} @@ -938,6 +941,52 @@ false + + jdk8 + + 1.8 + + + + + + + jdk11 + + 11 + + + + + + + jdk17 + + [17,) + + + --add-exports java.base/sun.net.util=ALL-UNNAMED + --add-exports java.rmi/sun.rmi.registry=ALL-UNNAMED + --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED + --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED + --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED + --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED + --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED + --add-exports java.security.jgss/sun.security.krb5=ALL-UNNAMED + --add-opens java.base/java.lang=ALL-UNNAMED + --add-opens java.base/java.net=ALL-UNNAMED + --add-opens java.base/java.io=ALL-UNNAMED + --add-opens java.base/java.nio=ALL-UNNAMED + --add-opens java.base/sun.nio.ch=ALL-UNNAMED + --add-opens java.base/java.lang.reflect=ALL-UNNAMED + --add-opens java.base/java.text=ALL-UNNAMED + --add-opens java.base/java.time=ALL-UNNAMED + --add-opens java.base/java.util=ALL-UNNAMED + --add-opens java.base/java.util.concurrent=ALL-UNNAMED + --add-opens java.base/java.util.concurrent.atomic=ALL-UNNAMED + --add-opens java.base/java.util.concurrent.locks=ALL-UNNAMED + + diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/serialize/DefaultSeaTunnelRowDeserializer.java b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/serialize/DefaultSeaTunnelRowDeserializer.java index a531befe9a6..a764a38d88f 100644 --- a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/serialize/DefaultSeaTunnelRowDeserializer.java +++ b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/serialize/DefaultSeaTunnelRowDeserializer.java @@ -23,8 +23,10 @@ import org.apache.seatunnel.api.table.type.SeaTunnelRow; import org.apache.seatunnel.api.table.type.SeaTunnelRowType; import org.apache.seatunnel.common.exception.CommonError; +import org.apache.seatunnel.common.utils.DateTimeUtils; +import org.apache.seatunnel.common.utils.DateUtils; -import lombok.AllArgsConstructor; +import lombok.RequiredArgsConstructor; import software.amazon.awssdk.core.SdkBytes; import software.amazon.awssdk.services.dynamodb.model.AttributeValue; @@ -34,16 +36,29 @@ import java.time.LocalDate; import java.time.LocalDateTime; import java.time.LocalTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatterBuilder; +import java.time.temporal.ChronoField; +import java.time.temporal.TemporalAccessor; +import java.time.temporal.TemporalQueries; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -@AllArgsConstructor +@RequiredArgsConstructor public class DefaultSeaTunnelRowDeserializer implements SeaTunnelRowDeserializer { private final SeaTunnelRowType typeInfo; + public static DateTimeFormatter TIME_FORMAT = + new DateTimeFormatterBuilder() + .appendPattern("HH:mm:ss") + .appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, true) + .toFormatter(); + + public Map fieldFormatterMap = new HashMap<>(); + @Override public SeaTunnelRow deserialize(Map item) { SeaTunnelDataType[] seaTunnelDataTypes = typeInfo.getFieldTypes(); @@ -63,7 +78,9 @@ private List convertRow( } private Object convert( - String field, SeaTunnelDataType seaTunnelDataType, AttributeValue attributeValue) { + String fieldName, + SeaTunnelDataType seaTunnelDataType, + AttributeValue attributeValue) { if (attributeValue.type().equals(AttributeValue.Type.NUL)) { return null; } @@ -90,11 +107,26 @@ private Object convert( case STRING: return attributeValue.s(); case TIME: - return LocalTime.parse(attributeValue.s()); + return TIME_FORMAT.parse(attributeValue.s()); case DATE: - return LocalDate.parse(attributeValue.s()); + DateTimeFormatter dateFormatter = fieldFormatterMap.get(fieldName); + if (dateFormatter == null) { + dateFormatter = DateUtils.matchDateFormatter(attributeValue.s()); + fieldFormatterMap.put(fieldName, dateFormatter); + } + + return dateFormatter.parse(attributeValue.s()).query(TemporalQueries.localDate()); case TIMESTAMP: - return LocalDateTime.parse(attributeValue.s()); + DateTimeFormatter dateTimeFormatter = fieldFormatterMap.get(fieldName); + if (dateTimeFormatter == null) { + dateTimeFormatter = DateTimeUtils.matchDateTimeFormatter(attributeValue.s()); + fieldFormatterMap.put(fieldName, dateTimeFormatter); + } + + TemporalAccessor parsedTimestamp = dateTimeFormatter.parse(attributeValue.s()); + LocalTime localTime = parsedTimestamp.query(TemporalQueries.localTime()); + LocalDate localDate = parsedTimestamp.query(TemporalQueries.localDate()); + return LocalDateTime.of(localDate, localTime); case BYTES: return attributeValue.b().asByteArray(); case MAP: @@ -106,7 +138,7 @@ private Object convert( seatunnelMap.put( s, convert( - field, + fieldName, ((MapType) seaTunnelDataType).getValueType(), attributeValueInfo)); }); @@ -126,7 +158,7 @@ private Object convert( array, index, convert( - field, + fieldName, ((ArrayType) seaTunnelDataType).getElementType(), datas.get(index))); } @@ -149,7 +181,7 @@ private Object convert( return array; default: throw CommonError.convertToSeaTunnelTypeError( - "AmazonDynamodb", seaTunnelDataType.getSqlType().toString(), field); + "AmazonDynamodb", seaTunnelDataType.getSqlType().toString(), fieldName); } } } diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/serialize/DefaultSeaTunnelRowSerializer.java b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/serialize/DefaultSeaTunnelRowSerializer.java index 5cef8d2bc6b..c02b97290c4 100644 --- a/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/serialize/DefaultSeaTunnelRowSerializer.java +++ b/seatunnel-connectors-v2/connector-amazondynamodb/src/main/java/org/apache/seatunnel/connectors/seatunnel/amazondynamodb/serialize/DefaultSeaTunnelRowSerializer.java @@ -22,6 +22,7 @@ import org.apache.seatunnel.api.table.type.SeaTunnelDataType; import org.apache.seatunnel.api.table.type.SeaTunnelRow; import org.apache.seatunnel.api.table.type.SeaTunnelRowType; +import org.apache.seatunnel.api.table.type.SqlType; import org.apache.seatunnel.common.exception.CommonError; import org.apache.seatunnel.connectors.seatunnel.amazondynamodb.config.AmazonDynamoDBSourceOptions; @@ -29,6 +30,12 @@ import software.amazon.awssdk.services.dynamodb.model.AttributeValue; import software.amazon.awssdk.services.dynamodb.model.PutItemRequest; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatterBuilder; +import java.time.temporal.ChronoField; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -37,12 +44,22 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import static java.time.format.DateTimeFormatter.ISO_LOCAL_DATE; +import static java.time.format.DateTimeFormatter.ISO_LOCAL_DATE_TIME; + public class DefaultSeaTunnelRowSerializer implements SeaTunnelRowSerializer { private final SeaTunnelRowType seaTunnelRowType; private final AmazonDynamoDBSourceOptions amazondynamodbSourceOptions; private final List measurementsType; + private static final int MAX_TIME_PRECISION = 9; + public static final DateTimeFormatter TIME_FORMAT = + new DateTimeFormatterBuilder() + .appendPattern("HH:mm:ss") + .appendFraction(ChronoField.NANO_OF_SECOND, 0, MAX_TIME_PRECISION, true) + .toFormatter(); + public DefaultSeaTunnelRowSerializer( SeaTunnelRowType seaTunnelRowType, AmazonDynamoDBSourceOptions amazondynamodbSourceOptions) { @@ -125,6 +142,19 @@ private AttributeValue convertItem( .n(Integer.toString(((Number) value).intValue())) .build(); case S: + if (seaTunnelDataType.getSqlType().equals(SqlType.DATE)) { + return AttributeValue.builder() + .s(ISO_LOCAL_DATE.format((LocalDate) value)) + .build(); + } else if (seaTunnelDataType.getSqlType().equals(SqlType.TIME)) { + return AttributeValue.builder() + .s(TIME_FORMAT.format((LocalTime) value)) + .build(); + } else if (seaTunnelDataType.getSqlType().equals(SqlType.TIMESTAMP)) { + return AttributeValue.builder() + .s(ISO_LOCAL_DATE_TIME.format((LocalDateTime) value)) + .build(); + } return AttributeValue.builder().s(String.valueOf(value)).build(); case BOOL: return AttributeValue.builder().bool((Boolean) value).build(); diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/pom.xml b/seatunnel-connectors-v2/connector-file/connector-file-base/pom.xml index f091e7023d9..9d7f8897fd1 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-base/pom.xml +++ b/seatunnel-connectors-v2/connector-file/connector-file-base/pom.xml @@ -45,7 +45,7 @@ org.apache.seatunnel - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber ${project.version} optional provided @@ -116,7 +116,7 @@ org.apache.seatunnel - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber ${project.version} optional provided diff --git a/seatunnel-connectors-v2/connector-file/connector-file-ftp/pom.xml b/seatunnel-connectors-v2/connector-file/connector-file-ftp/pom.xml index 0ab14d7e04b..7a83d4b3777 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-ftp/pom.xml +++ b/seatunnel-connectors-v2/connector-file/connector-file-ftp/pom.xml @@ -37,7 +37,7 @@ org.apache.seatunnel - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber ${project.version} optional provided diff --git a/seatunnel-connectors-v2/connector-file/connector-file-hadoop/pom.xml b/seatunnel-connectors-v2/connector-file/connector-file-hadoop/pom.xml index 59da8bf8446..73748aa4cc1 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-hadoop/pom.xml +++ b/seatunnel-connectors-v2/connector-file/connector-file-hadoop/pom.xml @@ -37,7 +37,7 @@ org.apache.seatunnel - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber ${project.version} optional provided diff --git a/seatunnel-connectors-v2/connector-file/connector-file-local/pom.xml b/seatunnel-connectors-v2/connector-file/connector-file-local/pom.xml index 9c242cd3a2a..effb823611c 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-local/pom.xml +++ b/seatunnel-connectors-v2/connector-file/connector-file-local/pom.xml @@ -37,7 +37,7 @@ org.apache.seatunnel - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber ${project.version} optional provided diff --git a/seatunnel-connectors-v2/connector-file/connector-file-oss/pom.xml b/seatunnel-connectors-v2/connector-file/connector-file-oss/pom.xml index c904257b90d..330a9f45db7 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-oss/pom.xml +++ b/seatunnel-connectors-v2/connector-file/connector-file-oss/pom.xml @@ -44,7 +44,7 @@ org.apache.seatunnel - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber ${project.version} optional provided diff --git a/seatunnel-connectors-v2/connector-file/connector-file-s3/pom.xml b/seatunnel-connectors-v2/connector-file/connector-file-s3/pom.xml index fbf0016fced..9ad842fa1d5 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-s3/pom.xml +++ b/seatunnel-connectors-v2/connector-file/connector-file-s3/pom.xml @@ -54,7 +54,7 @@ org.apache.seatunnel - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber ${project.version} optional provided diff --git a/seatunnel-connectors-v2/connector-file/connector-file-sftp/pom.xml b/seatunnel-connectors-v2/connector-file/connector-file-sftp/pom.xml index 816184caf00..c1d744de7b7 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-sftp/pom.xml +++ b/seatunnel-connectors-v2/connector-file/connector-file-sftp/pom.xml @@ -37,7 +37,7 @@ org.apache.seatunnel - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber ${project.version} optional provided diff --git a/seatunnel-connectors-v2/connector-hive/pom.xml b/seatunnel-connectors-v2/connector-hive/pom.xml index a1e12d019de..16c7cd09fb0 100644 --- a/seatunnel-connectors-v2/connector-hive/pom.xml +++ b/seatunnel-connectors-v2/connector-hive/pom.xml @@ -42,7 +42,7 @@ org.apache.seatunnel - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber @@ -63,7 +63,7 @@ org.apache.seatunnel - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber ${project.version} optional provided diff --git a/seatunnel-connectors-v2/connector-iceberg/pom.xml b/seatunnel-connectors-v2/connector-iceberg/pom.xml index 309900b09c7..32d0487f021 100644 --- a/seatunnel-connectors-v2/connector-iceberg/pom.xml +++ b/seatunnel-connectors-v2/connector-iceberg/pom.xml @@ -86,6 +86,12 @@ orc-core 1.7.5 nohive + + + org.apache.hadoop + hadoop-client-api + + org.apache.iceberg @@ -134,11 +140,27 @@ org.apache.avro * + + org.apache.hadoop + hadoop-common + + + org.apache.hadoop + hadoop-yarn-api + + + org.apache.hadoop + hadoop-yarn-common + + + org.apache.hadoop + hadoop-annotations + org.apache.seatunnel - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber ${project.version} optional provided diff --git a/seatunnel-connectors-v2/connector-jdbc/pom.xml b/seatunnel-connectors-v2/connector-jdbc/pom.xml index cfe0ed44ecd..7cddf8804e4 100644 --- a/seatunnel-connectors-v2/connector-jdbc/pom.xml +++ b/seatunnel-connectors-v2/connector-jdbc/pom.xml @@ -30,8 +30,8 @@ SeaTunnel : Connectors V2 : Jdbc - 8.0.27 - 42.4.3 + 8.0.32 + 42.7.2 8.1.2.141 9.2.1.jre8 5.2.5-HBase-2.x diff --git a/seatunnel-connectors-v2/connector-kudu/pom.xml b/seatunnel-connectors-v2/connector-kudu/pom.xml index 9dcdc87853a..2207614b95e 100644 --- a/seatunnel-connectors-v2/connector-kudu/pom.xml +++ b/seatunnel-connectors-v2/connector-kudu/pom.xml @@ -59,7 +59,7 @@ org.apache.seatunnel - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber ${project.version} optional provided diff --git a/seatunnel-connectors-v2/connector-kudu/src/main/java/org/apache/seatunnel/connectors/seatunnel/kudu/catalog/KuduCatalog.java b/seatunnel-connectors-v2/connector-kudu/src/main/java/org/apache/seatunnel/connectors/seatunnel/kudu/catalog/KuduCatalog.java index b5ba121fb51..930b84497ea 100644 --- a/seatunnel-connectors-v2/connector-kudu/src/main/java/org/apache/seatunnel/connectors/seatunnel/kudu/catalog/KuduCatalog.java +++ b/seatunnel-connectors-v2/connector-kudu/src/main/java/org/apache/seatunnel/connectors/seatunnel/kudu/catalog/KuduCatalog.java @@ -34,7 +34,7 @@ import org.apache.seatunnel.connectors.seatunnel.kudu.kuduclient.KuduTypeMapper; import org.apache.seatunnel.connectors.seatunnel.kudu.util.KuduUtil; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.kudu.ColumnSchema; import org.apache.kudu.Schema; import org.apache.kudu.client.KuduClient; diff --git a/seatunnel-connectors-v2/connector-kudu/src/main/java/org/apache/seatunnel/connectors/seatunnel/kudu/util/KuduUtil.java b/seatunnel-connectors-v2/connector-kudu/src/main/java/org/apache/seatunnel/connectors/seatunnel/kudu/util/KuduUtil.java index 27374d0a3a5..06963153e78 100644 --- a/seatunnel-connectors-v2/connector-kudu/src/main/java/org/apache/seatunnel/connectors/seatunnel/kudu/util/KuduUtil.java +++ b/seatunnel-connectors-v2/connector-kudu/src/main/java/org/apache/seatunnel/connectors/seatunnel/kudu/util/KuduUtil.java @@ -24,7 +24,7 @@ import org.apache.seatunnel.connectors.seatunnel.kudu.exception.KuduConnectorErrorCode; import org.apache.seatunnel.connectors.seatunnel.kudu.exception.KuduConnectorException; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authentication.util.KerberosName; diff --git a/seatunnel-connectors-v2/connector-paimon/pom.xml b/seatunnel-connectors-v2/connector-paimon/pom.xml index 80934e68a2b..34698108a02 100644 --- a/seatunnel-connectors-v2/connector-paimon/pom.xml +++ b/seatunnel-connectors-v2/connector-paimon/pom.xml @@ -56,7 +56,7 @@ org.apache.seatunnel - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber optional provided diff --git a/seatunnel-connectors-v2/connector-starrocks/pom.xml b/seatunnel-connectors-v2/connector-starrocks/pom.xml index 08e49bc0f02..320f3acfc18 100644 --- a/seatunnel-connectors-v2/connector-starrocks/pom.xml +++ b/seatunnel-connectors-v2/connector-starrocks/pom.xml @@ -33,7 +33,7 @@ connector.starrocks 4.5.13 4.4.4 - 8.0.16 + 8.0.32 1.0.1 5.0.0 diff --git a/seatunnel-core/seatunnel-flink-starter/pom.xml b/seatunnel-core/seatunnel-flink-starter/pom.xml index 3b4f5442c32..b0f75fc2a82 100644 --- a/seatunnel-core/seatunnel-flink-starter/pom.xml +++ b/seatunnel-core/seatunnel-flink-starter/pom.xml @@ -92,7 +92,7 @@ org.apache.logging.log4j:log4j-slf4j-impl org.apache.logging.log4j:log4j-1.2-api org.apache.logging.log4j:log4j-to-slf4j - org.apache.seatunnel:seatunnel-hadoop3-3.1.4-uber + org.apache.seatunnel:seatunnel-hadoop3-3.3.4-uber diff --git a/seatunnel-core/seatunnel-spark-starter/pom.xml b/seatunnel-core/seatunnel-spark-starter/pom.xml index 4502d72c976..e4b77a59bf9 100644 --- a/seatunnel-core/seatunnel-spark-starter/pom.xml +++ b/seatunnel-core/seatunnel-spark-starter/pom.xml @@ -99,7 +99,7 @@ org.apache.logging.log4j:log4j-slf4j-impl org.apache.logging.log4j:log4j-to-slf4j - org.apache.seatunnel:seatunnel-hadoop3-3.1.4-uber + org.apache.seatunnel:seatunnel-hadoop3-3.3.4-uber diff --git a/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-3-starter/src/test/java/org/apache/seatunnel/core/starter/spark/SparkCommandArgsTest.java b/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-3-starter/src/test/java/org/apache/seatunnel/core/starter/spark/SparkCommandArgsTest.java index da997e33559..c263b05b481 100644 --- a/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-3-starter/src/test/java/org/apache/seatunnel/core/starter/spark/SparkCommandArgsTest.java +++ b/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-3-starter/src/test/java/org/apache/seatunnel/core/starter/spark/SparkCommandArgsTest.java @@ -34,7 +34,7 @@ import static org.apache.seatunnel.api.common.CommonOptions.PLUGIN_NAME; @DisabledOnJre( - value = JRE.JAVA_11, + value = {JRE.JAVA_11, JRE.JAVA_17}, disabledReason = "We should update apache common lang3 version to 3.8 to avoid NPE, " + "see https://github.com/apache/commons-lang/commit/50ce8c44e1601acffa39f5568f0fc140aade0564") diff --git a/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-3-starter/src/test/java/org/apache/seatunnel/core/starter/spark/multitable/MultiTableSinkTest.java b/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-3-starter/src/test/java/org/apache/seatunnel/core/starter/spark/multitable/MultiTableSinkTest.java index 61ed44515c7..08734be5f98 100644 --- a/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-3-starter/src/test/java/org/apache/seatunnel/core/starter/spark/multitable/MultiTableSinkTest.java +++ b/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-3-starter/src/test/java/org/apache/seatunnel/core/starter/spark/multitable/MultiTableSinkTest.java @@ -46,7 +46,7 @@ public class MultiTableSinkTest { @Test @DisabledOnJre( - value = JRE.JAVA_11, + value = {JRE.JAVA_11, JRE.JAVA_17}, disabledReason = "We should update apache common lang3 version to 3.8 to avoid NPE, " + "see https://github.com/apache/commons-lang/commit/50ce8c44e1601acffa39f5568f0fc140aade0564") diff --git a/seatunnel-core/seatunnel-starter/pom.xml b/seatunnel-core/seatunnel-starter/pom.xml index 85f9d7d4a13..88120743042 100644 --- a/seatunnel-core/seatunnel-starter/pom.xml +++ b/seatunnel-core/seatunnel-starter/pom.xml @@ -116,7 +116,7 @@ commons-logging:* ch.qos.logback:* org.apache.logging.log4j:log4j-to-slf4j - org.apache.seatunnel:seatunnel-hadoop3-3.1.4-uber + org.apache.seatunnel:seatunnel-hadoop3-3.3.4-uber diff --git a/seatunnel-dist/pom.xml b/seatunnel-dist/pom.xml index a5dd203f837..e94838ad12d 100644 --- a/seatunnel-dist/pom.xml +++ b/seatunnel-dist/pom.xml @@ -83,8 +83,8 @@ - 8.0.27 - 42.4.3 + 8.0.32 + 42.7.2 2.5.1 8.1.2.141 9.2.1.jre8 @@ -702,7 +702,7 @@ com.amazonaws aws-java-sdk-bundle - 1.11.271 + ${aws-java-sdk.version} provided @@ -725,7 +725,7 @@ org.apache.seatunnel - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber ${project.version} optional provided @@ -883,12 +883,6 @@ ${project.version} provided - - com.amazonaws - aws-java-sdk-bundle - ${aws-java-sdk.version} - provided - org.apache.seatunnel @@ -913,7 +907,7 @@ org.apache.seatunnel - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber ${project.version} optional provided diff --git a/seatunnel-dist/src/main/assembly/assembly-bin-ci.xml b/seatunnel-dist/src/main/assembly/assembly-bin-ci.xml index cc48ac86a2c..cb9b2b8b779 100644 --- a/seatunnel-dist/src/main/assembly/assembly-bin-ci.xml +++ b/seatunnel-dist/src/main/assembly/assembly-bin-ci.xml @@ -169,7 +169,7 @@ org.apache.seatunnel:seatunnel-transforms-v2:jar org.apache.hadoop:hadoop-aws:jar com.amazonaws:aws-java-sdk-bundle:jar - org.apache.seatunnel:seatunnel-hadoop3-3.1.4-uber:jar:*:optional + org.apache.seatunnel:seatunnel-hadoop3-3.3.4-uber:jar:*:optional org.apache.hadoop:hadoop-aliyun:jar com.aliyun.oss:aliyun-sdk-oss:jar diff --git a/seatunnel-dist/src/main/assembly/assembly-bin.xml b/seatunnel-dist/src/main/assembly/assembly-bin.xml index 30fc5a6336a..a67db42784f 100644 --- a/seatunnel-dist/src/main/assembly/assembly-bin.xml +++ b/seatunnel-dist/src/main/assembly/assembly-bin.xml @@ -168,7 +168,7 @@ false org.apache.seatunnel:seatunnel-transforms-v2:jar - org.apache.seatunnel:seatunnel-hadoop3-3.1.4-uber:jar:*:optional + org.apache.seatunnel:seatunnel-hadoop3-3.3.4-uber:jar:*:optional ${artifact.file.name} /lib diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-amazondynamodb-e2e/src/test/java/org/apache/seatunnel/e2e/connector/amazondynamodb/AmazondynamodbIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-amazondynamodb-e2e/src/test/java/org/apache/seatunnel/e2e/connector/amazondynamodb/AmazondynamodbIT.java index 07ab5c05302..22d77039bdc 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-amazondynamodb-e2e/src/test/java/org/apache/seatunnel/e2e/connector/amazondynamodb/AmazondynamodbIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-amazondynamodb-e2e/src/test/java/org/apache/seatunnel/e2e/connector/amazondynamodb/AmazondynamodbIT.java @@ -26,6 +26,7 @@ import org.apache.seatunnel.api.table.type.SeaTunnelDataType; import org.apache.seatunnel.api.table.type.SeaTunnelRow; import org.apache.seatunnel.api.table.type.SeaTunnelRowType; +import org.apache.seatunnel.api.table.type.SqlType; import org.apache.seatunnel.e2e.common.TestResource; import org.apache.seatunnel.e2e.common.TestSuiteBase; import org.apache.seatunnel.e2e.common.container.TestContainer; @@ -69,6 +70,11 @@ import java.net.URI; import java.time.LocalDate; import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatterBuilder; +import java.time.temporal.ChronoField; +import java.time.temporal.ChronoUnit; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -77,6 +83,8 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import static java.time.format.DateTimeFormatter.ISO_LOCAL_DATE; +import static java.time.format.DateTimeFormatter.ISO_LOCAL_DATE_TIME; import static org.awaitility.Awaitility.given; @Slf4j @@ -92,6 +100,13 @@ public class AmazondynamodbIT extends TestSuiteBase implements TestResource { private GenericContainer dynamoDB; protected DynamoDbClient dynamoDbClient; + private static final int MAX_TIME_PRECISION = 9; + public static final DateTimeFormatter TIME_FORMAT = + new DateTimeFormatterBuilder() + .appendPattern("HH:mm:ss") + .appendFraction(ChronoField.NANO_OF_SECOND, 0, MAX_TIME_PRECISION, true) + .toFormatter(); + @TestTemplate public void testAmazondynamodb(TestContainer container) throws Exception { assertHasData(SOURCE_TABLE); @@ -254,7 +269,7 @@ private Map randomRow() { BigDecimal.valueOf(11, 1), "test".getBytes(), LocalDate.now(), - LocalDateTime.now() + LocalDateTime.now().truncatedTo(ChronoUnit.MILLIS) }); Map data = new HashMap<>(seatunnelRowType.getTotalFields()); @@ -325,6 +340,19 @@ private AttributeValue convertItem( .n(Integer.toString(((Number) value).intValue())) .build(); case S: + if (seaTunnelDataType.getSqlType().equals(SqlType.DATE)) { + return AttributeValue.builder() + .s(ISO_LOCAL_DATE.format((LocalDate) value)) + .build(); + } else if (seaTunnelDataType.getSqlType().equals(SqlType.TIME)) { + return AttributeValue.builder() + .s(TIME_FORMAT.format((LocalTime) value)) + .build(); + } else if (seaTunnelDataType.getSqlType().equals(SqlType.TIMESTAMP)) { + return AttributeValue.builder() + .s(ISO_LOCAL_DATE_TIME.format((LocalDateTime) value)) + .build(); + } return AttributeValue.builder().s(String.valueOf(value)).build(); case BOOL: return AttributeValue.builder().bool((Boolean) value).build(); diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mongodb-e2e/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mongodb-e2e/pom.xml index a8814c11ee6..45cd5632c82 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mongodb-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mongodb-e2e/pom.xml @@ -26,7 +26,7 @@ SeaTunnel : E2E : Connector V2 : CDC Mongodb - 8.0.16 + 8.0.32 diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/pom.xml index af85d92acef..8d9684648c0 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/pom.xml @@ -26,7 +26,7 @@ SeaTunnel : E2E : Connector V2 : Doris - 8.0.31 + 8.0.32 diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/pom.xml index ea120abdd31..e1867539a31 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/pom.xml @@ -37,10 +37,16 @@ connector-file-local ${project.version} test + + + io.airlift + aircompressor + + org.apache.seatunnel - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber ${project.version} optional test @@ -57,6 +63,12 @@ ${project.version} test + + io.airlift + aircompressor + 0.21 + test + diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/local/LocalFileIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/local/LocalFileIT.java index e278159efab..3311fa7ab45 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/local/LocalFileIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/local/LocalFileIT.java @@ -36,6 +36,8 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.TestTemplate; +import org.junit.jupiter.api.condition.DisabledOnJre; +import org.junit.jupiter.api.condition.JRE; import org.testcontainers.containers.GenericContainer; import org.testcontainers.shaded.com.github.dockerjava.core.command.ExecStartResultCallback; @@ -60,6 +62,7 @@ type = {}, disabledReason = "The apache-compress version is not compatible with apache-poi") @Slf4j +@DisabledOnJre(JRE.JAVA_17) public class LocalFileIT extends TestSuiteBase { private GenericContainer baseContainer; diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-oss-e2e/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-oss-e2e/pom.xml index cea8e9a4a9a..e4f7eff3f0a 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-oss-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-oss-e2e/pom.xml @@ -49,7 +49,7 @@ org.apache.seatunnel - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber ${project.version} optional test diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-s3-e2e/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-s3-e2e/pom.xml index e33b6273a89..71d0366fe6b 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-s3-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-s3-e2e/pom.xml @@ -55,7 +55,7 @@ org.apache.seatunnel - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber ${project.version} optional test diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hive-e2e/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hive-e2e/pom.xml index bd9a112b6ac..0ecddaf03df 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hive-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hive-e2e/pom.xml @@ -40,7 +40,7 @@ org.apache.seatunnel - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber ${project.version} optional test diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-e2e/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-e2e/pom.xml index 11c147432d0..26618e1332e 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-e2e/pom.xml @@ -63,7 +63,7 @@ org.apache.seatunnel - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber ${project.version} optional test diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-hadoop3-e2e/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-hadoop3-e2e/pom.xml index cb6e73c3177..43bbf84fbb4 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-hadoop3-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-hadoop3-e2e/pom.xml @@ -25,10 +25,6 @@ connector-iceberg-hadoop3-e2e SeaTunnel : E2E : Connector V2 : Iceberg : Hadoop3 - - 3.3.4 - - org.apache.seatunnel @@ -49,16 +45,11 @@ test - org.apache.hadoop - hadoop-client - ${hadoop-client.version} + org.apache.seatunnel + seatunnel-hadoop3-3.3.4-uber + ${project.version} + optional test - - - org.slf4j - slf4j-reload4j - - diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-s3-e2e/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-s3-e2e/pom.xml index a44c8d630fa..af809997558 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-s3-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-s3-e2e/pom.xml @@ -28,7 +28,6 @@ 1.19.1 8.5.6 - 3.1.4 @@ -65,21 +64,10 @@ test - org.apache.hadoop - hadoop-client - ${hadoop3.version} - test - - - org.slf4j - slf4j-reload4j - - - - - org.apache.hadoop - hadoop-aws - ${hadoop3.version} + org.apache.seatunnel + seatunnel-hadoop3-3.3.4-uber + ${project.version} + optional test diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-s3-e2e/src/test/java/org/apache/seatunnel/e2e/connector/iceberg/s3/IcebergSourceIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-s3-e2e/src/test/java/org/apache/seatunnel/e2e/connector/iceberg/s3/IcebergSourceIT.java index 35101528929..1edee84722d 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-s3-e2e/src/test/java/org/apache/seatunnel/e2e/connector/iceberg/s3/IcebergSourceIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-s3-e2e/src/test/java/org/apache/seatunnel/e2e/connector/iceberg/s3/IcebergSourceIT.java @@ -83,12 +83,12 @@ value = {TestContainerId.SPARK_2_4}, type = {EngineType.FLINK, EngineType.SEATUNNEL}, disabledReason = - "Needs hadoop-aws,aws-java-sdk jar for flink, spark2.4. For the seatunnel engine, it crashes on seatunnel-hadoop3-3.1.4-uber.jar.") + "Needs hadoop-aws,aws-java-sdk jar for flink, spark2.4. For the seatunnel engine, it crashes on seatunnel-hadoop3-3.3.4-uber.jar.") @Slf4j public class IcebergSourceIT extends TestSuiteBase implements TestResource { public static final String HADOOP_AWS_DOWNLOAD = - "https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.1.4/hadoop-aws-3.1.4.jar"; + "https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.3.4/hadoop-aws-3.3.4.jar"; public static final String AWS_SDK_DOWNLOAD = "https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/1.11.271/aws-java-sdk-bundle-1.11.271.jar"; diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/pom.xml index fa2e1930cce..128f167fe8b 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/pom.xml @@ -109,11 +109,6 @@ ${testcontainer.version} test - - mysql - mysql-connector-java - test - diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-paimon-e2e/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-paimon-e2e/pom.xml index 69ea9a9f74f..a026851058a 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-paimon-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-paimon-e2e/pom.xml @@ -40,7 +40,7 @@ org.apache.seatunnel - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber optional test diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rabbitmq-e2e/src/test/java/org/apache/seatunnel/e2e/connector/rabbitmq/RabbitmqIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rabbitmq-e2e/src/test/java/org/apache/seatunnel/e2e/connector/rabbitmq/RabbitmqIT.java index 7052aa9bef8..f679e045e6e 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rabbitmq-e2e/src/test/java/org/apache/seatunnel/e2e/connector/rabbitmq/RabbitmqIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rabbitmq-e2e/src/test/java/org/apache/seatunnel/e2e/connector/rabbitmq/RabbitmqIT.java @@ -27,6 +27,8 @@ import org.apache.seatunnel.api.table.type.SeaTunnelRow; import org.apache.seatunnel.api.table.type.SeaTunnelRowType; import org.apache.seatunnel.common.Handover; +import org.apache.seatunnel.common.utils.DateTimeUtils; +import org.apache.seatunnel.common.utils.DateTimeUtils.Formatter; import org.apache.seatunnel.connectors.seatunnel.rabbitmq.client.RabbitmqClient; import org.apache.seatunnel.connectors.seatunnel.rabbitmq.config.RabbitmqConfig; import org.apache.seatunnel.e2e.common.TestResource; @@ -80,36 +82,9 @@ public class RabbitmqIT extends TestSuiteBase implements TestResource { generateTestDataSet(); private static final JsonSerializationSchema JSON_SERIALIZATION_SCHEMA = new JsonSerializationSchema(TEST_DATASET.getKey()); - - private GenericContainer rabbitmqContainer; Connection connection; RabbitmqClient rabbitmqClient; - - @BeforeAll - @Override - public void startUp() throws Exception { - this.rabbitmqContainer = - new GenericContainer<>(DockerImageName.parse(IMAGE)) - .withNetwork(NETWORK) - .withNetworkAliases(HOST) - .withExposedPorts(PORT, 15672) - .withLogConsumer(new Slf4jLogConsumer(DockerLoggerFactory.getLogger(IMAGE))) - .waitingFor( - new HostPortWaitStrategy() - .withStartupTimeout(Duration.ofMinutes(2))); - Startables.deepStart(Stream.of(rabbitmqContainer)).join(); - log.info("rabbitmq container started"); - this.initRabbitMQ(); - } - - private void initSourceData() throws IOException, InterruptedException { - List rows = TEST_DATASET.getValue(); - for (int i = 0; i < rows.size(); i++) { - rabbitmqClient.write( - new String(JSON_SERIALIZATION_SCHEMA.serialize(rows.get(1))) - .getBytes(StandardCharsets.UTF_8)); - } - } + private GenericContainer rabbitmqContainer; private static Pair> generateTestDataSet() { @@ -152,6 +127,8 @@ private static Pair> generateTestDataSet() List rows = new ArrayList<>(); for (int i = 0; i < 10; i++) { + final String datetime = "2023-12-22 00:00:00"; + LocalDateTime parse = DateTimeUtils.parse(datetime, Formatter.YYYY_MM_DD_HH_MM_SS); SeaTunnelRow row = new SeaTunnelRow( new Object[] { @@ -169,13 +146,48 @@ private static Pair> generateTestDataSet() BigDecimal.valueOf(11, 1), "test".getBytes(), LocalDate.now(), - LocalDateTime.now() + parse }); rows.add(row); } return Pair.of(rowType, rows); } + @BeforeAll + @Override + public void startUp() throws Exception { + this.rabbitmqContainer = + new GenericContainer<>(DockerImageName.parse(IMAGE)) + .withNetwork(NETWORK) + .withNetworkAliases(HOST) + .withExposedPorts(PORT, 15672) + .withLogConsumer(new Slf4jLogConsumer(DockerLoggerFactory.getLogger(IMAGE))) + .waitingFor( + new HostPortWaitStrategy() + .withStartupTimeout(Duration.ofMinutes(2))); + Startables.deepStart(Stream.of(rabbitmqContainer)).join(); + log.info("rabbitmq container started"); + this.initRabbitMQ(); + } + + @AfterAll + @Override + public void tearDown() throws Exception { + if (connection != null) { + connection.close(); + } + rabbitmqContainer.close(); + } + + private void initSourceData() throws IOException, InterruptedException { + List rows = TEST_DATASET.getValue(); + for (int i = 0; i < rows.size(); i++) { + rabbitmqClient.write( + new String(JSON_SERIALIZATION_SCHEMA.serialize(rows.get(1))) + .getBytes(StandardCharsets.UTF_8)); + } + } + private void initRabbitMQ() { try { RabbitmqConfig config = new RabbitmqConfig(); @@ -207,15 +219,6 @@ private RabbitmqClient initSinkRabbitMQ() { } } - @AfterAll - @Override - public void tearDown() throws Exception { - if (connection != null) { - connection.close(); - } - rabbitmqContainer.close(); - } - @TestTemplate public void testRabbitMQ(TestContainer container) throws Exception { // send data to source queue before executeJob start in every testContainer @@ -245,13 +248,8 @@ public void testRabbitMQ(TestContainer container) throws Exception { sinkRabbitmqClient.close(); // assert source and sink data Assertions.assertTrue(resultSet.size() > 0); - Assertions.assertTrue( - resultSet.stream() - .findAny() - .get() - .equals( - new String( - JSON_SERIALIZATION_SCHEMA.serialize( - TEST_DATASET.getValue().get(1))))); + Assertions.assertEquals( + resultSet.stream().findAny().get(), + new String(JSON_SERIALIZATION_SCHEMA.serialize(TEST_DATASET.getValue().get(1)))); } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-starrocks-e2e/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-starrocks-e2e/pom.xml index 38e4f7eabee..f9b031215b8 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-starrocks-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-starrocks-e2e/pom.xml @@ -26,7 +26,7 @@ SeaTunnel : E2E : Connector V2 : StarRocks - 8.0.27 + 8.0.32 diff --git a/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/container/seatunnel/ConnectorPackageServiceContainer.java b/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/container/seatunnel/ConnectorPackageServiceContainer.java index 4f5ea990295..8e20359e13c 100644 --- a/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/container/seatunnel/ConnectorPackageServiceContainer.java +++ b/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/container/seatunnel/ConnectorPackageServiceContainer.java @@ -84,8 +84,8 @@ public void startUp() throws Exception { server1.withCopyFileToContainer( MountableFile.forHostPath( PROJECT_ROOT_PATH - + "/seatunnel-shade/seatunnel-hadoop3-3.1.4-uber/target/seatunnel-hadoop3-3.1.4-uber.jar"), - Paths.get(SEATUNNEL_HOME, "lib/seatunnel-hadoop3-3.1.4-uber.jar").toString()); + + "/seatunnel-shade/seatunnel-hadoop3-3.3.4-uber/target/seatunnel-hadoop3-3.3.4-uber.jar"), + Paths.get(SEATUNNEL_HOME, "lib/seatunnel-hadoop3-3.3.4-uber.jar").toString()); server2 = new GenericContainer<>(getDockerImage()) @@ -110,8 +110,8 @@ public void startUp() throws Exception { server2.withCopyFileToContainer( MountableFile.forHostPath( PROJECT_ROOT_PATH - + "/seatunnel-shade/seatunnel-hadoop3-3.1.4-uber/target/seatunnel-hadoop3-3.1.4-uber.jar"), - Paths.get(SEATUNNEL_HOME, "lib/seatunnel-hadoop3-3.1.4-uber.jar").toString()); + + "/seatunnel-shade/seatunnel-hadoop3-3.3.4-uber/target/seatunnel-hadoop3-3.3.4-uber.jar"), + Paths.get(SEATUNNEL_HOME, "lib/seatunnel-hadoop3-3.3.4-uber.jar").toString()); server3 = new GenericContainer<>(getDockerImage()) @@ -136,8 +136,8 @@ public void startUp() throws Exception { server3.withCopyFileToContainer( MountableFile.forHostPath( PROJECT_ROOT_PATH - + "/seatunnel-shade/seatunnel-hadoop3-3.1.4-uber/target/seatunnel-hadoop3-3.1.4-uber.jar"), - Paths.get(SEATUNNEL_HOME, "lib/seatunnel-hadoop3-3.1.4-uber.jar").toString()); + + "/seatunnel-shade/seatunnel-hadoop3-3.3.4-uber/target/seatunnel-hadoop3-3.3.4-uber.jar"), + Paths.get(SEATUNNEL_HOME, "lib/seatunnel-hadoop3-3.3.4-uber.jar").toString()); Startables.deepStart(Stream.of(server1)).join(); Startables.deepStart(Stream.of(server2)).join(); diff --git a/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/container/seatunnel/SeaTunnelContainer.java b/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/container/seatunnel/SeaTunnelContainer.java index 2d9e76ea3b5..a255a36f6e2 100644 --- a/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/container/seatunnel/SeaTunnelContainer.java +++ b/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/container/seatunnel/SeaTunnelContainer.java @@ -107,8 +107,8 @@ private GenericContainer createSeaTunnelServer() throws IOException, Interrup server.withCopyFileToContainer( MountableFile.forHostPath( PROJECT_ROOT_PATH - + "/seatunnel-shade/seatunnel-hadoop3-3.1.4-uber/target/seatunnel-hadoop3-3.1.4-uber.jar"), - Paths.get(SEATUNNEL_HOME, "lib/seatunnel-hadoop3-3.1.4-uber.jar").toString()); + + "/seatunnel-shade/seatunnel-hadoop3-3.3.4-uber/target/seatunnel-hadoop3-3.3.4-uber.jar"), + Paths.get(SEATUNNEL_HOME, "lib/seatunnel-hadoop3-3.3.4-uber.jar").toString()); // execute extra commands executeExtraCommands(server); @@ -150,8 +150,8 @@ protected GenericContainer createSeaTunnelContainerWithFakeSourceAndInMemoryS server.withCopyFileToContainer( MountableFile.forHostPath( PROJECT_ROOT_PATH - + "/seatunnel-shade/seatunnel-hadoop3-3.1.4-uber/target/seatunnel-hadoop3-3.1.4-uber.jar"), - Paths.get(SEATUNNEL_HOME, "lib/seatunnel-hadoop3-3.1.4-uber.jar").toString()); + + "/seatunnel-shade/seatunnel-hadoop3-3.3.4-uber/target/seatunnel-hadoop3-3.3.4-uber.jar"), + Paths.get(SEATUNNEL_HOME, "lib/seatunnel-hadoop3-3.3.4-uber.jar").toString()); server.start(); // execute extra commands diff --git a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/pom.xml b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/pom.xml index 973fe6434c7..df971e18822 100644 --- a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/pom.xml +++ b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/pom.xml @@ -66,7 +66,7 @@ org.apache.seatunnel - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber ${project.version} optional test diff --git a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/CheckpointEnableIT.java b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/CheckpointEnableIT.java index 66785d5b649..4723d69eb7e 100644 --- a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/CheckpointEnableIT.java +++ b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/CheckpointEnableIT.java @@ -44,7 +44,9 @@ import static org.awaitility.Awaitility.await; @Slf4j -@DisabledOnJre(value = JRE.JAVA_11, disabledReason = "slf4j jar conflict, we should fix it later") +@DisabledOnJre( + value = {JRE.JAVA_11, JRE.JAVA_17}, + disabledReason = "slf4j jar conflict, we should fix it later") public class CheckpointEnableIT extends TestSuiteBase { @TestTemplate diff --git a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/ClusterSeaTunnelContainer.java b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/ClusterSeaTunnelContainer.java index bf16cab75d2..4c1acbbdc4a 100644 --- a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/ClusterSeaTunnelContainer.java +++ b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/ClusterSeaTunnelContainer.java @@ -64,7 +64,7 @@ public class ClusterSeaTunnelContainer extends SeaTunnelContainer { private static final Path binPath = Paths.get(SEATUNNEL_HOME, "bin", SERVER_SHELL); private static final Path config = Paths.get(SEATUNNEL_HOME, "config"); private static final Path hadoopJar = - Paths.get(SEATUNNEL_HOME, "lib/seatunnel-hadoop3-3.1.4-uber.jar"); + Paths.get(SEATUNNEL_HOME, "lib/seatunnel-hadoop3-3.3.4-uber.jar"); private static final long CUSTOM_JOB_ID = 862969647010611201L; @@ -364,7 +364,7 @@ private GenericContainer createServer(String networkAlias) server.withCopyFileToContainer( MountableFile.forHostPath( PROJECT_ROOT_PATH - + "/seatunnel-shade/seatunnel-hadoop3-3.1.4-uber/target/seatunnel-hadoop3-3.1.4-uber.jar"), + + "/seatunnel-shade/seatunnel-hadoop3-3.3.4-uber/target/seatunnel-hadoop3-3.3.4-uber.jar"), hadoopJar.toString()); server.start(); // execute extra commands diff --git a/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-k8s-e2e/src/test/java/org/apache/seatunnel/engine/e2e/k8s/KubernetesIT.java b/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-k8s-e2e/src/test/java/org/apache/seatunnel/engine/e2e/k8s/KubernetesIT.java index 2de7caeddb5..a4633626c61 100644 --- a/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-k8s-e2e/src/test/java/org/apache/seatunnel/engine/e2e/k8s/KubernetesIT.java +++ b/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-k8s-e2e/src/test/java/org/apache/seatunnel/engine/e2e/k8s/KubernetesIT.java @@ -174,8 +174,8 @@ private void copyFileToCurrentResources(String targetPath) throws IOException { Files.copy( Paths.get( PROJECT_ROOT_PATH - + "/seatunnel-shade/seatunnel-hadoop3-3.1.4-uber/target/seatunnel-hadoop3-3.1.4-uber.jar"), - Paths.get(targetPath + "/jars/seatunnel-hadoop3-3.1.4-uber.jar"), + + "/seatunnel-shade/seatunnel-hadoop3-3.3.4-uber/target/seatunnel-hadoop3-3.3.4-uber.jar"), + Paths.get(targetPath + "/jars/seatunnel-hadoop3-3.3.4-uber.jar"), StandardCopyOption.REPLACE_EXISTING); Files.copy( Paths.get( diff --git a/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-k8s-e2e/src/test/resources/seatunnel_dockerfile b/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-k8s-e2e/src/test/resources/seatunnel_dockerfile index d0a230230f2..844b1be5682 100644 --- a/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-k8s-e2e/src/test/resources/seatunnel_dockerfile +++ b/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-k8s-e2e/src/test/resources/seatunnel_dockerfile @@ -17,7 +17,7 @@ FROM openjdk:8u162-jdk ENV SEATUNNEL_HOME="/opt/seatunnel" -COPY /jars/seatunnel-hadoop3-3.1.4-uber.jar ${SEATUNNEL_HOME}/lib/seatunnel-hadoop3-3.1.4-uber.jar +COPY /jars/seatunnel-hadoop3-3.3.4-uber.jar ${SEATUNNEL_HOME}/lib/seatunnel-hadoop3-3.3.4-uber.jar COPY /jars/seatunnel-transforms-v2.jar ${SEATUNNEL_HOME}/lib/sseatunnel-transforms-v2.jar COPY /jars/seatunnel-starter.jar ${SEATUNNEL_HOME}/starter/seatunnel-starter.jar COPY /bin ${SEATUNNEL_HOME}/bin diff --git a/seatunnel-engine/seatunnel-engine-client/pom.xml b/seatunnel-engine/seatunnel-engine-client/pom.xml index 975c557303c..0e3288bd4ac 100644 --- a/seatunnel-engine/seatunnel-engine-client/pom.xml +++ b/seatunnel-engine/seatunnel-engine-client/pom.xml @@ -88,7 +88,7 @@ org.apache.seatunnel - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber ${project.version} optional test diff --git a/seatunnel-engine/seatunnel-engine-server/pom.xml b/seatunnel-engine/seatunnel-engine-server/pom.xml index abd296cb1e3..f10f012bc74 100644 --- a/seatunnel-engine/seatunnel-engine-server/pom.xml +++ b/seatunnel-engine/seatunnel-engine-server/pom.xml @@ -85,11 +85,16 @@ org.apache.seatunnel - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber ${project.version} optional provided + + com.squareup.okhttp3 + okhttp + ${okhttp.version} + org.apache.seatunnel seatunnel-e2e-common @@ -102,6 +107,12 @@ mockwebserver 2.7.5 test + + + com.squareup.okio + okio + + diff --git a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/event/JobEventHttpReportHandler.java b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/event/JobEventHttpReportHandler.java index 1182c726730..37dca3f5d20 100644 --- a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/event/JobEventHttpReportHandler.java +++ b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/event/JobEventHttpReportHandler.java @@ -28,13 +28,13 @@ import com.hazelcast.ringbuffer.ReadResultSet; import com.hazelcast.ringbuffer.Ringbuffer; import com.hazelcast.ringbuffer.impl.RingbufferProxy; -import com.squareup.okhttp.MediaType; -import com.squareup.okhttp.OkHttpClient; -import com.squareup.okhttp.Request; -import com.squareup.okhttp.RequestBody; -import com.squareup.okhttp.Response; -import com.squareup.okhttp.ResponseBody; import lombok.extern.slf4j.Slf4j; +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; +import okhttp3.ResponseBody; import java.io.IOException; import java.time.Duration; @@ -147,9 +147,9 @@ public void close() { } private OkHttpClient createHttpClient() { - OkHttpClient client = new OkHttpClient(); - client.setConnectTimeout(30, TimeUnit.SECONDS); - client.setWriteTimeout(10, TimeUnit.SECONDS); - return client; + OkHttpClient.Builder builder = new OkHttpClient.Builder(); + builder.connectTimeout(30, TimeUnit.SECONDS); + builder.writeTimeout(10, TimeUnit.SECONDS); + return builder.build(); } } diff --git a/seatunnel-engine/seatunnel-engine-storage/checkpoint-storage-plugins/checkpoint-storage-hdfs/pom.xml b/seatunnel-engine/seatunnel-engine-storage/checkpoint-storage-plugins/checkpoint-storage-hdfs/pom.xml index f7107f9f32b..504f0de1885 100644 --- a/seatunnel-engine/seatunnel-engine-storage/checkpoint-storage-plugins/checkpoint-storage-hdfs/pom.xml +++ b/seatunnel-engine/seatunnel-engine-storage/checkpoint-storage-plugins/checkpoint-storage-hdfs/pom.xml @@ -37,7 +37,7 @@ org.apache.seatunnel - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber ${project.version} optional provided diff --git a/seatunnel-engine/seatunnel-engine-storage/imap-storage-plugins/imap-storage-file/pom.xml b/seatunnel-engine/seatunnel-engine-storage/imap-storage-plugins/imap-storage-file/pom.xml index c7eb61012e6..26d224f3490 100644 --- a/seatunnel-engine/seatunnel-engine-storage/imap-storage-plugins/imap-storage-file/pom.xml +++ b/seatunnel-engine/seatunnel-engine-storage/imap-storage-plugins/imap-storage-file/pom.xml @@ -47,7 +47,7 @@ org.apache.seatunnel - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber ${project.version} optional provided diff --git a/seatunnel-examples/seatunnel-engine-examples/pom.xml b/seatunnel-examples/seatunnel-engine-examples/pom.xml index 5ebe8ab7305..d80f5b6b5ac 100644 --- a/seatunnel-examples/seatunnel-engine-examples/pom.xml +++ b/seatunnel-examples/seatunnel-engine-examples/pom.xml @@ -44,7 +44,7 @@ org.apache.seatunnel - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber ${project.version} diff --git a/seatunnel-shade/pom.xml b/seatunnel-shade/pom.xml index cc761e66409..877ba7827a9 100644 --- a/seatunnel-shade/pom.xml +++ b/seatunnel-shade/pom.xml @@ -27,7 +27,7 @@ SeaTunnel : Shade : - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber seatunnel-jackson seatunnel-guava seatunnel-arrow-5.0 diff --git a/seatunnel-shade/seatunnel-hadoop3-3.1.4-uber/pom.xml b/seatunnel-shade/seatunnel-hadoop3-3.3.4-uber/pom.xml similarity index 86% rename from seatunnel-shade/seatunnel-hadoop3-3.1.4-uber/pom.xml rename to seatunnel-shade/seatunnel-hadoop3-3.3.4-uber/pom.xml index 322c1ed65b5..7973514b963 100644 --- a/seatunnel-shade/seatunnel-hadoop3-3.1.4-uber/pom.xml +++ b/seatunnel-shade/seatunnel-hadoop3-3.3.4-uber/pom.xml @@ -22,11 +22,11 @@ ${revision} - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber SeaTunnel : Shade : Hadoop3 - 3.1.4 + 3.3.4 27.0-jre @@ -46,6 +46,27 @@ org.apache.hadoop hadoop-client ${hadoop3.version} + + + ch.qos.reload4j + reload4j + + + + + org.apache.hadoop + hadoop-client-runtime + ${hadoop3.version} + + + org.apache.hadoop + hadoop-client-api + ${hadoop3.version} + + + org.apache.hadoop + hadoop-aws + ${hadoop3.version} org.xerial.snappy @@ -66,7 +87,7 @@ package - seatunnel-hadoop3-3.1.4-uber + seatunnel-hadoop3-3.3.4-uber true true false @@ -122,7 +143,7 @@ - ${basedir}/target/seatunnel-hadoop3-3.1.4-uber.jar + ${basedir}/target/seatunnel-hadoop3-3.3.4-uber.jar jar optional diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/test/java/org/apache/seatunnel/translation/spark/sink/SparkSinkTest.java b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/test/java/org/apache/seatunnel/translation/spark/sink/SparkSinkTest.java index b426f175480..cf8e6aa6ac0 100644 --- a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/test/java/org/apache/seatunnel/translation/spark/sink/SparkSinkTest.java +++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-3.3/src/test/java/org/apache/seatunnel/translation/spark/sink/SparkSinkTest.java @@ -59,7 +59,7 @@ public class SparkSinkTest { @Test @DisabledOnJre( - value = JRE.JAVA_11, + value = {JRE.JAVA_11, JRE.JAVA_17}, disabledReason = "We should update apache common lang3 version to 3.8 to avoid NPE, " + "see https://github.com/apache/commons-lang/commit/50ce8c44e1601acffa39f5568f0fc140aade0564") diff --git a/tools/dependencies/checkLicense.sh b/tools/dependencies/checkLicense.sh index 836ca91d354..99f5baa842f 100755 --- a/tools/dependencies/checkLicense.sh +++ b/tools/dependencies/checkLicense.sh @@ -23,7 +23,7 @@ if [ -d "/tmp/seatunnel-dependencies" ]; then rm -rf /tmp/seatunnel-dependencies/* fi -./mvnw clean -pl '!seatunnel-dist' --batch-mode --no-snapshot-updates dependency:copy-dependencies -DincludeScope=runtime -DoutputDirectory=/tmp/seatunnel-dependencies +./mvnw clean -pl '!seatunnel-dist' --batch-mode --no-snapshot-updates dependency:copy-dependencies -Prelease -DincludeScope=runtime -DoutputDirectory=/tmp/seatunnel-dependencies # List all modules(jars) that belong to the SeaTunnel itself, these will be ignored when checking the dependency ls /tmp/seatunnel-dependencies | sort > all-dependencies.txt diff --git a/tools/dependencies/known-dependencies.txt b/tools/dependencies/known-dependencies.txt index 8532f7cba43..c2d623fa420 100755 --- a/tools/dependencies/known-dependencies.txt +++ b/tools/dependencies/known-dependencies.txt @@ -13,7 +13,7 @@ jackson-core-2.13.3.jar jackson-databind-2.13.3.jar jackson-dataformat-properties-2.13.3.jar jackson-datatype-jsr310-2.13.3.jar -jcl-over-slf4j-1.7.25.jar +jcl-over-slf4j-1.7.36.jar jcommander-1.81.jar log4j-api-2.17.1.jar log4j-core-2.17.1.jar @@ -27,7 +27,7 @@ scala-library-2.12.15.jar seatunnel-jackson-2.3.6-SNAPSHOT-optional.jar seatunnel-guava-2.3.6-SNAPSHOT-optional.jar seatunnel-hazelcast-shade-2.3.6-SNAPSHOT-optional.jar -slf4j-api-1.7.25.jar +slf4j-api-1.7.36.jar jsqlparser-4.5.jar animal-sniffer-annotations-1.17.jar checker-qual-3.10.0.jar @@ -43,5 +43,13 @@ json-smart-2.4.7.jar accessors-smart-2.4.7.jar asm-9.1.jar avro-1.11.1.jar +annotations-13.0.jar +kotlin-stdlib-1.8.21.jar +kotlin-stdlib-common-1.9.10.jar +kotlin-stdlib-jdk7-1.8.21.jar +kotlin-stdlib-jdk8-1.8.21.jar +okhttp-4.12.0.jar +okio-3.6.0.jar +okio-jvm-3.6.0.jar groovy-4.0.16.jar -seatunnel-janino-2.3.6-SNAPSHOT-optional.jar \ No newline at end of file +seatunnel-janino-2.3.6-SNAPSHOT-optional.jar