Support flink 1.18

Signed-off-by: PengFei Li <[email protected]>
banmoy · Nov 2, 2023 · ee3ba41 · ee3ba41
1 parent 5da6be2
commit ee3ba41
Show file tree

Hide file tree

Showing 4 changed files with 111 additions and 53 deletions.
diff --git a/build.sh b/build.sh
@@ -18,40 +18,25 @@
 # specific language governing permissions and limitations
 # under the License.
 
-set -eo pipefail
+source "$(dirname "$0")"/common.sh
 
-# check maven
-MVN_CMD=mvn
-if [[ ! -z ${CUSTOM_MVN} ]]; then
-    MVN_CMD=${CUSTOM_MVN}
-fi
-if ! ${MVN_CMD} --version; then
-    echo "Error: mvn is not found"
-    exit 1
-fi
-export MVN_CMD
-
-supported_minor_version=("1.15" "1.16" "1.17")
-version_msg=$(IFS=, ; echo "${supported_minor_version[*]}")
 if [ ! $1 ]
 then
     echo "Usage:"
     echo "   sh build.sh <flink_version>"
-    echo "   supported flink version: ${version_msg}"
+    echo "   supported flink version: ${VERSION_MESSAGE}"
     exit 1
 fi
 
 flink_minor_version=$1
-if [[ " ${supported_minor_version[*]} " == *" $flink_minor_version "* ]];
-then
-    echo "Compiling connector for flink version $flink_minor_version"
-else
-    echo "Error: only support flink version: ${version_msg}"
-    exit 1
-fi
+check_flink_version_supported $flink_minor_version
+flink_version="$(get_flink_version $flink_minor_version)"
+kafka_connector_version="$(get_kafka_connector_version $flink_minor_version)"
 
-flink_version=${flink_minor_version}.0
-${MVN_CMD} clean package -DskipTests -Dflink.minor.version=${flink_minor_version} -Dflink.version=${flink_version}
+${MVN_CMD} clean package -DskipTests \
+  -Dflink.minor.version=${flink_minor_version} \
+  -Dflink.version=${flink_version} \
+  -Dkafka.connector.version=${kafka_connector_version}
 
 echo "*********************************************************************"
 echo "Successfully build Flink StarRocks Connector for Flink $flink_minor_version"

diff --git a/common.sh b/common.sh
@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+#
+# Copyright 2021-present StarRocks, Inc. All rights reserved.
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -eo pipefail
+
+# check maven
+MVN_CMD=mvn
+if [[ ! -z ${CUSTOM_MVN} ]]; then
+    MVN_CMD=${CUSTOM_MVN}
+fi
+if ! ${MVN_CMD} --version; then
+    echo "Error: mvn is not found"
+    exit 1
+fi
+export MVN_CMD
+
+SUPPORTED_MINOR_VERSION=("1.15" "1.16" "1.17" "1.18")
+# version formats are different among flink versions
+SUPPORTED_KAFKA_CONNECTOR_VERSION=("1.15.0" "1.16.0" "1.17.0" "3.0.1-1.18")
+VERSION_MESSAGE=$(IFS=, ; echo "${SUPPORTED_MINOR_VERSION[*]}")
+
+function check_flink_version_supported() {
+  local FLINK_MINOR_VERSION=$1
+  if [[ " ${SUPPORTED_MINOR_VERSION[*]} " != *" $FLINK_MINOR_VERSION "* ]];
+  then
+      echo "Error: only support flink version: ${VERSION_MESSAGE}"
+      exit 1
+  fi
+}
+
+function get_flink_version() {
+  local FLINK_MINOR_VERSION=$1
+  echo "${FLINK_MINOR_VERSION}.0"
+}
+
+function get_kafka_connector_version() {
+  local FLINK_MINOR_VERSION=$1
+  local index=-1
+  for ((i=0; i<${#SUPPORTED_MINOR_VERSION[@]}; i++)); do
+      if [ "${SUPPORTED_MINOR_VERSION[i]}" = "$FLINK_MINOR_VERSION" ]; then
+          index=$i
+          break
+      fi
+  done
+
+  if [ "$index" != -1 ];
+  then
+    local KAFKA_CONNECTOR_VERSION="${SUPPORTED_KAFKA_CONNECTOR_VERSION[index]}"
+    echo $KAFKA_CONNECTOR_VERSION
+  else
+    echo "Can't find kafka connector version for flink-${FLINK_MINOR_VERSION}"
+    exit 1
+  fi
+}
diff --git a/deploy.sh b/deploy.sh
@@ -18,40 +18,25 @@
 # specific language governing permissions and limitations
 # under the License.
 
-set -eo pipefail
+source "$(dirname "$0")"/common.sh
 
-# check maven
-MVN_CMD=mvn
-if [[ ! -z ${CUSTOM_MVN} ]]; then
-    MVN_CMD=${CUSTOM_MVN}
-fi
-if ! ${MVN_CMD} --version; then
-    echo "Error: mvn is not found"
-    exit 1
-fi
-export MVN_CMD
-
-supported_minor_version=("1.15" "1.16" "1.17")
-version_msg=$(IFS=, ; echo "${supported_minor_version[*]}")
 if [ ! $1 ]
 then
     echo "Usage:"
-    echo "   sh build.sh <flink_version>"
-    echo "   supported flink version: ${version_msg}"
+    echo "   sh deploy.sh <flink_version>"
+    echo "   supported flink version: ${VERSION_MESSAGE}"
     exit 1
 fi
 
 flink_minor_version=$1
-if [[ " ${supported_minor_version[*]} " == *" $flink_minor_version "* ]];
-then
-    echo "Compiling connector for flink version $flink_minor_version"
-else
-    echo "Error: only support flink version: ${version_msg}"
-    exit 1
-fi
+check_flink_version_supported $flink_minor_version
+flink_version="$(get_flink_version $flink_minor_version)"
+kafka_connector_version="$(get_kafka_connector_version $flink_minor_version)"
 
-flink_version=${flink_minor_version}.0
-${MVN_CMD} clean deploy -Prelease -DskipTests -Dflink.minor.version=${flink_minor_version} -Dflink.version=${flink_version}
+${MVN_CMD} clean deploy -Prelease -DskipTests \
+  -Dflink.minor.version=${flink_minor_version} \
+  -Dflink.version=${flink_version} \
+  -Dkafka.connector.version=${kafka_connector_version}
 
 echo "*********************************************************************"
 echo "Successfully deploy Flink StarRocks Connector for Flink $flink_minor_version"

diff --git a/pom.xml b/pom.xml
@@ -53,8 +53,9 @@ limitations under the License.
         <file_encoding>UTF-8</file_encoding>
         <maven-enforcer-plugin.version>3.0.0-M3</maven-enforcer-plugin.version>
         <maven-surefire-plugin.version>3.0.0-M4</maven-surefire-plugin.version>
-        <flink.minor.version>1.17</flink.minor.version>
-        <flink.version>1.17.0</flink.version>
+        <flink.minor.version>1.18</flink.minor.version>
+        <flink.version>1.18.0</flink.version>
+        <kafka.connector.version>3.0.1-1.18</kafka.connector.version>
         <arrow.version>5.0.0</arrow.version>
         <kafka.version>2.8.1</kafka.version>
         <scala.binary.version>2.12</scala.binary.version>
@@ -128,19 +129,24 @@ limitations under the License.
             <artifactId>arrow-vector</artifactId>
             <version>${arrow.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.apache.arrow</groupId>
+            <artifactId>arrow-memory-netty</artifactId>
+            <version>${arrow.version}</version>
+        </dependency>
 
         <!--Following dependencies are mainly used for tests of kafka -> starrocks pipelines, and copied from flink-->
         <dependency>
             <groupId>org.apache.flink</groupId>
             <artifactId>flink-connector-kafka</artifactId>
-            <version>${flink.version}</version>
+            <version>${kafka.connector.version}</version>
             <scope>test</scope>
         </dependency>
 
         <dependency>
             <groupId>org.apache.flink</groupId>
             <artifactId>flink-connector-kafka</artifactId>
-            <version>${flink.version}</version>
+            <version>${kafka.connector.version}</version>
             <type>test-jar</type>
             <scope>test</scope>
         </dependency>
@@ -438,6 +444,14 @@ limitations under the License.
                                     <pattern>com.google</pattern>
                                     <shadedPattern>${shading.prefix}.com.google</shadedPattern>
                                 </relocation>
+                                <relocation>
+                                    <!--
+                                    arrow depends on fastxml, and relocate to com.starrocks.streamload.shade so that
+                                    we can reuse the fastxml from streamload sdk
+                                    -->
+                                    <pattern>com.fasterxml.jackson</pattern>
+                                    <shadedPattern>com.starrocks.streamload.shade.com.fasterxml.jackson</shadedPattern>
+                                </relocation>
                             </relocations>
                             <artifactSet>
                                 <includes>
@@ -446,6 +460,8 @@ limitations under the License.
                                     <include>com.starrocks:starrocks-stream-load-sdk</include>
                                     <include>com.starrocks:starrocks-thrift-sdk</include>
                                     <include>org.apache.arrow:*</include>
+                                    <include>io.netty:*</include>
+                                    <include>com.google.flatbuffers:flatbuffers-java</include>
                                     <include>com.google.guava:*</include>
                                 </includes>
                             </artifactSet>