diff --git a/build.sh b/build.sh index 3d503c58..21bcf4dc 100755 --- a/build.sh +++ b/build.sh @@ -18,40 +18,25 @@ # specific language governing permissions and limitations # under the License. -set -eo pipefail +source "$(dirname "$0")"/common.sh -# check maven -MVN_CMD=mvn -if [[ ! -z ${CUSTOM_MVN} ]]; then - MVN_CMD=${CUSTOM_MVN} -fi -if ! ${MVN_CMD} --version; then - echo "Error: mvn is not found" - exit 1 -fi -export MVN_CMD - -supported_minor_version=("1.15" "1.16" "1.17") -version_msg=$(IFS=, ; echo "${supported_minor_version[*]}") if [ ! $1 ] then echo "Usage:" echo " sh build.sh " - echo " supported flink version: ${version_msg}" + echo " supported flink version: ${VERSION_MESSAGE}" exit 1 fi flink_minor_version=$1 -if [[ " ${supported_minor_version[*]} " == *" $flink_minor_version "* ]]; -then - echo "Compiling connector for flink version $flink_minor_version" -else - echo "Error: only support flink version: ${version_msg}" - exit 1 -fi +check_flink_version_supported $flink_minor_version +flink_version="$(get_flink_version $flink_minor_version)" +kafka_connector_version="$(get_kafka_connector_version $flink_minor_version)" -flink_version=${flink_minor_version}.0 -${MVN_CMD} clean package -DskipTests -Dflink.minor.version=${flink_minor_version} -Dflink.version=${flink_version} +${MVN_CMD} clean package -DskipTests \ + -Dflink.minor.version=${flink_minor_version} \ + -Dflink.version=${flink_version} \ + -Dkafka.connector.version=${kafka_connector_version} echo "*********************************************************************" echo "Successfully build Flink StarRocks Connector for Flink $flink_minor_version" diff --git a/common.sh b/common.sh new file mode 100644 index 00000000..89e5584d --- /dev/null +++ b/common.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# +# Copyright 2021-present StarRocks, Inc. All rights reserved. +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -eo pipefail + +# check maven +MVN_CMD=mvn +if [[ ! -z ${CUSTOM_MVN} ]]; then + MVN_CMD=${CUSTOM_MVN} +fi +if ! ${MVN_CMD} --version; then + echo "Error: mvn is not found" + exit 1 +fi +export MVN_CMD + +SUPPORTED_MINOR_VERSION=("1.15" "1.16" "1.17" "1.18") +# version formats are different among flink versions +SUPPORTED_KAFKA_CONNECTOR_VERSION=("1.15.0" "1.16.0" "1.17.0" "3.0.1-1.18") +VERSION_MESSAGE=$(IFS=, ; echo "${SUPPORTED_MINOR_VERSION[*]}") + +function check_flink_version_supported() { + local FLINK_MINOR_VERSION=$1 + if [[ " ${SUPPORTED_MINOR_VERSION[*]} " != *" $FLINK_MINOR_VERSION "* ]]; + then + echo "Error: only support flink version: ${VERSION_MESSAGE}" + exit 1 + fi +} + +function get_flink_version() { + local FLINK_MINOR_VERSION=$1 + echo "${FLINK_MINOR_VERSION}.0" +} + +function get_kafka_connector_version() { + local FLINK_MINOR_VERSION=$1 + local index=-1 + for ((i=0; i<${#SUPPORTED_MINOR_VERSION[@]}; i++)); do + if [ "${SUPPORTED_MINOR_VERSION[i]}" = "$FLINK_MINOR_VERSION" ]; then + index=$i + break + fi + done + + if [ "$index" != -1 ]; + then + local KAFKA_CONNECTOR_VERSION="${SUPPORTED_KAFKA_CONNECTOR_VERSION[index]}" + echo $KAFKA_CONNECTOR_VERSION + else + echo "Can't find kafka connector version for flink-${FLINK_MINOR_VERSION}" + exit 1 + fi +} \ No newline at end of file diff --git a/deploy.sh b/deploy.sh index 678316de..e44b110f 100644 --- a/deploy.sh +++ b/deploy.sh @@ -18,40 +18,25 @@ # specific language governing permissions and limitations # under the License. -set -eo pipefail +source "$(dirname "$0")"/common.sh -# check maven -MVN_CMD=mvn -if [[ ! -z ${CUSTOM_MVN} ]]; then - MVN_CMD=${CUSTOM_MVN} -fi -if ! ${MVN_CMD} --version; then - echo "Error: mvn is not found" - exit 1 -fi -export MVN_CMD - -supported_minor_version=("1.15" "1.16" "1.17") -version_msg=$(IFS=, ; echo "${supported_minor_version[*]}") if [ ! $1 ] then echo "Usage:" - echo " sh build.sh " - echo " supported flink version: ${version_msg}" + echo " sh deploy.sh " + echo " supported flink version: ${VERSION_MESSAGE}" exit 1 fi flink_minor_version=$1 -if [[ " ${supported_minor_version[*]} " == *" $flink_minor_version "* ]]; -then - echo "Compiling connector for flink version $flink_minor_version" -else - echo "Error: only support flink version: ${version_msg}" - exit 1 -fi +check_flink_version_supported $flink_minor_version +flink_version="$(get_flink_version $flink_minor_version)" +kafka_connector_version="$(get_kafka_connector_version $flink_minor_version)" -flink_version=${flink_minor_version}.0 -${MVN_CMD} clean deploy -Prelease -DskipTests -Dflink.minor.version=${flink_minor_version} -Dflink.version=${flink_version} +${MVN_CMD} clean deploy -Prelease -DskipTests \ + -Dflink.minor.version=${flink_minor_version} \ + -Dflink.version=${flink_version} \ + -Dkafka.connector.version=${kafka_connector_version} echo "*********************************************************************" echo "Successfully deploy Flink StarRocks Connector for Flink $flink_minor_version" diff --git a/pom.xml b/pom.xml index 61a45576..6c0120df 100644 --- a/pom.xml +++ b/pom.xml @@ -53,8 +53,9 @@ limitations under the License. UTF-8 3.0.0-M3 3.0.0-M4 - 1.17 - 1.17.0 + 1.18 + 1.18.0 + 3.0.1-1.18 5.0.0 2.8.1 2.12 @@ -128,19 +129,24 @@ limitations under the License. arrow-vector ${arrow.version} + + org.apache.arrow + arrow-memory-netty + ${arrow.version} + org.apache.flink flink-connector-kafka - ${flink.version} + ${kafka.connector.version} test org.apache.flink flink-connector-kafka - ${flink.version} + ${kafka.connector.version} test-jar test @@ -438,6 +444,14 @@ limitations under the License. com.google ${shading.prefix}.com.google + + + com.fasterxml.jackson + com.starrocks.streamload.shade.com.fasterxml.jackson + @@ -446,6 +460,8 @@ limitations under the License. com.starrocks:starrocks-stream-load-sdk com.starrocks:starrocks-thrift-sdk org.apache.arrow:* + io.netty:* + com.google.flatbuffers:flatbuffers-java com.google.guava:*