diff --git a/.github/workflows/extended-ci-longevity-large-partition-asymmetric-cluster-3h-test.yml b/.github/workflows/extended-ci-longevity-large-partition-asymmetric-cluster-3h-test.yml new file mode 100644 index 00000000..0a44c74a --- /dev/null +++ b/.github/workflows/extended-ci-longevity-large-partition-asymmetric-cluster-3h-test.yml @@ -0,0 +1,65 @@ +name: Build scylla-bench docker image with gocql PR + +on: + pull_request_target: + types: [opened, synchronize, reopened, labeled] + +jobs: + trigger-longevity-large-partition-asymmetric-cluster-3h-test: + if: contains(github.event.pull_request.labels.*.name, 'extended-ci') + runs-on: ubuntu-latest + strategy: + matrix: + scylla-version: [ENTERPRISE-RELEASE, OSS-RELEASE] + steps: + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Check out the scylla-bench repository + uses: actions/checkout@v2 + with: + repository: scylladb/scylla-bench + path: scylla-bench + + - name: Checkout GoCQL PR Repository + uses: actions/checkout@v2 + with: + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.ref }} + path: gocql + + - name: Build and push Scylla-bench Docker Image with gocql from PR + run: | + cd scylla-bench + GOCQL_REPO="github.com/${{ github.event.pull_request.head.repo.full_name }}" GOCQL_VERSION="${{ github.event.pull_request.head.ref }}" make build-with-custom-gocql-version + DOCKER_IMAGE_TAG="scylladb/gocql-extended-ci:scylla-bench-${{ github.event.pull_request.head.sha }}" DOCKER_IMAGE_LABELS="com.scylladb.gocql-version=${{ github.event.pull_request.head.sha }}" make build-sct-docker-image + docker push "scylladb/gocql-extended-ci:scylla-bench-${{ github.event.pull_request.head.sha }}" + + - name: Get scylla version + id: scylla-version + run: | + if [[ "${{ matrix.scylla-version }}" == "ENTERPRISE-RELEASE" ]]; then + echo "value=$(python3 ci/version_fetch.py --version-index 1 scylla-enterprise-stable:1 | tr -d '\"')" | tee -a $GITHUB_OUTPUT + elif [[ "${{ matrix.scylla-version }}" == "OSS-RELEASE" ]]; then + echo "value=$(python3 ci/version_fetch.py --version-index 1 scylla-oss-stable:1 | tr -d '\"')" | tee -a $GITHUB_OUTPUT + elif echo "${{ matrix.scylla-version }}" | grep -P '^[0-9\.]+'; then # If you want to run specific version do just that + echo "value=${{ matrix.scylla-version }}" | tee -a $GITHUB_OUTPUT + else + echo "Unknown scylla version name `${{ matrix.scylla-version }}`" + exit 1 + fi + + - name: Start Jenkins job + uses: sylwiaszunejko/jenkins_client@main + with: + jenkins_job_name: 'scylla-drivers/gocql/extended-ci/longevity-large-partition-asymmetric-cluster-3h-test' + jenkins_job_parameters: '{"email_recipients": "scylla-drivers@scylladb.com", "scylla_version": "release:${{ steps.scylla-version.outputs.value }}", "extra_environment_variables": "SCT_STRESS_IMAGE.scylla-bench=scylladb/gocql-extended-ci:scylla-bench-${{ github.event.pull_request.head.sha }}"}' + jenkins_base_url: 'https://jenkins.scylladb.com/' + jenkins_user: ${{ secrets.JENKINS_USERNAME }} + jenkins_password: ${{ secrets.JENKINS_TOKEN }} + wait_for_result: 'True' + polling_interval: '120' + polling_timeout: '12600' diff --git a/ci/version_fetch.py b/ci/version_fetch.py new file mode 100644 index 00000000..8e2ec61d --- /dev/null +++ b/ci/version_fetch.py @@ -0,0 +1,212 @@ +#!/usr/bin/python3 + +""" +This Python script allows you to list the +latest version numbers of Scylla and Cassandra. +You can specify whether you want the +versions of Scylla OSS or Scylla Enterprise, +either N latest stable X.Y.latest or +all non-obsolete RCs. You can also fetch +the latest version of Cassandra 3. +How are those versions fetched? We use Docker Hub +tags API. +""" + +import requests +import argparse +import re +import json +import sys + +DOCKER_HUB_TAGS_ENDPOINT = 'https://hub.docker.com/v2/namespaces/%s/repositories/%s/tags?page_size=1000' +DOCKER_HUB_SCYLLA_NAMESPACE = 'scylladb' + +SCYLLA_OSS = (DOCKER_HUB_SCYLLA_NAMESPACE, 'scylla') +SCYLLA_ENTERPRISE = (DOCKER_HUB_SCYLLA_NAMESPACE, 'scylla-enterprise') + +CASSANDRA_ENDPOINT = 'https://dlcdn.apache.org/cassandra/' +CASSANDRA_REGEX = re.compile(r'a href="([0-9])\.(\d+)\.(\d+)/"') + +VERSION_DEFINITION_RE = re.compile( + r'((?:(scylla-oss-stable):(\d+))|(?:(scylla-enterprise-stable)(:\d+)?)|(?:(cassandra3-stable)(:\d+)?)|(?:(cassandra4-stable)(:\d+)?)|(?:(scylla-oss-rc)(:\d+)?)|(?:(scylla-enterprise-rc)(:\d+)?))') +ONLY_DIGITS_RE = re.compile("^[0-9]+") + +class Version: + raw = "" + invalid = False + major = 0 + minor = 0 + minor_raw = "" + minor_rest = "" + patch = 0 + patch_raw = "" + patch_rest = "" + is_dev = False + + def __init__(self, ver): + self.raw = ver + chunks = ver.split(".", maxsplit=3) + if len(chunks) < 3: + self.invalid = True + return + + self.major, self.minor_raw, self.patch_raw = chunks + + try: + self.major = int(self.major) + except Exception: + self.invalid = True + return + + digits = ONLY_DIGITS_RE.match(self.minor_raw) + if digits: + try: + self.minor = int(self.minor_raw[digits.regs[0][0]:digits.regs[0][1]]) + self.minor_rest = self.minor_raw[digits.regs[0][1]:] + if self.minor_rest: + self.is_dev = True + except Exception: + self.invalid = True + else: + self.minor = 0 + + digits = ONLY_DIGITS_RE.match(self.patch_raw) + if digits: + try: + self.patch = int(self.patch_raw[digits.regs[0][0]:digits.regs[0][1]]) + self.patch_rest = self.patch_raw[digits.regs[0][1]:] + if self.patch_rest: + self.is_dev = True + except Exception: + self.invalid = True + else: + self.patch = 0 + + def __str__(self): + return self.raw + + def __repr__(self): + return self.raw + + def __lt__(self, other): + if self.major != other.major: + return self.major < other.major + if self.minor != other.minor: + return self.minor < other.minor + if self.patch != other.patch: + return self.patch < other.patch + if self.is_dev == other.is_dev: + return self.patch_rest < other.patch_rest + return self.is_dev + + +def fetch_docker_hub_tags(namespace, repository): + tags = [] + + # Fetch all pages of tags for a given repository + current_page_endpoint = DOCKER_HUB_TAGS_ENDPOINT % (namespace, repository) + while True: + # Fetch a page + tags_data = requests.get(current_page_endpoint).json() + + # Extract all tags from the response + tags.extend(map(lambda e: e['name'], tags_data['results'])) + + # Move to the next page if it's needed + if tags_data['next'] is not None: + current_page_endpoint = tags_data['next'] + else: + break + + result = [] + for tag in tags: + try: + ver = Version(tag) + if not ver.patch_raw or ver.invalid: + continue + result.append(ver) + except Exception: + continue + + return reversed(sorted(result)) + +rc_only = lambda x: x.is_dev +release_only = lambda x: not x.is_dev +scylla_oss_only = lambda x: x.major < 2000 +scylla_enterprise_only = lambda x: x.major > 2000 +cassandra_3_only = lambda x: x.major == 3 +cassandra_4_only = lambda x: x.major == 4 + +def filter_versions(all_versions, count, *filters): + result = [] + for ver in filter(lambda ver: all(map(lambda ft: ft(ver), filters)), all_versions): + for existing_ver in result: + if ver.major == existing_ver.major and ver.minor == existing_ver.minor: + break + else: + if len(result) >= count: + return result + result.append(ver) + return result + +def fetch_all_cassandra_versions(): + # Download folder listing for Cassandra download site + data = requests.get(CASSANDRA_ENDPOINT).text + + # Parse only those version numbers which match '3.NUM.NUM' + # into tuple (3, NUM, NUM) + data = CASSANDRA_REGEX.finditer(data) + data = map(lambda e: e.groups(), data) + data = map(lambda e: Version(f"{e[0]}.{e[1]}.{e[2]}"), data) + return reversed(sorted(data)) + + +if __name__ == '__main__': + total_result = [] + + def version_definition_type(arg): + if not VERSION_DEFINITION_RE.match(arg): + raise argparse.ArgumentTypeError(f"invalid version definition `{arg}`") + return arg + + parser = argparse.ArgumentParser(description='Get scylla and cassandra versions.') + parser.add_argument('definitions', metavar='VERSION-DEFINITION', type=version_definition_type, nargs='+', + help='A list of scylla and cassandra version definitions: scylla-oss-stable[:COUNT] | scylla-enterprise-stable[:COUNT] | cassandra3-stable[:COUNT] | cassandra4-stable[:COUNT] | scylla-oss-rc[:COUNT] | scylla-enterprise-rc[:COUNT]') + + parser.add_argument('--version-index', dest='version_index', type=int, default=0, help='print single version only') + + args = parser.parse_args() + + for version_def in args.definitions: + groups = VERSION_DEFINITION_RE.match(version_def).groups() + groups = [g for g in groups if g][1:] + + mode_name = groups[0] + versions_count = int(groups[1].replace(":","")) if len(groups) > 1 else 1 + + result = [] + if mode_name == 'scylla-oss-stable': + result = filter_versions(fetch_docker_hub_tags(*SCYLLA_OSS), versions_count, scylla_oss_only, release_only) + elif mode_name == 'scylla-enterprise-stable': + result = filter_versions(fetch_docker_hub_tags(*SCYLLA_ENTERPRISE), versions_count, scylla_enterprise_only, release_only) + elif mode_name == 'cassandra3-stable': + result = filter_versions(fetch_all_cassandra_versions(), versions_count, cassandra_3_only) + elif mode_name == 'cassandra4-stable': + result = filter_versions(fetch_all_cassandra_versions(), versions_count, cassandra_4_only) + elif mode_name == 'scylla-oss-rc': + result = filter_versions(fetch_docker_hub_tags(*SCYLLA_OSS), versions_count, scylla_oss_only, rc_only) + elif mode_name == 'scylla-enterprise-rc': + result = filter_versions(fetch_docker_hub_tags(*SCYLLA_ENTERPRISE), versions_count, scylla_enterprise_only, rc_only) + + total_result += result + + total_result = [str(k) for k in reversed(sorted(total_result))] + + if args.version_index == 0: + print(json.dumps(list(total_result))) + sys.exit(0) + if len(total_result) < args.version_index: + print("No versions found", file=sys.stderr) + sys.exit(1) + print(json.dumps(total_result[args.version_index-1])) + sys.exit(0)