ccmlib: Remove usages of distutils

.github/workflows/integration-tests.yml

@@ -16,7 +16,7 @@ jobs:
 
     strategy:
       matrix:
-        python-version: ["3.8", "3.11"]
+        python-version: ["3.8", "3.11", "3.12"]
 
     steps:
     - uses: actions/checkout@v3

README.md

@@ -90,7 +90,7 @@ Windows only:
 Installation
 ------------
 
-ccm uses python distutils so from the source directory run:
+ccm uses python setuptools (with distutils fallback) so from the source directory run:
 
     sudo ./setup.py install
 

ccmlib/repository.py

@@ -11,10 +11,11 @@
 import tarfile
 import tempfile
 import time
+from typing import List
 import urllib
-from distutils.version import LooseVersion
 
 
+from ccmlib.utils.version import ComparableCassandraVersion
 from ccmlib.common import (ArgumentError, CCMError, get_default_path,
                            platform_binary, rmdirs, validate_install_dir,
                            assert_jdk_valid_for_cassandra_version, get_version_from_build)
@@ -373,7 +374,7 @@ def get_tagged_version_numbers(series='stable'):
     """Retrieve git tags and find version numbers for a release series
 
     series - 'stable', 'oldstable', or 'testing'"""
-    releases = []
+    releases: List[ComparableCassandraVersion] = []
     if series == 'testing':
         # Testing releases always have a hyphen after the version number:
         tag_regex = re.compile(r'^refs/tags/cassandra-([0-9]+\.[0-9]+\.[0-9]+-.*$)')
@@ -385,15 +386,15 @@ def get_tagged_version_numbers(series='stable'):
     for ref in (i.get('ref', '') for i in json.loads(tag_url.read())):
         m = tag_regex.match(ref)
         if m:
-            releases.append(LooseVersion(m.groups()[0]))
+            releases.append(ComparableCassandraVersion(m.groups()[0]))
 
     # Sort by semver:
     releases.sort(reverse=True)
 
-    stable_major_version = LooseVersion(str(releases[0].version[0]) + "." + str(releases[0].version[1]))
+    stable_major_version = ComparableCassandraVersion(str(releases[0].version[0]) + "." + str(releases[0].version[1]))
     stable_releases = [r for r in releases if r >= stable_major_version]
     oldstable_releases = [r for r in releases if r not in stable_releases]
-    oldstable_major_version = LooseVersion(str(oldstable_releases[0].version[0]) + "." + str(oldstable_releases[0].version[1]))
+    oldstable_major_version = ComparableCassandraVersion(str(oldstable_releases[0].version[0]) + "." + str(oldstable_releases[0].version[1]))
     oldstable_releases = [r for r in oldstable_releases if r >= oldstable_major_version]
 
     if series == 'testing':

ccmlib/scylla_cluster.py

@@ -8,14 +8,14 @@
 import uuid
 import datetime
 
-from distutils.version import LooseVersion
 
 from ccmlib import common
 from ccmlib.cluster import Cluster
 from ccmlib.scylla_node import ScyllaNode
 from ccmlib.node import NodeError
 from ccmlib import scylla_repository
 from ccmlib.utils.sni_proxy import stop_sni_proxy
+from ccmlib.utils.version import ComparableScyllaVersion
 
 SNITCH = 'org.apache.cassandra.locator.GossipingPropertyFileSnitch'
 
@@ -298,7 +298,7 @@ def __init__(self, scylla_cluster, install_dir=None):
     def version(self):
         stdout, _ = self.sctool(["version"], ignore_exit_status=True)
         version_string = stdout[stdout.find(": ") + 2:].strip()  # Removing unnecessary information
-        version_code = LooseVersion(version_string)
+        version_code = ComparableScyllaVersion(version_string)
         return version_code
 
     def _install(self, install_dir):
@@ -318,8 +318,7 @@ def _update_config(self, install_dir=None):
             data['database'] = {}
         data['database']['hosts'] = [self.scylla_cluster.get_node_ip(1)]
         data['database']['replication_factor'] = 3
-        if install_dir and (self.version < LooseVersion("2.5") or
-                            LooseVersion('666') < self.version < LooseVersion('666.dev-0.20210430.2217cc84')):
+        if install_dir and self.version < ComparableScyllaVersion("2.5"):
             data['database']['migrate_dir'] = os.path.join(install_dir, 'schema', 'cql')
         if 'https' in data:
             del data['https']
@@ -332,7 +331,7 @@ def _update_config(self, install_dir=None):
         data['logger']['mode'] = 'stderr'
         if not 'repair' in data:
             data['repair'] = {}
-        if self.version < LooseVersion("2.2"):
+        if self.version < ComparableScyllaVersion("2.2"):
             data['repair']['segments_per_repair'] = 16
         data['prometheus'] = f"{self.scylla_cluster.get_node_ip(1)}:56091"
         # Changing port to 56091 since the manager and the first node share the same ip and 56090 is already in use

ccmlib/utils/sni_proxy.py

@@ -8,7 +8,7 @@
 from contextlib import contextmanager
 import tempfile
 from textwrap import dedent
-import distutils.dir_util
+from shutil import copytree
 from dataclasses import dataclass
 
 import yaml
@@ -198,7 +198,7 @@ def refresh_certs(cluster, nodes_info):
         dns_names = ['cql.cluster-id.scylla.com'] + \
                     [f'{node.host_id}.cql.cluster-id.scylla.com' for node in nodes_info]
         generate_ssl_stores(tmp_dir, dns_names=dns_names)
-        distutils.dir_util.copy_tree(tmp_dir, cluster.get_path())
+        copytree(tmp_dir, cluster.get_path(), dirs_exist_ok=True)
 
 
 if __name__ == "__main__":

ccmlib/utils/version.py

@@ -1,6 +1,127 @@
+import re
 from packaging.version import parse, Version
 
 
+# NOTE: following regex is taken from the 'semver' package as is:
+#       https://python-semver.readthedocs.io/en/2.10.0/readme.html
+SEMVER_REGEX = re.compile(
+    r"""
+        ^
+        (?P<major>0|[1-9]\d*)
+        \.
+        (?P<minor>0|[1-9]\d*)
+        \.
+        (?P<patch>0|[1-9]\d*)
+        (?:-(?P<prerelease>
+            (?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)
+            (?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*
+        ))?
+        (?:\+(?P<build>
+            [0-9a-zA-Z-]+
+            (?:\.[0-9a-zA-Z-]+)*
+        ))?
+        $
+    """,
+    re.VERBOSE,
+)
+
+class ComparableScyllaVersion:
+    """Accepts and compares known 'non-semver' and 'semver'-like Scylla versions."""
+
+    def __init__(self, version_string: str):
+        parsed_version = self.parse(version_string)
+        self.v_major = int(parsed_version[0])
+        self.v_minor = int(parsed_version[1])
+        self.v_patch = int(parsed_version[2])
+        self.v_pre_release = parsed_version[3] or ''
+        self.v_build = parsed_version[4] or ''
+
+    @staticmethod
+    def parse(version_string: str):
+        """Parse scylla-binary and scylla-docker-tag versions into a proper semver structure."""
+        # NOTE: remove 'with build-id' part if exists and other possible non-semver parts
+        _scylla_version = (version_string or '').split(" ")[0]
+
+        # NOTE: replace '~' which gets returned by the scylla binary
+        _scylla_version = _scylla_version.replace('~', '-')
+
+        # NOTE: remove docker-specific parts if version is taken from a docker tag
+        _scylla_version = _scylla_version.replace('-aarch64', '')
+        _scylla_version = _scylla_version.replace('-x86_64', '')
+
+        # NOTE: transform gce-image version like '2024.2.0.dev.0.20231219.c7cdb16538f2.1'
+        if gce_image_v_match := re.search(r"(\d+\.\d+\.\d+\.)([a-z0-9]+\.)(.*)", _scylla_version):
+            _scylla_version = f"{gce_image_v_match[1][:-1]}-{gce_image_v_match[2][:-1]}-{gce_image_v_match[3]}"
+
+        # NOTE: make short scylla version like '5.2' be correct semver string
+        _scylla_version_parts = re.split(r'\.|-', _scylla_version)
+        if len(_scylla_version_parts) == 2:
+            _scylla_version = f"{_scylla_version}.0"
+        elif len(_scylla_version_parts) > 2 and re.search(
+                r"\D+", _scylla_version_parts[2].split("-")[0]):
+            _scylla_version = f"{_scylla_version_parts[0]}.{_scylla_version_parts[1]}.0-{_scylla_version_parts[2]}"
+            for part in _scylla_version_parts[3:]:
+                _scylla_version += f".{part}"
+
+        # NOTE: replace '-0' with 'dev-0', '-1' with 'dev-1' and so on
+        #       to match docker and scylla binary version structures correctly.
+        if no_dev_match := re.search(r"(\d+\.\d+\.\d+)(\-\d+)(\.20[0-9]{6}.*)", _scylla_version):
+            _scylla_version = f"{no_dev_match[1]}-dev{no_dev_match[2]}{no_dev_match[3]}"
+
+        # NOTE: replace '.' with '+' symbol between build date and build commit
+        #       to satisfy semver structure
+        if dotted_build_id_match := re.search(r"(.*\.20[0-9]{6})(\.)([\.\d\w]+)", _scylla_version):
+            _scylla_version = f"{dotted_build_id_match[1]}+{dotted_build_id_match[3]}"
+
+        if match := SEMVER_REGEX.match(_scylla_version):
+            return match.groups()
+        raise ValueError(
+            f"Cannot parse provided '{version_string}' scylla_version for the comparison. "
+            f"Transformed scylla_version: {_scylla_version}")
+
+    def __str__(self):
+        result = f"{self.v_major}.{self.v_minor}.{self.v_patch}"
+        if self.v_pre_release:
+            result += f"-{self.v_pre_release}"
+        if self.v_build:
+            result += f"+{self.v_build}"
+        return result
+
+    def _transform_to_comparable(self, other):
+        if isinstance(other, str):
+            return self.__class__(other)
+        elif isinstance(other, self.__class__):
+            return other
+        raise ValueError("Got unexpected type for the comparison: %s" % type(other))
+
+    def as_comparable(self):
+        # NOTE: absence of the 'pre-release' part means we have 'GA' version which is newer than
+        #       any of the 'pre-release' ones.
+        #       So, make empty 'pre-release' prevail over any defined one.
+        return (self.v_major, self.v_minor, self.v_patch, self.v_pre_release or 'xyz')
+
+    def __lt__(self, other):
+        return self.as_comparable() < self._transform_to_comparable(other).as_comparable()
+
+    def __le__(self, other):
+        return self.as_comparable() <= self._transform_to_comparable(other).as_comparable()
+
+    def __eq__(self, other):
+        return self.as_comparable() == self._transform_to_comparable(other).as_comparable()
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+    def __ge__(self, other):
+        return not self.__lt__(other)
+
+    def __gt__(self, other):
+        return not self.__le__(other)
+
+
+class ComparableCassandraVersion(ComparableScyllaVersion):
+    pass
+
 def parse_version(v: str) -> Version:
     v = v.replace('~', '-')
     return parse(v)

tests/test_config.py

@@ -1,7 +1,6 @@
-import distutils.util
 import os
 
-DEV_MODE = bool(distutils.util.strtobool(os.environ.get("DEV_MODE", "False")))
+
 RESULTS_DIR = "test_results"
 TEST_ID = os.environ.get("CCM_TEST_ID", None)
 SCYLLA_DOCKER_IMAGE = os.environ.get(

tests/test_utils_version.py

@@ -0,0 +1,99 @@
+import pytest
+
+from ccmlib.utils.version import ComparableScyllaVersion
+
+@pytest.mark.parametrize("version_string, expected", (
+    ("5.1", (5, 1, 0, '', '')),
+    ("5.1.0", (5, 1, 0, '', '')),
+    ("5.1.1", (5, 1, 1, '', '')),
+    ("5.1.0-rc1", (5, 1, 0, 'rc1', '')),
+    ("5.1.0~rc1", (5, 1, 0, 'rc1', '')),
+    ("5.1.rc1", (5, 1, 0, 'rc1', '')),
+    ("2022.1.3-0.20220922.539a55e35", (2022, 1, 3, "dev-0.20220922", "539a55e35")),
+    ("2022.1.3-0.20220922.539a55e35 with build-id d1fb2faafd95058a04aad30b675ff7d2b930278d",
+     (2022, 1, 3, "dev-0.20220922", "539a55e35")),
+    ("2022.1.3-dev-0.20220922.539a55e35", (2022, 1, 3, "dev-0.20220922", "539a55e35")),
+    ("5.2.0~rc1-0.20230207.8ff4717fd010", (5, 2, 0, "rc1-0.20230207", "8ff4717fd010")),
+    ("5.2.0-dev-0.20230109.08b3a9c786d9", (5,  2, 0, "dev-0.20230109", "08b3a9c786d9")),
+    ("5.2.0-dev-0.20230109.08b3a9c786d9-x86_64", (5, 2, 0, "dev-0.20230109", "08b3a9c786d9")),
+    ("5.2.0-dev-0.20230109.08b3a9c786d9-aarch64", (5, 2, 0, "dev-0.20230109", "08b3a9c786d9")),
+    ("2024.2.0.dev.0.20231219.c7cdb16538f2.1", (2024, 2, 0, "dev-0.20231219", "c7cdb16538f2.1")),
+    ("2024.1.0.rc2.0.20231218.a063c2c16185.1", (2024, 1, 0, "rc2-0.20231218", "a063c2c16185.1")),
+    ("2.6-dev-0.20211108.5f1e01cbb34-SNAPSHOT-5f1e01cbb34", (2, 6, 0, "dev.0.20211108", '5f1e01cbb34.SNAPSHOT.5f1e01cbb34')),
+))
+def test_comparable_scylla_version_init_positive(version_string, expected):
+    comparable_scylla_version = ComparableScyllaVersion(version_string)
+    assert comparable_scylla_version.v_major == expected[0]
+    assert comparable_scylla_version.v_minor == expected[1]
+    assert comparable_scylla_version.v_patch == expected[2]
+    assert comparable_scylla_version.v_pre_release == expected[3]
+    assert comparable_scylla_version.v_build == expected[4]
+
+
+@pytest.mark.parametrize("version_string", (None, "", "5", "2023", "2023.dev"))
+def test_comparable_scylla_versions_init_negative(version_string):
+    try:
+        ComparableScyllaVersion(version_string)
+    except ValueError:
+        pass
+    else:
+        assert False, (
+            f"'ComparableScyllaVersion' must raise a ValueError for the '{version_string}' "
+            "provided input")
+
+
+def _compare_versions(version_string_left, version_string_right,
+                      is_left_greater, is_equal, comparable_class):
+    comparable_version_left = comparable_class(version_string_left)
+    comparable_version_right = comparable_class(version_string_right)
+
+    compare_expected_result_err_msg = (
+        "One of 'is_left_greater' and 'is_equal' must be 'True' and another one must be 'False'")
+    assert is_left_greater or is_equal, compare_expected_result_err_msg
+    assert not (is_left_greater and is_equal)
+    if is_left_greater:
+        assert comparable_version_left > comparable_version_right
+        assert comparable_version_left >= comparable_version_right
+        assert comparable_version_left > version_string_right
+        assert comparable_version_left >= version_string_right
+        assert comparable_version_right < comparable_version_left
+        assert comparable_version_right <= comparable_version_left
+        assert comparable_version_right < version_string_left
+        assert comparable_version_right <= version_string_left
+    else:
+        assert comparable_version_left == comparable_version_right
+        assert comparable_version_left == version_string_right
+        assert comparable_version_left >= comparable_version_right
+        assert comparable_version_left >= version_string_right
+        assert comparable_version_right <= comparable_version_left
+        assert comparable_version_right <= version_string_left
+
+
+@pytest.mark.parametrize(
+    "version_string_left, version_string_right, is_left_greater, is_equal, comparable_class", (
+        ("5.2.2", "5.2.2", False, True, ComparableScyllaVersion),
+        ("5.2.0", "5.1.2", True, False, ComparableScyllaVersion),
+        ("5.2.1", "5.2.0", True, False, ComparableScyllaVersion),
+        ("5.2.10", "5.2.9", True, False, ComparableScyllaVersion),
+        ("5.2.0", "5.2.0~rc1-0.20230207.8ff4717fd010", True, False, ComparableScyllaVersion),
+        ("5.2.0", "5.2.0-dev-0.20230109.08b3a9c786d9", True, False, ComparableScyllaVersion),
+        ("2023.1.0", "2023.1.rc1", True, False, ComparableScyllaVersion),
+        ("5.2.0", "5.1.rc1", True, False, ComparableScyllaVersion),
+        ("5.2.0-dev-0.20230109.8ff4717fd010", "5.2.0-dev-0.20230109.08b3a9c786d9",
+         False, True, ComparableScyllaVersion),
+    ))
+def test_comparable_scylla_versions_compare(version_string_left, version_string_right,
+                                            is_left_greater, is_equal, comparable_class):
+    _compare_versions(
+        version_string_left, version_string_right, is_left_greater, is_equal, comparable_class)
+
+
+@pytest.mark.parametrize("version_string_input, version_string_output", (
+    ("5.2.2", "5.2.2"),
+    ("2023.1.13", "2023.1.13"),
+    ("5.2.0~rc0-0.20230207", "5.2.0-rc0-0.20230207"),
+    ("5.2.0-rc1-0.20230207", "5.2.0-rc1-0.20230207"),
+    ("5.2.0~dev-0.20230207.8ff4717fd010", "5.2.0-dev-0.20230207+8ff4717fd010"),
+))
+def test_comparable_scylla_versions_to_str(version_string_input, version_string_output):
+    assert str(ComparableScyllaVersion(version_string_input)) == version_string_output