From 27475153553311c9c9dab1aaf9bd6b5f4f0e4925 Mon Sep 17 00:00:00 2001 From: Israel Fruchter Date: Sun, 12 Nov 2023 19:21:05 +0200 Subject: [PATCH] switch to use ruamel.yaml since ruamel.yaml can preserve yaml comment so we can have the original yaml files of scylla --- README.md | 24 +++++++----------------- ccmlib/cluster.py | 6 ++++-- ccmlib/cluster_factory.py | 6 ++++-- ccmlib/common.py | 12 +++++++----- ccmlib/dse_node.py | 9 ++++++--- ccmlib/node.py | 15 +++++++++------ ccmlib/scylla_cluster.py | 12 +++++++----- ccmlib/scylla_docker_cluster.py | 4 +++- ccmlib/scylla_node.py | 21 ++++++++++++--------- ccmlib/scylla_repository.py | 8 ++++++-- ccmlib/utils/sni_proxy.py | 9 ++++++--- flake.lock | 12 ++++++------ flake.nix | 2 +- setup.py | 2 +- tests/test_common.py | 4 ++-- 15 files changed, 81 insertions(+), 65 deletions(-) diff --git a/README.md b/README.md index 96f2ed75..9aad3eee 100644 --- a/README.md +++ b/README.md @@ -55,12 +55,11 @@ ccm create my_cluster -n 3 --scylla --vnodes \ Requirements ------------ -- A working python installation (tested to work with python 3.11). -- pyYAML (http://pyyaml.org/ -- `sudo easy_install pyYaml`) -- ant (http://ant.apache.org/, on Mac OS X, `brew install ant`) -- psutil (https://pypi.python.org/pypi/psutil) -- Java (which version depends on the version of Cassandra you plan to use. If - unsure, use Java 7 as it is known to work with current versions of Cassandra). +- A working python installation (tested to work with python 3.12). +- `pip install -e .` to install the required dependencies. +- Java if cassandra is used or older scylla < 6.0 (which version depends on the version + of Cassandra you plan to use. If unsure, use Java 8 as it is known to + work with current versions of Cassandra). - ccm only works on localhost for now. If you want to create multiple node clusters, the simplest way is to use multiple loopback aliases. On modern linux distributions you probably don't need to do anything, but @@ -75,17 +74,8 @@ Requirements Known issues ------------ -Windows only: - - `node start` pops up a window, stealing focus. - - cli and cqlsh started from ccm show incorrect prompts on command-prompt - - non nodetool-based command-line options fail (sstablesplit, scrub, etc) - - cli_session does not accept commands. - - To install psutil, you must use the .msi from pypi. pip install psutil will not work - - You will need ant.bat in your PATH in order to build C* from source - - You must run with an Unrestricted Powershell Execution-Policy if using Cassandra 2.1.0+ - - Ant installed via [chocolatey](https://chocolatey.org/) will not be found by ccm, so you must create a symbolic - link in order to fix the issue (as administrator): - - cmd /c mklink C:\ProgramData\chocolatey\bin\ant.bat C:\ProgramData\chocolatey\bin\ant.exe + +- this fork of ccm doesn't support Windows Installation ------------ diff --git a/ccmlib/cluster.py b/ccmlib/cluster.py index 51eac814..fa82e7e4 100644 --- a/ccmlib/cluster.py +++ b/ccmlib/cluster.py @@ -8,13 +8,15 @@ from collections import OrderedDict, defaultdict from concurrent.futures import ThreadPoolExecutor -import yaml +from ruamel.yaml import YAML from ccmlib import common, repository from ccmlib.node import Node, NodeError from ccmlib.common import logger from ccmlib.utils.version import parse_version +yaml = YAML() +yaml.default_flow_style = False class Cluster(object): @@ -675,7 +677,7 @@ def _update_config(self, install_dir=None): cluster_config['sni_proxy_listen_port'] = self.sni_proxy_listen_port with open(filename, 'w') as f: - yaml.safe_dump(cluster_config, f) + yaml.dump(cluster_config, f) def __update_pids(self, started): for node, p, _ in started: diff --git a/ccmlib/cluster_factory.py b/ccmlib/cluster_factory.py index c1d56750..7e108969 100644 --- a/ccmlib/cluster_factory.py +++ b/ccmlib/cluster_factory.py @@ -1,6 +1,6 @@ import os -import yaml +from ruamel.yaml import YAML from ccmlib import common, repository from ccmlib.cluster import Cluster @@ -10,6 +10,8 @@ from ccmlib import repository from ccmlib.node import Node +yaml = YAML() +yaml.default_flow_style = False class ClusterFactory(): @@ -18,7 +20,7 @@ def load(path, name): cluster_path = os.path.join(path, name) filename = os.path.join(cluster_path, 'cluster.conf') with open(filename, 'r') as f: - data = yaml.safe_load(f) + data = yaml.load(f) try: install_dir = None scylla_manager_install_path = data.get('scylla_manager_install_path') diff --git a/ccmlib/common.py b/ccmlib/common.py index 12fb4562..668563e0 100644 --- a/ccmlib/common.py +++ b/ccmlib/common.py @@ -21,12 +21,14 @@ from typing import Callable, Optional, TextIO, Union, List from pathlib import Path -import yaml import psutil +from ruamel.yaml import YAML from boto3.session import Session from botocore import UNSIGNED from botocore.client import Config +yaml = YAML() +yaml.default_flow_style = False BIN_DIR = "bin" CASSANDRA_CONF_DIR = "conf" @@ -248,7 +250,7 @@ def get_config(): return {} with open(config_path, 'r') as f: - return yaml.safe_load(f) + return yaml.load(f) def now_ms(): @@ -827,7 +829,7 @@ def parse_settings(args): elif val.lower() == "false": val = False else: - val = yaml.safe_load(val) + val = yaml.load(val) splitted = key.split('.') if len(splitted) == 2: try: @@ -886,7 +888,7 @@ def get_version_from_build(install_dir=None, node_path=None): def get_default_scylla_yaml(install_dir): scylla_yaml_path = Path(install_dir) / SCYLLA_CONF_DIR / SCYLLA_CONF with scylla_yaml_path.open() as f: - return yaml.safe_load(f) + return yaml.load(f) def _get_scylla_version(install_dir): scylla_version_files = [ @@ -968,7 +970,7 @@ def is_dse_cluster(path): cluster_path = os.path.join(path, name) filename = os.path.join(cluster_path, 'cluster.conf') with open(filename, 'r') as f: - data = yaml.safe_load(f) + data = yaml.load(f) if 'dse_dir' in data: return True except IOError: diff --git a/ccmlib/dse_node.py b/ccmlib/dse_node.py index 89e2bba2..12d54198 100644 --- a/ccmlib/dse_node.py +++ b/ccmlib/dse_node.py @@ -9,11 +9,14 @@ import subprocess import time -import yaml +from ruamel.yaml import YAML from ccmlib import common from ccmlib.node import Node, NodeError +yaml = YAML() +yaml.default_flow_style = False + class DseNode(Node): @@ -293,7 +296,7 @@ def import_bin_files(self): def __update_yaml(self): conf_file = os.path.join(self.get_path(), 'resources', 'dse', 'conf', 'dse.yaml') with open(conf_file, 'r') as f: - data = yaml.safe_load(f) + data = yaml.load(f) data['system_key_directory'] = os.path.join(self.get_path(), 'keys') @@ -310,7 +313,7 @@ def __update_yaml(self): data[name] = full_options[name] with open(conf_file, 'w') as f: - yaml.safe_dump(data, f, default_flow_style=False) + yaml.dump(data, f) def _update_log4j(self): super(DseNode, self)._update_log4j() diff --git a/ccmlib/node.py b/ccmlib/node.py index 0b1e5a98..92ccb803 100644 --- a/ccmlib/node.py +++ b/ccmlib/node.py @@ -17,13 +17,16 @@ import locale from collections import namedtuple -import yaml +from ruamel.yaml import YAML from ccmlib import common from ccmlib.cli_session import CliSession from ccmlib.repository import setup from ccmlib.utils.version import parse_version +yaml = YAML() +yaml.default_flow_style = False + class Status(): UNINITIALIZED = "UNINITIALIZED" UP = "UP" @@ -135,7 +138,7 @@ def load(path, name, cluster): node_path = os.path.join(path, name) filename = os.path.join(node_path, 'node.conf') with open(filename, 'r') as f: - data = yaml.safe_load(f) + data = yaml.load(f) try: itf = data['interfaces'] initial_token = None @@ -1583,12 +1586,12 @@ def _update_config(self): if self.workload is not None: values['workload'] = self.workload with open(filename, 'w') as f: - yaml.safe_dump(values, f) + yaml.dump(values, f) def __update_yaml(self): conf_file = os.path.join(self.get_conf_dir(), common.CASSANDRA_CONF) with open(conf_file, 'r') as f: - data = yaml.safe_load(f) + data = yaml.load(f) with open(conf_file, 'r') as f: yaml_text = f.read() @@ -1638,7 +1641,7 @@ def __update_yaml(self): data[name] = full_options[name] with open(conf_file, 'w') as f: - yaml.safe_dump(data, f, default_flow_style=False) + yaml.dump(data, f) def _update_log4j(self): append_pattern = 'log4j.appender.R.File=' @@ -1960,7 +1963,7 @@ def _clean_win_jmx(self): def get_conf_option(self, option): conf_file = os.path.join(self.get_conf_dir(), common.CASSANDRA_CONF) with open(conf_file, 'r') as f: - data = yaml.safe_load(f) + data = yaml.load(f) if option in data: return data[option] diff --git a/ccmlib/scylla_cluster.py b/ccmlib/scylla_cluster.py index 559954c0..a155989b 100644 --- a/ccmlib/scylla_cluster.py +++ b/ccmlib/scylla_cluster.py @@ -4,10 +4,10 @@ import time import subprocess import signal -import yaml import uuid import datetime +from ruamel.yaml import YAML from ccmlib import common from ccmlib.cluster import Cluster @@ -19,6 +19,8 @@ SNITCH = 'org.apache.cassandra.locator.GossipingPropertyFileSnitch' +yaml = YAML() +yaml.default_flow_style = False class ScyllaCluster(Cluster): @@ -244,7 +246,7 @@ def _update_config(self, install_dir=None): filename = os.path.join(self.get_path(), 'cluster.conf') with open(filename, 'r') as f: - data = yaml.safe_load(f) + data = yaml.load(f) if self.is_scylla_reloc(): data['scylla_version'] = self.scylla_version @@ -253,7 +255,7 @@ def _update_config(self, install_dir=None): data['scylla_manager_install_path'] = self._scylla_manager.install_dir with open(filename, 'w') as f: - yaml.safe_dump(data, f) + yaml.dump(data, f) def sctool(self, cmd): if self._scylla_manager == None: @@ -313,7 +315,7 @@ def _get_api_address(self): def _update_config(self, install_dir=None): conf_file = os.path.join(self._get_path(), common.SCYLLAMANAGER_CONF) with open(conf_file, 'r') as f: - data = yaml.safe_load(f) + data = yaml.load(f) data['http'] = self._get_api_address() if not 'database' in data: data['database'] = {} @@ -349,7 +351,7 @@ def _update_config(self, install_dir=None): for key in keys_to_delete: del data[key] with open(conf_file, 'w') as f: - yaml.safe_dump(data, f, default_flow_style=False) + yaml.dump(data, f) def _copy_config_files(self, install_dir): conf_dir = os.path.join(install_dir, 'etc') diff --git a/ccmlib/scylla_docker_cluster.py b/ccmlib/scylla_docker_cluster.py index 08d24d2e..04cae3eb 100644 --- a/ccmlib/scylla_docker_cluster.py +++ b/ccmlib/scylla_docker_cluster.py @@ -3,7 +3,7 @@ from subprocess import check_call, run, PIPE import logging -import yaml +from ruamel.yaml import YAML from ccmlib.scylla_cluster import ScyllaCluster from ccmlib.scylla_node import ScyllaNode @@ -12,6 +12,8 @@ LOGGER = logging.getLogger("ccm") +yaml = YAML() +yaml.default_flow_style = False class ScyllaDockerCluster(ScyllaCluster): def __init__(self, *args, **kwargs): diff --git a/ccmlib/scylla_node.py b/ccmlib/scylla_node.py index d2735d24..02b82019 100644 --- a/ccmlib/scylla_node.py +++ b/ccmlib/scylla_node.py @@ -19,7 +19,7 @@ import logging import psutil -import yaml +from ruamel.yaml import YAML import glob import re import requests @@ -35,6 +35,9 @@ from ccmlib.utils.version import parse_version +yaml = YAML() +yaml.default_flow_style = False + class ScyllaNode(Node): """ @@ -372,18 +375,18 @@ def _create_agent_config(self): data['s3'] = {"endpoint": os.getenv("AWS_S3_ENDPOINT"), "provider": "Minio"} with open(conf_file, 'w') as f: - yaml.safe_dump(data, f, default_flow_style=False) + yaml.dump(data, f) return conf_file def update_agent_config(self, new_settings, restart_agent_after_change=True): conf_file = os.path.join(self.get_conf_dir(), 'scylla-manager-agent.yaml') with open(conf_file, 'r') as f: - current_config = yaml.safe_load(f) + current_config = yaml.load(f) current_config.update(new_settings) with open(conf_file, 'w') as f: - yaml.safe_dump(current_config, f, default_flow_style=False) + yaml.dump(current_config, f) if restart_agent_after_change: self.restart_scylla_manager_agent(gently=True, recreate_config=False) @@ -416,7 +419,7 @@ def start_scylla_manager_agent(self, create_config=True): pid_file.write(str(self._process_agent.pid)) with open(config_file, 'r') as f: - current_config = yaml.safe_load(f) + current_config = yaml.load(f) # Extracting currently configured port current_listening_port = int(current_config['https'].split(":")[1]) @@ -569,7 +572,7 @@ def start(self, join_ring=True, no_wait=False, verbose=False, # from config file scylla#59 conf_file = os.path.join(self.get_conf_dir(), common.SCYLLA_CONF) with open(conf_file, 'r') as f: - data = yaml.safe_load(f) + data = yaml.load(f) jvm_args = jvm_args + ['--api-address', data['api_address']] args = [launch_bin, '--options-file', options_file, '--log-to-stdout', '1'] @@ -1144,7 +1147,7 @@ def update_yaml(self): # TODO: copied from node.py conf_file = os.path.join(self.get_conf_dir(), common.SCYLLA_CONF) with open(conf_file, 'r') as f: - data = yaml.safe_load(f) + data = yaml.load(f) data['cluster_name'] = self.cluster.name data['auto_bootstrap'] = self.auto_bootstrap @@ -1205,7 +1208,7 @@ def update_yaml(self): if 'alternator_port' in data or 'alternator_https_port' in data: data['alternator_address'] = data['listen_address'] with open(conf_file, 'w') as f: - yaml.safe_dump(data, f, default_flow_style=False) + yaml.dump(data, f) # TODO: - for now create a cassandra conf file leaving only # cassandra config items - this should be removed once tools are @@ -1315,7 +1318,7 @@ def update_yaml(self): cassandra_data[key] = data[key] with open(cassandra_conf_file, 'w') as f: - yaml.safe_dump(cassandra_data, f, default_flow_style=False) + yaml.dump(cassandra_data, f) def __update_yaml_dse(self): raise NotImplementedError('ScyllaNode.__update_yaml_dse') diff --git a/ccmlib/scylla_repository.py b/ccmlib/scylla_repository.py index 2909c9b6..79dc47d6 100644 --- a/ccmlib/scylla_repository.py +++ b/ccmlib/scylla_repository.py @@ -14,7 +14,7 @@ from typing import NamedTuple, Literal import requests -import yaml +from ruamel.yaml import YAML import packaging.version from ccmlib.common import ( @@ -23,6 +23,10 @@ from ccmlib.utils.download import download_file, download_version_from_s3, get_url_hash, save_source_file from ccmlib.utils.version import parse_version +yaml = YAML() +yaml.default_flow_style = False + + GIT_REPO = "http://github.com/scylladb/scylla.git" CORE_PACKAGE_DIR_NAME = 'scylla-core-package' @@ -193,7 +197,7 @@ def read_build_manifest(url): # # url-id: 2022-08-29T08:05:34Z # docker-image-name: scylla-nightly:5.2.0-dev-0.20220829.67c91e8bcd61 - return yaml.safe_load(res.content) + return yaml.load(res.content) def normalize_scylla_version(version): diff --git a/ccmlib/utils/sni_proxy.py b/ccmlib/utils/sni_proxy.py index f798316f..ea683e5e 100644 --- a/ccmlib/utils/sni_proxy.py +++ b/ccmlib/utils/sni_proxy.py @@ -11,13 +11,16 @@ from shutil import copytree from dataclasses import dataclass -import yaml +from ruamel.yaml import YAML from ccmlib.utils.ssl_utils import generate_ssl_stores from ccmlib.common import wait_for logger = logging.getLogger(__name__) +yaml = YAML() +yaml.default_flow_style = False + @contextmanager def file_or_memory(path=None, data=None): # since we can't read keys/cert from memory yet @@ -64,7 +67,7 @@ def encode_base64(filename): currentContext='default') with open(os.path.join(ssl_dir, 'config_data.yaml'), 'w') as config_file: - config_file.write(yaml.safe_dump(config, sort_keys=False)) + yaml.dump(config, config_file) datacenters = {} @@ -83,7 +86,7 @@ def encode_base64(filename): currentContext='default') with open(os.path.join(ssl_dir, 'config_path.yaml'), 'w') as config_file: - config_file.write(yaml.safe_dump(config, sort_keys=False)) + yaml.dump(config, config_file) return os.path.join(ssl_dir, 'config_data.yaml'), os.path.join(ssl_dir, 'config_path.yaml') diff --git a/flake.lock b/flake.lock index adae6d0a..e988ac85 100644 --- a/flake.lock +++ b/flake.lock @@ -5,11 +5,11 @@ "systems": "systems" }, "locked": { - "lastModified": 1687709756, - "narHash": "sha256-Y5wKlQSkgEK2weWdOu4J3riRd+kV/VCgHsqLNTTWQ/0=", + "lastModified": 1701680307, + "narHash": "sha256-kAuep2h5ajznlPMD9rnQyffWG8EM/C73lejGofXvdM8=", "owner": "numtide", "repo": "flake-utils", - "rev": "dbabf0ca0c0c4bce6ea5eaf65af5cb694d2082c7", + "rev": "4022d587cbbfd70fe950c1e2083a02621806a725", "type": "github" }, "original": { @@ -20,11 +20,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1688221086, - "narHash": "sha256-cdW6qUL71cNWhHCpMPOJjlw0wzSRP0pVlRn2vqX/VVg=", + "lastModified": 1703134684, + "narHash": "sha256-SQmng1EnBFLzS7WSRyPM9HgmZP2kLJcPAz+Ug/nug6o=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "cd99c2b3c9f160cd004318e0697f90bbd5960825", + "rev": "d6863cbcbbb80e71cecfc03356db1cda38919523", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 1852fbdd..57e59fac 100644 --- a/flake.nix +++ b/flake.nix @@ -9,7 +9,7 @@ localhost."; outputs = { self, nixpkgs, flake-utils }: let - prepare_python_requirements = python: python.withPackages (ps: with ps; [ pytest pyyaml psutil six requests packaging boto3 tqdm setuptools ]); + prepare_python_requirements = python: python.withPackages (ps: with ps; [ pytest ruamel-yaml psutil six requests packaging boto3 tqdm setuptools ]); make_ccm_package = {python, jdk, pkgs}: python.pkgs.buildPythonApplication { pname = "scylla_ccm"; version = "0.1"; diff --git a/setup.py b/setup.py index 35fff3b7..dfb91cba 100755 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ url='https://github.com/pcmanus/ccm', packages=['ccmlib', 'ccmlib.cmds', 'ccmlib.utils'], scripts=[ccmscript], - install_requires=['pyYaml', 'psutil', 'requests', 'packaging', 'boto3', 'tqdm', 'urllib3<2'], + install_requires=['ruamel.yaml', 'psutil', 'requests', 'packaging', 'boto3', 'tqdm', 'urllib3<2'], tests_require=['pytest'], classifiers=[ "License :: OSI Approved :: Apache Software License", diff --git a/tests/test_common.py b/tests/test_common.py index d3f809d5..c16e8e6b 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -2,7 +2,7 @@ import os import pytest -import yaml +import ruamel from ccmlib.common import scylla_extract_mode, LockFile, parse_settings @@ -94,5 +94,5 @@ def test_parse_settings(): assert res == {'experimental_features': ['udf', 'tablets']} # would break if incorrect yaml format is passed in the value - with pytest.raises(yaml.parser.ParserError): + with pytest.raises(ruamel.yaml.parser.ParserError): parse_settings(["experimental_features:['udf',\"tablets\""])