Skip to content

Commit

Permalink
switch to use ruamel.yaml
Browse files Browse the repository at this point in the history
since ruamel.yaml can preserve yaml comment
so we can have the original yaml files
of scylla

(cherry picked from commit 2747515)
  • Loading branch information
fruch committed Sep 15, 2024
1 parent 9935dd2 commit db97bb1
Show file tree
Hide file tree
Showing 15 changed files with 84 additions and 63 deletions.
24 changes: 7 additions & 17 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -55,12 +55,11 @@ ccm create my_cluster -n 3 --scylla --vnodes \
Requirements
------------

- A working python installation (tested to work with python 3.11).
- pyYAML (http://pyyaml.org/ -- `sudo easy_install pyYaml`)
- ant (http://ant.apache.org/, on Mac OS X, `brew install ant`)
- psutil (https://pypi.python.org/pypi/psutil)
- Java (which version depends on the version of Cassandra you plan to use. If
unsure, use Java 7 as it is known to work with current versions of Cassandra).
- A working python installation (tested to work with python 3.12).
- `pip install -e .` to install the required dependencies.
- Java if cassandra is used or older scylla < 6.0 (which version depends on the version
of Cassandra you plan to use. If unsure, use Java 8 as it is known to
work with current versions of Cassandra).
- ccm only works on localhost for now. If you want to create multiple
node clusters, the simplest way is to use multiple loopback aliases. On
modern linux distributions you probably don't need to do anything, but
Expand All @@ -75,17 +74,8 @@ Requirements

Known issues
------------
Windows only:
- `node start` pops up a window, stealing focus.
- cli and cqlsh started from ccm show incorrect prompts on command-prompt
- non nodetool-based command-line options fail (sstablesplit, scrub, etc)
- cli_session does not accept commands.
- To install psutil, you must use the .msi from pypi. pip install psutil will not work
- You will need ant.bat in your PATH in order to build C* from source
- You must run with an Unrestricted Powershell Execution-Policy if using Cassandra 2.1.0+
- Ant installed via [chocolatey](https://chocolatey.org/) will not be found by ccm, so you must create a symbolic
link in order to fix the issue (as administrator):
- cmd /c mklink C:\ProgramData\chocolatey\bin\ant.bat C:\ProgramData\chocolatey\bin\ant.exe

- this fork of ccm doesn't support Windows

Installation
------------
Expand Down
6 changes: 4 additions & 2 deletions ccmlib/cluster.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,15 @@
from collections import OrderedDict, defaultdict
from concurrent.futures import ThreadPoolExecutor

import yaml
from ruamel.yaml import YAML

from ccmlib import common, repository
from ccmlib.node import Node, NodeError
from ccmlib.common import logger
from ccmlib.utils.version import parse_version

yaml = YAML()
yaml.default_flow_style = False

class Cluster(object):

Expand Down Expand Up @@ -675,7 +677,7 @@ def _update_config(self, install_dir=None):
cluster_config['sni_proxy_listen_port'] = self.sni_proxy_listen_port

with open(filename, 'w') as f:
yaml.safe_dump(cluster_config, f)
yaml.dump(cluster_config, f)

def __update_pids(self, started):
for node, p, _ in started:
Expand Down
6 changes: 4 additions & 2 deletions ccmlib/cluster_factory.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import os

import yaml
from ruamel.yaml import YAML

from ccmlib import common, repository
from ccmlib.cluster import Cluster
Expand All @@ -10,6 +10,8 @@
from ccmlib import repository
from ccmlib.node import Node

yaml = YAML()
yaml.default_flow_style = False

class ClusterFactory():

Expand All @@ -18,7 +20,7 @@ def load(path, name):
cluster_path = os.path.join(path, name)
filename = os.path.join(cluster_path, 'cluster.conf')
with open(filename, 'r') as f:
data = yaml.safe_load(f)
data = yaml.load(f)
try:
install_dir = None
scylla_manager_install_path = data.get('scylla_manager_install_path')
Expand Down
13 changes: 10 additions & 3 deletions ccmlib/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,18 @@
from typing import Callable, Optional, TextIO, Union, List
from pathlib import Path

<<<<<<< HEAD
import yaml
=======
import psutil
from ruamel.yaml import YAML
>>>>>>> 2747515 (switch to use ruamel.yaml)
from boto3.session import Session
from botocore import UNSIGNED
from botocore.client import Config

yaml = YAML()
yaml.default_flow_style = False

BIN_DIR = "bin"
CASSANDRA_CONF_DIR = "conf"
Expand Down Expand Up @@ -246,7 +253,7 @@ def get_config():
return {}

with open(config_path, 'r') as f:
return yaml.safe_load(f)
return yaml.load(f)


def now_ms():
Expand Down Expand Up @@ -787,7 +794,7 @@ def parse_settings(args):
elif val.lower() == "false":
val = False
else:
val = yaml.safe_load(val)
val = yaml.load(val)
splitted = key.split('.')
if len(splitted) == 2:
try:
Expand Down Expand Up @@ -924,7 +931,7 @@ def is_dse_cluster(path):
cluster_path = os.path.join(path, name)
filename = os.path.join(cluster_path, 'cluster.conf')
with open(filename, 'r') as f:
data = yaml.safe_load(f)
data = yaml.load(f)
if 'dse_dir' in data:
return True
except IOError:
Expand Down
9 changes: 6 additions & 3 deletions ccmlib/dse_node.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,14 @@
import subprocess
import time

import yaml
from ruamel.yaml import YAML

from ccmlib import common
from ccmlib.node import Node, NodeError

yaml = YAML()
yaml.default_flow_style = False


class DseNode(Node):

Expand Down Expand Up @@ -293,7 +296,7 @@ def import_bin_files(self):
def __update_yaml(self):
conf_file = os.path.join(self.get_path(), 'resources', 'dse', 'conf', 'dse.yaml')
with open(conf_file, 'r') as f:
data = yaml.safe_load(f)
data = yaml.load(f)

data['system_key_directory'] = os.path.join(self.get_path(), 'keys')

Expand All @@ -310,7 +313,7 @@ def __update_yaml(self):
data[name] = full_options[name]

with open(conf_file, 'w') as f:
yaml.safe_dump(data, f, default_flow_style=False)
yaml.dump(data, f)

def _update_log4j(self):
super(DseNode, self)._update_log4j()
Expand Down
15 changes: 9 additions & 6 deletions ccmlib/node.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,16 @@
import locale
from collections import namedtuple

import yaml
from ruamel.yaml import YAML

from ccmlib import common
from ccmlib.cli_session import CliSession
from ccmlib.repository import setup
from ccmlib.utils.version import parse_version

yaml = YAML()
yaml.default_flow_style = False

class Status():
UNINITIALIZED = "UNINITIALIZED"
UP = "UP"
Expand Down Expand Up @@ -135,7 +138,7 @@ def load(path, name, cluster):
node_path = os.path.join(path, name)
filename = os.path.join(node_path, 'node.conf')
with open(filename, 'r') as f:
data = yaml.safe_load(f)
data = yaml.load(f)
try:
itf = data['interfaces']
initial_token = None
Expand Down Expand Up @@ -1581,12 +1584,12 @@ def _update_config(self):
if self.workload is not None:
values['workload'] = self.workload
with open(filename, 'w') as f:
yaml.safe_dump(values, f)
yaml.dump(values, f)

def __update_yaml(self):
conf_file = os.path.join(self.get_conf_dir(), common.CASSANDRA_CONF)
with open(conf_file, 'r') as f:
data = yaml.safe_load(f)
data = yaml.load(f)

with open(conf_file, 'r') as f:
yaml_text = f.read()
Expand Down Expand Up @@ -1636,7 +1639,7 @@ def __update_yaml(self):
data[name] = full_options[name]

with open(conf_file, 'w') as f:
yaml.safe_dump(data, f, default_flow_style=False)
yaml.dump(data, f)

def _update_log4j(self):
append_pattern = 'log4j.appender.R.File='
Expand Down Expand Up @@ -1958,7 +1961,7 @@ def _clean_win_jmx(self):
def get_conf_option(self, option):
conf_file = os.path.join(self.get_conf_dir(), common.CASSANDRA_CONF)
with open(conf_file, 'r') as f:
data = yaml.safe_load(f)
data = yaml.load(f)

if option in data:
return data[option]
Expand Down
12 changes: 7 additions & 5 deletions ccmlib/scylla_cluster.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,10 @@
import time
import subprocess
import signal
import yaml
import uuid
import datetime

from ruamel.yaml import YAML

from ccmlib import common
from ccmlib.cluster import Cluster
Expand All @@ -19,6 +19,8 @@

SNITCH = 'org.apache.cassandra.locator.GossipingPropertyFileSnitch'

yaml = YAML()
yaml.default_flow_style = False

class ScyllaCluster(Cluster):

Expand Down Expand Up @@ -244,7 +246,7 @@ def _update_config(self, install_dir=None):
filename = os.path.join(self.get_path(), 'cluster.conf')

with open(filename, 'r') as f:
data = yaml.safe_load(f)
data = yaml.load(f)

if self.is_scylla_reloc():
data['scylla_version'] = self.scylla_version
Expand All @@ -253,7 +255,7 @@ def _update_config(self, install_dir=None):
data['scylla_manager_install_path'] = self._scylla_manager.install_dir

with open(filename, 'w') as f:
yaml.safe_dump(data, f)
yaml.dump(data, f)

def sctool(self, cmd):
if self._scylla_manager == None:
Expand Down Expand Up @@ -313,7 +315,7 @@ def _get_api_address(self):
def _update_config(self, install_dir=None):
conf_file = os.path.join(self._get_path(), common.SCYLLAMANAGER_CONF)
with open(conf_file, 'r') as f:
data = yaml.safe_load(f)
data = yaml.load(f)
data['http'] = self._get_api_address()
if not 'database' in data:
data['database'] = {}
Expand Down Expand Up @@ -349,7 +351,7 @@ def _update_config(self, install_dir=None):
for key in keys_to_delete:
del data[key]
with open(conf_file, 'w') as f:
yaml.safe_dump(data, f, default_flow_style=False)
yaml.dump(data, f)

def _copy_config_files(self, install_dir):
conf_dir = os.path.join(install_dir, 'etc')
Expand Down
4 changes: 3 additions & 1 deletion ccmlib/scylla_docker_cluster.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
from subprocess import check_call, run, PIPE
import logging

import yaml
from ruamel.yaml import YAML

from ccmlib.scylla_cluster import ScyllaCluster
from ccmlib.scylla_node import ScyllaNode
Expand All @@ -12,6 +12,8 @@

LOGGER = logging.getLogger("ccm")

yaml = YAML()
yaml.default_flow_style = False

class ScyllaDockerCluster(ScyllaCluster):
def __init__(self, *args, **kwargs):
Expand Down
21 changes: 12 additions & 9 deletions ccmlib/scylla_node.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
import logging

import psutil
import yaml
from ruamel.yaml import YAML
import glob
import re
import requests
Expand All @@ -35,6 +35,9 @@
from ccmlib.utils.version import parse_version


yaml = YAML()
yaml.default_flow_style = False

class ScyllaNode(Node):

"""
Expand Down Expand Up @@ -372,18 +375,18 @@ def _create_agent_config(self):
data['s3'] = {"endpoint": os.getenv("AWS_S3_ENDPOINT"), "provider": "Minio"}

with open(conf_file, 'w') as f:
yaml.safe_dump(data, f, default_flow_style=False)
yaml.dump(data, f)
return conf_file

def update_agent_config(self, new_settings, restart_agent_after_change=True):
conf_file = os.path.join(self.get_conf_dir(), 'scylla-manager-agent.yaml')
with open(conf_file, 'r') as f:
current_config = yaml.safe_load(f)
current_config = yaml.load(f)

current_config.update(new_settings)

with open(conf_file, 'w') as f:
yaml.safe_dump(current_config, f, default_flow_style=False)
yaml.dump(current_config, f)

if restart_agent_after_change:
self.restart_scylla_manager_agent(gently=True, recreate_config=False)
Expand Down Expand Up @@ -416,7 +419,7 @@ def start_scylla_manager_agent(self, create_config=True):
pid_file.write(str(self._process_agent.pid))

with open(config_file, 'r') as f:
current_config = yaml.safe_load(f)
current_config = yaml.load(f)
# Extracting currently configured port
current_listening_port = int(current_config['https'].split(":")[1])

Expand Down Expand Up @@ -569,7 +572,7 @@ def start(self, join_ring=True, no_wait=False, verbose=False,
# from config file scylla#59
conf_file = os.path.join(self.get_conf_dir(), common.SCYLLA_CONF)
with open(conf_file, 'r') as f:
data = yaml.safe_load(f)
data = yaml.load(f)
jvm_args = jvm_args + ['--api-address', data['api_address']]

args = [launch_bin, '--options-file', options_file, '--log-to-stdout', '1']
Expand Down Expand Up @@ -1142,7 +1145,7 @@ def update_yaml(self):
# TODO: copied from node.py
conf_file = os.path.join(self.get_conf_dir(), common.SCYLLA_CONF)
with open(conf_file, 'r') as f:
data = yaml.safe_load(f)
data = yaml.load(f)

data['cluster_name'] = self.cluster.name
data['auto_bootstrap'] = self.auto_bootstrap
Expand Down Expand Up @@ -1203,7 +1206,7 @@ def update_yaml(self):
if 'alternator_port' in data or 'alternator_https_port' in data:
data['alternator_address'] = data['listen_address']
with open(conf_file, 'w') as f:
yaml.safe_dump(data, f, default_flow_style=False)
yaml.dump(data, f)

# TODO: - for now create a cassandra conf file leaving only
# cassandra config items - this should be removed once tools are
Expand Down Expand Up @@ -1313,7 +1316,7 @@ def update_yaml(self):
cassandra_data[key] = data[key]

with open(cassandra_conf_file, 'w') as f:
yaml.safe_dump(cassandra_data, f, default_flow_style=False)
yaml.dump(cassandra_data, f)

def __update_yaml_dse(self):
raise NotImplementedError('ScyllaNode.__update_yaml_dse')
Expand Down
Loading

0 comments on commit db97bb1

Please sign in to comment.