Skip to content

Commit

Permalink
ccmlib/scylla_node.py: prepare for the post-jmx world
Browse files Browse the repository at this point in the history 
Soon the jmx package will be dropped from the unified package, and the
scylla.git repo. Prepare for this:
* Add `use_jmx` property, based on probing the reloc package for the
  presence of the `jmx` directory, or the `tools/jmx` directory in the
  scylla repo.
* Use this new property to fence off jmx-related code. nodetool() now
  routes to the native nodetool directly, when use_jmx == False.
  • Loading branch information
@denesb @fruch
denesb authored and fruch committed Apr 2, 2024
1 parent cdf7ae6 commit b18f85d
Showing 1 changed file with 66 additions and 39 deletions.
105 changes: 66 additions & 39 deletions ccmlib/scylla_node.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ def __init__(self, name, cluster, auto_bootstrap, thrift_interface,
self._node_install_dir = None
self._node_scylla_version = None
self._relative_repos_root = None
self._has_jmx = None
super().__init__(name, cluster, auto_bootstrap,
thrift_interface, storage_interface,
jmx_port, remote_debug_port,
Expand Down Expand Up @@ -83,6 +84,8 @@ def node_install_dir(self):
@node_install_dir.setter
def node_install_dir(self, install_dir):
self._node_install_dir = install_dir
# Force re-check based on the new install dir
self._has_jmx = None

@property
def node_scylla_version(self):
Expand All @@ -98,6 +101,17 @@ def node_scylla_version(self, install_dir):
def scylla_build_id(self):
return self._run_scylla_executable_with_option(option="--build-id")

@property
def has_jmx(self):
if self._has_jmx is None:
install_dir = self.node_install_dir
if self.is_scylla_reloc():
self._has_jmx = os.path.isdir(os.path.join(install_dir, "jmx"))
else:
self._has_jmx = os.path.isdir(os.path.join(install_dir, "tools", "jmx"))

return self._has_jmx

def scylla_mode(self):
return self.cluster.get_scylla_mode()

Expand Down Expand Up @@ -688,14 +702,15 @@ def process_opts(opts):
wait_normal_token_owner=wait_normal_token_owner,
wait_for_binary_proto=wait_for_binary_proto,
ext_env=ext_env)
self._start_jmx(data)
if self.has_jmx:
self._start_jmx(data)

ip_addr, _ = self.network_interfaces['storage']
jmx_port = int(self.jmx_port)
if not self._wait_java_up(ip_addr, jmx_port):
e_msg = "Error starting node {}: unable to connect to scylla-jmx port {}:{}".format(
self.name, ip_addr, jmx_port)
raise NodeError(e_msg, scylla_process)
ip_addr, _ = self.network_interfaces['storage']
jmx_port = int(self.jmx_port)
if not self._wait_java_up(ip_addr, jmx_port):
e_msg = "Error starting node {}: unable to connect to scylla-jmx port {}:{}".format(
self.name, ip_addr, jmx_port)
raise NodeError(e_msg, scylla_process)

self._update_pid(scylla_process)
wait_for(func=lambda: self.is_running(), timeout=10, step=0.01)
Expand Down Expand Up @@ -761,9 +776,19 @@ def _update_jmx_pid(self, wait=True):
self.jmx_pid = None

def nodetool(self, cmd, capture_output=True, wait=True, timeout=None, verbose=True):
"""
Kill scylla-jmx in case of timeout, to supply enough debugging information
"""
if not self.has_jmx:
if self.is_docker():
host = 'localhost'
else:
host = self.address()
nodetool = [os.path.join(self.get_bin_dir(), "scylla"), "nodetool"]
nodetool.extend(['-h', host, '-p', '10000'])
nodetool.extend(cmd.split())
return self._do_run_nodetool(nodetool, capture_output, wait, timeout, verbose)

# Fall-back to the java nodetool for pre 5.5.0~dev versions, which don't yet have the native nodetool
# Kill scylla-jmx in case of timeout, to supply enough debugging information

# pass the api_port to nodetool. if it is the nodetool-wrapper. it should
# interpret the command line and use it for the -p option
cmd = f"-Dcom.scylladb.apiPort=10000 {cmd}"
Expand Down Expand Up @@ -809,7 +834,8 @@ def do_stop(self, gently=True):
"""

did_stop = False
self._update_jmx_pid(wait=False)
if self.has_jmx:
self._update_jmx_pid(wait=False)
if self.scylla_manager and self.scylla_manager.is_agent_available:
self._update_scylla_agent_pid()
for proc in [self._process_jmx, self._process_scylla, self._process_agent]:
Expand Down Expand Up @@ -1061,34 +1087,35 @@ def run_patchelf(patchelf_cmd):
if returncode != 0:
raise RuntimeError(f'{patchelf_cmd} exited with status {returncode}.\nstdout:{stdout}\nstderr:\n{stderr}')

if 'scylla-repository' in self.node_install_dir:
self.hard_link_or_copy(os.path.join(self.get_jmx_dir(), 'scylla-jmx-1.0.jar'),
os.path.join(self.get_bin_dir(), 'scylla-jmx-1.0.jar'), replace=replace)
self.hard_link_or_copy(os.path.join(self.get_jmx_dir(), 'scylla-jmx'),
os.path.join(self.get_bin_dir(), 'scylla-jmx'), replace=replace)
select_java = Path(self.get_jmx_dir()) / 'select-java'
else:
self.hard_link_or_copy(os.path.join(self.get_jmx_dir(), 'target', 'scylla-jmx-1.0.jar'),
os.path.join(self.get_bin_dir(), 'scylla-jmx-1.0.jar'), replace=replace)
self.hard_link_or_copy(os.path.join(self.get_jmx_dir(), 'scripts', 'scylla-jmx'),
os.path.join(self.get_bin_dir(), 'scylla-jmx'), replace=replace)
select_java = Path(self.get_jmx_dir()) / 'scripts' / 'select-java'

os.makedirs(os.path.join(self.get_bin_dir(), 'symlinks'), exist_ok=exist_ok)
scylla_jmx_file = os.path.join(self.get_bin_dir(), 'symlinks', 'scylla-jmx')
if os.path.exists(scylla_jmx_file) and replace:
os.remove(scylla_jmx_file)
if java_home := os.environ.get('JAVA_HOME'):
# user selecting specific Java
java_exe = Path(java_home) / 'bin' / 'java'
os.symlink(java_exe, scylla_jmx_file)
elif select_java.exists():
# JMX lookup logic
os.symlink(select_java, scylla_jmx_file)
else:
# older scylla versions, just use default java
java_exe = Path('/usr') / 'bin' / 'java'
os.symlink(java_exe, scylla_jmx_file)
if self.has_jmx:
if 'scylla-repository' in self.node_install_dir:
self.hard_link_or_copy(os.path.join(self.get_jmx_dir(), 'scylla-jmx-1.0.jar'),
os.path.join(self.get_bin_dir(), 'scylla-jmx-1.0.jar'), replace=replace)
self.hard_link_or_copy(os.path.join(self.get_jmx_dir(), 'scylla-jmx'),
os.path.join(self.get_bin_dir(), 'scylla-jmx'), replace=replace)
select_java = Path(self.get_jmx_dir()) / 'select-java'
else:
self.hard_link_or_copy(os.path.join(self.get_jmx_dir(), 'target', 'scylla-jmx-1.0.jar'),
os.path.join(self.get_bin_dir(), 'scylla-jmx-1.0.jar'), replace=replace)
self.hard_link_or_copy(os.path.join(self.get_jmx_dir(), 'scripts', 'scylla-jmx'),
os.path.join(self.get_bin_dir(), 'scylla-jmx'), replace=replace)
select_java = Path(self.get_jmx_dir()) / 'scripts' / 'select-java'

os.makedirs(os.path.join(self.get_bin_dir(), 'symlinks'), exist_ok=exist_ok)
scylla_jmx_file = os.path.join(self.get_bin_dir(), 'symlinks', 'scylla-jmx')
if os.path.exists(scylla_jmx_file) and replace:
os.remove(scylla_jmx_file)
if java_home := os.environ.get('JAVA_HOME'):
# user selecting specific Java
java_exe = Path(java_home) / 'bin' / 'java'
os.symlink(java_exe, scylla_jmx_file)
elif select_java.exists():
# JMX lookup logic
os.symlink(select_java, scylla_jmx_file)
else:
# older scylla versions, just use default java
java_exe = Path('/usr') / 'bin' / 'java'
os.symlink(java_exe, scylla_jmx_file)

parent_dir = os.path.dirname(os.path.realpath(__file__))
resources_bin_dir = os.path.join(parent_dir, 'resources', BIN_DIR)
Expand Down

0 comments on commit b18f85d

Please sign in to comment.