Enable JMX server configuration to be in cassandra.yaml

patch by Zhongxiang Zheng; reviewed by Stefan Miklosovic, Maulin Vasavada, Cheng Wang, Jordan West for CASSANDRA-11695 Co-authored-by: Stefan Miklosovic <[email protected]> Co-authored-by: Sam Tunnicliffe <[email protected]>
apache · Dec 16, 2024 · 2ff4155 · 2ff4155
1 parent 54e4688
commit 2ff4155
Show file tree

Hide file tree

Showing 38 changed files with 1,042 additions and 574 deletions.
diff --git a/CHANGES.txt b/CHANGES.txt
@@ -1,4 +1,5 @@
 5.1
+ * Enable JMX server configuration to be in cassandra.yaml (CASSANDRA-11695)
  * Parallelized UCS compactions (CASSANDRA-18802)
  * Avoid prepared statement invalidation race when committing schema changes (CASSANDRA-20116)
  * Restore optimization in MultiCBuilder around building one clustering (CASSANDRA-20129)

diff --git a/NEWS.txt b/NEWS.txt
@@ -88,7 +88,6 @@ New features
       generate a password of configured password strength policy upon role creation or alteration
       when 'GENERATED PASSWORD' clause is used. Character sets supported are: English, Cyrillic, modern Cyrillic,
       German, Polish and Czech.
-    - JMX SSL configuration can be now done in cassandra.yaml via jmx_encryption_options section instead of cassandra-env.sh
     - There is new MBean of name org.apache.cassandra.service.snapshot:type=SnapshotManager which exposes user-facing
       snapshot operations. Snapshot-related methods on StorageServiceMBean are still present and functional
       but marked as deprecated.
@@ -102,6 +101,12 @@ New features
       compactions. To avoid the possibility of starving background compactions from resources, the number of threads
       used for major compactions is limited to half the available compaction threads by default, and can be controlled
       by a new --jobs / -j option of nodetool compact.
+    - It is possible to configure JMX server in cassandra.yaml in jmx_server_options configuration section.
+      JMX SSL configuration can be configured via jmx_encryption_options in jmx_server_options. The old way of
+      configuring JMX is still present and default, but new way is preferable as it will e.g. not leak credentials for
+      JMX to JVM parameters. You have to opt-in to use the configuration via cassandra.yaml by uncommenting
+      the respective configuration sections and by commenting out `configure_jmx` function call in cassandra-env.sh.
+      Enabling both ways of configuring JMX will result in a node failing to start.
 
 
 Upgrading

diff --git a/bin/nodetool b/bin/nodetool
@@ -52,6 +52,19 @@ if [ -f "$CASSANDRA_CONF/cassandra-env.sh" ]; then
     JVM_OPTS="$JVM_OPTS_SAVE"
 fi
 
+# In case JMX_PORT is not set (when configure_jmx in cassandra-env.sh is commented out),
+# try to parse it from cassandra.yaml.
+if [ "x$JMX_PORT" = "x" ]; then
+    if [ -f "$CASSANDRA_CONF/cassandra.yaml" ]; then
+        JMX_PORT=`grep jmx_port $CASSANDRA_CONF/cassandra.yaml | cut -d ':' -f 2 | tr -d '[[:space:]]'`
+    fi
+fi
+
+# If, by any chance, it is not there either, set it to default.
+if [ "x$JMX_PORT" = "x" ]; then
+    JMX_PORT=7199
+fi
+
 # JMX Port passed via cmd line args (-p 9999 / --port 9999 / --port=9999)
 # should override the value from cassandra-env.sh
 ARGS=""

diff --git a/conf/cassandra-env.sh b/conf/cassandra-env.sh
@@ -218,55 +218,66 @@ if [ "x$LOCAL_JMX" = "x" ]; then
     LOCAL_JMX=yes
 fi
 
-# Specifies the default port over which Cassandra will be available for
-# JMX connections.
-# For security reasons, you should not expose this port to the internet.  Firewall it if needed.
-JMX_PORT="7199"
-
-if [ "$LOCAL_JMX" = "yes" ]; then
-  JVM_OPTS="$JVM_OPTS -Dcassandra.jmx.local.port=$JMX_PORT"
-  JVM_OPTS="$JVM_OPTS -Dcom.sun.management.jmxremote.authenticate=false"
-else
-  JVM_OPTS="$JVM_OPTS -Dcassandra.jmx.remote.port=$JMX_PORT"
-  # if ssl is enabled the same port cannot be used for both jmx and rmi so either
-  # pick another value for this property or comment out to use a random port (though see CASSANDRA-7087 for origins)
-  JVM_OPTS="$JVM_OPTS -Dcom.sun.management.jmxremote.rmi.port=$JMX_PORT"
-
-  # turn on JMX authentication. See below for further options
-  JVM_OPTS="$JVM_OPTS -Dcom.sun.management.jmxremote.authenticate=true"
-
-  # jmx ssl options
-  # Consider using the jmx_encryption_options section of cassandra.yaml instead
-  # to prevent sensitive information being exposed.
-  # In case jmx ssl options are configured in both the places - this file and cassandra.yaml, and
-  # if com.sun.management.jmxremote.ssl is configured to be true here and encryption_options are marked enabled in
-  # cassandra.yaml, then we will get exception at the startup
-  #JVM_OPTS="$JVM_OPTS -Dcom.sun.management.jmxremote.ssl=true"
-  #JVM_OPTS="$JVM_OPTS -Dcom.sun.management.jmxremote.ssl.need.client.auth=true"
-  #JVM_OPTS="$JVM_OPTS -Dcom.sun.management.jmxremote.ssl.enabled.protocols=<enabled-protocols>"
-  #JVM_OPTS="$JVM_OPTS -Dcom.sun.management.jmxremote.ssl.enabled.cipher.suites=<enabled-cipher-suites>"
-  #JVM_OPTS="$JVM_OPTS -Djavax.net.ssl.keyStore=/path/to/keystore"
-  #JVM_OPTS="$JVM_OPTS -Djavax.net.ssl.keyStorePassword=<keystore-password>"
-  #JVM_OPTS="$JVM_OPTS -Djavax.net.ssl.trustStore=/path/to/truststore"
-  #JVM_OPTS="$JVM_OPTS -Djavax.net.ssl.trustStorePassword=<truststore-password>"
-fi
+configure_jmx()
+{
+  JMX_PORT=$1
+
+  if [ "$LOCAL_JMX" = "yes" ]; then
+    JVM_OPTS="$JVM_OPTS -Dcassandra.jmx.local.port=$JMX_PORT"
+    JVM_OPTS="$JVM_OPTS -Dcom.sun.management.jmxremote.authenticate=false"
+  else
+    JVM_OPTS="$JVM_OPTS -Dcassandra.jmx.remote.port=$JMX_PORT"
+    # if ssl is enabled the same port cannot be used for both jmx and rmi so either
+    # pick another value for this property or comment out to use a random port (though see CASSANDRA-7087 for origins)
+    JVM_OPTS="$JVM_OPTS -Dcom.sun.management.jmxremote.rmi.port=$JMX_PORT"
+
+    # turn on JMX authentication. See below for further options
+    JVM_OPTS="$JVM_OPTS -Dcom.sun.management.jmxremote.authenticate=true"
+
+    # jmx ssl options
+    # Consider using the jmx_encryption_options section of jmx_server_options in cassandra.yaml instead
+    # to prevent sensitive information being exposed.
+    # In case jmx ssl options are configured in both the places - this file and cassandra.yaml, and
+    # if com.sun.management.jmxremote.ssl is configured to be true here and encryption_options are marked enabled in
+    # cassandra.yaml, then we will get exception at the startup
+    #JVM_OPTS="$JVM_OPTS -Dcom.sun.management.jmxremote.ssl=true"
+    #JVM_OPTS="$JVM_OPTS -Dcom.sun.management.jmxremote.ssl.need.client.auth=true"
+    #JVM_OPTS="$JVM_OPTS -Dcom.sun.management.jmxremote.ssl.enabled.protocols=<enabled-protocols>"
+    #JVM_OPTS="$JVM_OPTS -Dcom.sun.management.jmxremote.ssl.enabled.cipher.suites=<enabled-cipher-suites>"
+    #JVM_OPTS="$JVM_OPTS -Djavax.net.ssl.keyStore=/path/to/keystore"
+    #JVM_OPTS="$JVM_OPTS -Djavax.net.ssl.keyStorePassword=<keystore-password>"
+    #JVM_OPTS="$JVM_OPTS -Djavax.net.ssl.trustStore=/path/to/truststore"
+    #JVM_OPTS="$JVM_OPTS -Djavax.net.ssl.trustStorePassword=<truststore-password>"
+  fi
+
+  # jmx authentication and authorization options. By default, auth is only
+  # activated for remote connections but they can also be enabled for local only JMX
+  ## Basic file based authn & authz
+  JVM_OPTS="$JVM_OPTS -Dcom.sun.management.jmxremote.password.file=/etc/cassandra/jmxremote.password"
+  #JVM_OPTS="$JVM_OPTS -Dcom.sun.management.jmxremote.access.file=/etc/cassandra/jmxremote.access"
+  ## Custom auth settings which can be used as alternatives to JMX's out of the box auth utilities.
+  ## JAAS login modules can be used for authentication by uncommenting these two properties.
+  ## Cassandra ships with a LoginModule implementation - org.apache.cassandra.auth.CassandraLoginModule -
+  ## which delegates to the IAuthenticator configured in cassandra.yaml. See the sample JAAS configuration
+  ## file cassandra-jaas.config
+  #JVM_OPTS="$JVM_OPTS -Dcassandra.jmx.remote.login.config=CassandraLogin"
+  #JVM_OPTS="$JVM_OPTS -Djava.security.auth.login.config=$CASSANDRA_CONF/cassandra-jaas.config"
+
+  ## Cassandra also ships with a helper for delegating JMX authz calls to the configured IAuthorizer,
+  ## uncomment this to use it. Requires one of the two authentication options to be enabled
+  #JVM_OPTS="$JVM_OPTS -Dcassandra.jmx.authorizer=org.apache.cassandra.auth.jmx.AuthorizationProxy"
+}
 
-# jmx authentication and authorization options. By default, auth is only
-# activated for remote connections but they can also be enabled for local only JMX
-## Basic file based authn & authz
-JVM_OPTS="$JVM_OPTS -Dcom.sun.management.jmxremote.password.file=/etc/cassandra/jmxremote.password"
-#JVM_OPTS="$JVM_OPTS -Dcom.sun.management.jmxremote.access.file=/etc/cassandra/jmxremote.access"
-## Custom auth settings which can be used as alternatives to JMX's out of the box auth utilities.
-## JAAS login modules can be used for authentication by uncommenting these two properties.
-## Cassandra ships with a LoginModule implementation - org.apache.cassandra.auth.CassandraLoginModule -
-## which delegates to the IAuthenticator configured in cassandra.yaml. See the sample JAAS configuration
-## file cassandra-jaas.config
-#JVM_OPTS="$JVM_OPTS -Dcassandra.jmx.remote.login.config=CassandraLogin"
-#JVM_OPTS="$JVM_OPTS -Djava.security.auth.login.config=$CASSANDRA_CONF/cassandra-jaas.config"
-
-## Cassandra also ships with a helper for delegating JMX authz calls to the configured IAuthorizer,
-## uncomment this to use it. Requires one of the two authentication options to be enabled
-#JVM_OPTS="$JVM_OPTS -Dcassandra.jmx.authorizer=org.apache.cassandra.auth.jmx.AuthorizationProxy"
+# If this function call is commented out, then Cassandra will start with no system properties for JMX set whatsoever.
+# We will be expecting the settings in jmx_server_options and jmx_encryption_options respectively instead.
+# The argument specifies the default port over which Cassandra will be available for JMX connections.
+#
+# If you comment out configure_jmx method call, then JMX_PORT variable will not be set, which means
+# nodetool which sources this file will not see it either and port from cassandra.yaml will be parsed instead,
+# if not found there either, it defaults to 7199.
+#
+# For security reasons, you should not expose this port to the internet.  Firewall it if needed.
+configure_jmx 7199
 
 # To use mx4j, an HTML interface for JMX, add mx4j-tools.jar to the lib/
 # directory.

diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml
@@ -1721,14 +1721,59 @@ client_encryption_options:
 # JMX SSL.
 # Similar to `client/server_encryption_options`, you can specify PEM-based
 # key material or customize the SSL configuration using `ssl_context_factory` in `jmx_encryption_options`.
-#jmx_encryption_options:
-#  enabled: true
-#  cipher_suites: [TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256]
-#  accepted_protocols: [TLSv1.2,TLSv1.3,TLSv1.1]
-#  keystore: conf/cassandra_ssl.keystore
-#  keystore_password: cassandra
-#  truststore: conf/cassandra_ssl.truststore
-#  truststore_password: cassandra
+# If you uncomment this section, please be sure that you comment out configure_jmx function call in cassandra-env.sh
+# as it is errorneous to have JMX set by two ways, both in cassandra-env.sh and in this yaml.
+#jmx_server_options:
+  # enabled: true
+  # remote: false
+  # jmx_port: 7199
+  #
+  # Port used by the RMI registry when remote connections are enabled.
+  # To simplify firewall configs, this can be set to the same as the JMX server port (port). See CASSANDRA-7087.
+  # However, if ssl is enabled the same port cannot be used for both jmx and rmi so either
+  # pick another value for this property. Alternatively, comment out or set to 0 to use a random
+  # port (pre-CASSANDRA-7087 behaviour)
+  # rmi_port: 7199
+  #
+  # jmx ssl options - only apply when remote connections are enabled
+  #
+  # jmx_encryption_options:
+  #   enabled: true
+  #   cipher_suites: [TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256]
+  #   accepted_protocols: [TLSv1.2,TLSv1.3,TLSv1.1]
+  #   keystore: conf/cassandra_ssl.keystore
+  #   keystore_password: cassandra
+  #   truststore: conf/cassandra_ssl.truststore
+  #   truststore_password: cassandra
+  #
+  # jmx authentication and authorization options.
+  # authenticate: false
+  #
+  # Options for basic file based authentication & authorization
+  # password_file: /etc/cassandra/jmxremote.password
+  # access_file: /etc/cassandra/jmxremote.access
+  #
+  # Custom auth settings which can be used as alternatives to JMX's out of the box auth utilities.
+  # JAAS login modules can be used for authentication using this property.Cassandra ships with a
+  # LoginModule implementation - org.apache.cassandra.auth.CassandraLoginModule - which delegates
+  # to the IAuthenticator configured in cassandra.yaml.
+  #
+  # login_config_name refers to the Application Name in the JAAS configuration under which the
+  # desired LoginModule(s) are configured.
+  # The location of the JAAS config file may be set using the standard JVM mechanism, by setting
+  # the system property "java.security.auth.login.config". If this property is set, it's value
+  # will be used to locate the config file. For convenience, if the property is not already set
+  # at startup, a value can be supplied here via the login_config_file setting.
+  #
+  # The Application Name specified must be present in the JAAS config or an error will be thrown
+  # when authentication is attempted.
+  # See the sample JAAS configuration file conf/cassandra-jaas.config
+  # login_config_name: CassandraLogin
+  # login_config_file: conf/cassandra-jaas.config
+  #
+  # Cassandra also ships with a helper for delegating JMX authz calls to the configured IAuthorizer,
+  # uncomment this to use it. Requires one of the two authentication options to be enabled
+  # authorizer: org.apache.cassandra.auth.jmx.AuthorizationProxy
 
 # internode_compression controls whether traffic between nodes is
 # compressed.

diff --git a/conf/cassandra_latest.yaml b/conf/cassandra_latest.yaml
@@ -1685,14 +1685,59 @@ client_encryption_options:
 # JMX SSL.
 # Similar to `client/server_encryption_options`, you can specify PEM-based
 # key material or customize the SSL configuration using `ssl_context_factory` in `jmx_encryption_options`.
-#jmx_encryption_options:
-#  enabled: true
-#  cipher_suites: [TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256]
-#  accepted_protocols: [TLSv1.2,TLSv1.3,TLSv1.1]
-#  keystore: conf/cassandra_ssl.keystore
-#  keystore_password: cassandra
-#  truststore: conf/cassandra_ssl.truststore
-#  truststore_password: cassandra
+# If you uncomment this section, please be sure that you comment out configure_jmx function call in cassandra-env.sh
+# as it is errorneous to have JMX set by two ways, both in cassandra-env.sh and in this yaml.
+#jmx_server_options:
+  # enabled: true
+  # remote: false
+  # jmx_port: 7199
+  #
+  # Port used by the RMI registry when remote connections are enabled.
+  # To simplify firewall configs, this can be set to the same as the JMX server port (port). See CASSANDRA-7087.
+  # However, if ssl is enabled the same port cannot be used for both jmx and rmi so either
+  # pick another value for this property. Alternatively, comment out or set to 0 to use a random
+  # port (pre-CASSANDRA-7087 behaviour)
+  # rmi_port: 7199
+  #
+  # jmx ssl options - only apply when remote connections are enabled
+  #
+  # jmx_encryption_options:
+  #   enabled: true
+  #   cipher_suites: [TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256]
+  #   accepted_protocols: [TLSv1.2,TLSv1.3,TLSv1.1]
+  #   keystore: conf/cassandra_ssl.keystore
+  #   keystore_password: cassandra
+  #   truststore: conf/cassandra_ssl.truststore
+  #   truststore_password: cassandra
+  #
+  # jmx authentication and authorization options.
+  # authenticate: false
+  #
+  # Options for basic file based authentication & authorization
+  # password_file: /etc/cassandra/jmxremote.password
+  # access_file: /etc/cassandra/jmxremote.access
+  #
+  # Custom auth settings which can be used as alternatives to JMX's out of the box auth utilities.
+  # JAAS login modules can be used for authentication using this property.Cassandra ships with a
+  # LoginModule implementation - org.apache.cassandra.auth.CassandraLoginModule - which delegates
+  # to the IAuthenticator configured in cassandra.yaml.
+  #
+  # login_config_name refers to the Application Name in the JAAS configuration under which the
+  # desired LoginModule(s) are configured.
+  # The location of the JAAS config file may be set using the standard JVM mechanism, by setting
+  # the system property "java.security.auth.login.config". If this property is set, it's value
+  # will be used to locate the config file. For convenience, if the property is not already set
+  # at startup, a value can be supplied here via the login_config_file setting.
+  #
+  # The Application Name specified must be present in the JAAS config or an error will be thrown
+  # when authentication is attempted.
+  # See the sample JAAS configuration file conf/cassandra-jaas.config
+  # login_config_name: CassandraLogin
+  # login_config_file: conf/cassandra-jaas.config
+  #
+  # Cassandra also ships with a helper for delegating JMX authz calls to the configured IAuthorizer,
+  # uncomment this to use it. Requires one of the two authentication options to be enabled
+  # authorizer: org.apache.cassandra.auth.jmx.AuthorizationProxy
 
 # internode_compression controls whether traffic between nodes is
 # compressed.

diff --git a/doc/scripts/convert_yaml_to_adoc.py b/doc/scripts/convert_yaml_to_adoc.py
@@ -49,7 +49,7 @@
     'hints_compression',
     'server_encryption_options',
     'client_encryption_options',
-    'jmx_encryption_options',
+    'jmx_server_options',
     'transparent_data_encryption_options',
     'hinted_handoff_disabled_datacenters',
     'startup_checks',

diff --git a/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java b/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java
@@ -146,6 +146,8 @@ public enum CassandraRelevantProperties
     COM_SUN_MANAGEMENT_JMXREMOTE_PASSWORD_FILE("com.sun.management.jmxremote.password.file"),
     /** Port number to enable JMX RMI connections - com.sun.management.jmxremote.port */
     COM_SUN_MANAGEMENT_JMXREMOTE_PORT("com.sun.management.jmxremote.port"),
+    /** Enables SSL sockets for the RMI registry from which clients obtain the JMX connector stub */
+    COM_SUN_MANAGEMENT_JMXREMOTE_REGISTRY_SSL("com.sun.management.jmxremote.registry.ssl"),
     /**
      * The port number to which the RMI connector will be bound - com.sun.management.jmxremote.rmi.port.
      * An Integer object that represents the value of the second argument is returned
@@ -287,6 +289,10 @@ public enum CassandraRelevantProperties
     IO_NETTY_EVENTLOOP_THREADS("io.netty.eventLoopThreads"),
     IO_NETTY_TRANSPORT_ESTIMATE_SIZE_ON_SUBMIT("io.netty.transport.estimateSizeOnSubmit"),
     IO_NETTY_TRANSPORT_NONATIVE("io.netty.transport.noNative"),
+    JAVAX_NET_SSL_KEYSTORE("javax.net.ssl.keyStore"),
+    JAVAX_NET_SSL_KEYSTOREPASSWORD("javax.net.ssl.keyStorePassword"),
+    JAVAX_NET_SSL_TRUSTSTORE("javax.net.ssl.trustStore"),
+    JAVAX_NET_SSL_TRUSTSTOREPASSWORD("javax.net.ssl.trustStorePassword"),
     JAVAX_RMI_SSL_CLIENT_ENABLED_CIPHER_SUITES("javax.rmi.ssl.client.enabledCipherSuites"),
     JAVAX_RMI_SSL_CLIENT_ENABLED_PROTOCOLS("javax.rmi.ssl.client.enabledProtocols"),
     /** Java class path. */

diff --git a/src/java/org/apache/cassandra/config/Config.java b/src/java/org/apache/cassandra/config/Config.java
@@ -434,7 +434,8 @@ public static class SSTableConfig
 
     public EncryptionOptions.ServerEncryptionOptions server_encryption_options = new EncryptionOptions.ServerEncryptionOptions();
     public EncryptionOptions client_encryption_options = new EncryptionOptions();
-    public EncryptionOptions jmx_encryption_options = new EncryptionOptions();
+
+    public JMXServerOptions jmx_server_options;
 
     public InternodeCompression internode_compression = InternodeCompression.none;
 
@@ -1322,7 +1323,8 @@ public enum BatchlogEndpointStrategy
     private static final Set<String> SENSITIVE_KEYS = new HashSet<String>() {{
         add("client_encryption_options");
         add("server_encryption_options");
-        add("jmx_encryption_options");
+        // jmx_server_options output (JMXServerOptions.toString()) doesn't
+        // include sensitive encryption config so no need to blocklist here
     }};
 
     public static void log(Config config)