Adding new metrics for SAI in nodetool tablestats command.

src/java/org/apache/cassandra/index/sai/metrics/IndexGroupMetrics.java

@@ -25,6 +25,7 @@
 
 public class IndexGroupMetrics extends AbstractMetrics
 {
+    public static final String INDEX_GROUP_METRICS_TYPE = "IndexGroupMetrics";
     public IndexGroupMetrics(TableMetadata table, StorageAttachedIndexGroup group)
     {
         super(table.keyspace, table.name, "IndexGroupMetrics");

src/java/org/apache/cassandra/index/sai/metrics/TableQueryMetrics.java

@@ -71,6 +71,8 @@ public void release()
 
     public class PerQueryMetrics extends AbstractMetrics
     {
+        public static final String PER_QUERY_METRICS_TYPE = "PerQuery";
+
         private final Timer queryLatency;
 
         /**

src/java/org/apache/cassandra/tools/NodeProbe.java

@@ -90,6 +90,9 @@
 import org.apache.cassandra.gms.GossiperMBean;
 import org.apache.cassandra.hints.HintsService;
 import org.apache.cassandra.hints.HintsServiceMBean;
+import org.apache.cassandra.index.sai.metrics.IndexGroupMetrics;
+import org.apache.cassandra.index.sai.metrics.TableQueryMetrics;
+import org.apache.cassandra.index.sai.metrics.TableStateMetrics;
 import org.apache.cassandra.locator.DynamicEndpointSnitchMBean;
 import org.apache.cassandra.locator.EndpointSnitchInfoMBean;
 import org.apache.cassandra.metrics.CIDRAuthorizerMetrics;
@@ -1892,7 +1895,73 @@ public Object getThreadPoolMetric(String pathName, String poolName, String metri
       }
     }
 
-    /**
+    public Object getSaiMetric(String ks, String cf, String metricName)
+    {
+        try
+        {
+            String scope = getSaiMetricScope(metricName);
+            String objectNameStr = String.format("org.apache.cassandra.metrics:type=StorageAttachedIndex,keyspace=%s,table=%s,scope=%s,name=%s",ks, cf, scope, metricName);
+            ObjectName oName = new ObjectName(objectNameStr);
+
+            Set<ObjectName> matchingMBeans = mbeanServerConn.queryNames(oName, null);
+            if (matchingMBeans.isEmpty())
+                return null;
+
+            return getSaiMetricValue(metricName, oName);
+        } catch (MalformedObjectNameException e)
+        {
+            throw new RuntimeException("Invalid ObjectName format: " + e.getMessage(), e);
+        } catch (IOException e)
+        {
+            throw new RuntimeException("Error accessing MBean server: " + e.getMessage(), e);
+        }
+    }
+
+    private Object getSaiMetricValue(String metricName, ObjectName oName) throws IOException
+    {
+        switch (metricName)
+        {
+            case "QueryLatency":
+                return JMX.newMBeanProxy(mbeanServerConn, oName, CassandraMetricsRegistry.JmxTimerMBean.class);
+            case "PostFilteringReadLatency":
+            case "SSTableIndexesHit":
+            case "IndexSegmentsHit":
+            case "RowsFiltered":
+                return JMX.newMBeanProxy(mbeanServerConn, oName, CassandraMetricsRegistry.JmxHistogramMBean.class);
+            case "DiskUsedBytes":
+            case "TotalIndexCount":
+            case "TotalQueryableIndexCount":
+                return JMX.newMBeanProxy(mbeanServerConn, oName, CassandraMetricsRegistry.JmxGaugeMBean.class).getValue();
+            case "TotalQueryTimeouts":
+                return JMX.newMBeanProxy(mbeanServerConn, oName, CassandraMetricsRegistry.JmxCounterMBean.class).getCount();
+            default:
+                throw new IllegalArgumentException("Unknown metric name: " + metricName);
+        }
+    }
+
+    private String getSaiMetricScope(String metricName)
+    {
+        switch (metricName)
+        {
+            case "QueryLatency":
+            case "SSTableIndexesHit":
+            case "IndexSegmentsHit":
+            case "RowsFiltered":
+                return TableQueryMetrics.PerQueryMetrics.PER_QUERY_METRICS_TYPE;
+            case "PostFilteringReadLatency":
+            case "TotalQueryTimeouts":
+                return TableQueryMetrics.TABLE_QUERY_METRIC_TYPE;
+            case "DiskUsedBytes":
+                return IndexGroupMetrics.INDEX_GROUP_METRICS_TYPE;
+            case "TotalIndexCount":
+            case "TotalQueryableIndexCount":
+                return TableStateMetrics.TABLE_STATE_METRIC_TYPE;
+            default:
+                throw new IllegalArgumentException("Unknown metric name: " + metricName);
+        }
+    }
+
+        /**
      * Retrieve threadpool paths and names for threadpools with metrics.
      * @return Multimap from path (internal, request, etc.) to name
      */

src/java/org/apache/cassandra/tools/nodetool/TableStats.java

@@ -60,8 +60,10 @@ public class TableStats extends NodeToolCmd
                         + "memtable_off_heap_memory_used, memtable_switch_count, number_of_partitions_estimate, "
                         + "off_heap_memory_used_total, pending_flushes, percent_repaired, read_latency, reads, "
                         + "space_used_by_snapshots_total, space_used_live, space_used_total, "
-                        + "sstable_compression_ratio, sstable_count, table_name, write_latency, writes, " +
-                          "max_sstable_size, local_read_write_ratio, twcs_max_duration)")
+                        + "sstable_compression_ratio, sstable_count, table_name, write_latency, writes, "
+                        + "max_sstable_size, local_read_write_ratio, twcs_max_duration, sai_local_query_latency, "
+                        + "sai_post_filtering_read_latency, sai_disk_used_bytes, sai_sstable_indexes_hit, sai_index_segments_hit "
+                        + "sai_rows_filtered, sai_total_query_timeouts, sai_queryable_total_indexes)")
     private String sortKey = "";
 
     @Option(title = "top",

src/java/org/apache/cassandra/tools/nodetool/stats/StatsTable.java

@@ -82,4 +82,13 @@ public class StatsTable
     public double localReadWriteRatio;
     public Long twcsDurationInMillis;
     public String twcs;
+
+    public double saiQueryLatencyMs;
+    public double saiPostFilteringReadLatencyms;
+    public String saiDiskUsedBytes;
+    public double saiSSTableIndexesHit;
+    public double saiIndexSegmentsHit;
+    public double saiRowsFiltered;
+    public long saiTotalQueryTimeouts;
+    public String saiTotalQueryableIndexRatio;
 }

src/java/org/apache/cassandra/tools/nodetool/stats/StatsTableComparator.java

@@ -66,7 +66,10 @@ public class StatsTableComparator implements Comparator<StatsTable>
                                                        "space_used_by_snapshots_total", "space_used_live",
                                                        "space_used_total", "sstable_compression_ratio", "sstable_count",
                                                        "table_name", "write_latency", "writes", "max_sstable_size",
-                                                       "local_read_write_ratio", "twcs_max_duration"};
+                                                       "local_read_write_ratio", "twcs_max_duration", "sai_local_query_latency",
+                                                       "sai_post_filtering_read_latency","sai_disk_used_bytes","sai_sstable_indexes_hit",
+                                                       "sai_index_segments_hit","sai_rows_filtered","sai_total_query_timeouts",
+                                                       "sai_queryable_total_indexes"};
 
     public StatsTableComparator(String sortKey, boolean humanReadable)
     {
@@ -338,6 +341,40 @@ else if (sortKey.equals("table_name"))
         {
             return sign * stx.tableName.compareTo(sty.tableName);
         }
+        else if(sortKey.equals("sai_local_query_latency"))
+        {
+            result = compareDoubles(stx.saiQueryLatencyMs, sty.saiQueryLatencyMs);
+        }
+        else if(sortKey.equals("sai_post_filtering_read_latency"))
+        {
+            result = compareDoubles(stx.saiPostFilteringReadLatencyms, sty.saiPostFilteringReadLatencyms);
+        }
+        else if(sortKey.equals("sai_disk_used_bytes"))
+        {
+            result = compareFileSizes(stx.saiDiskUsedBytes,
+                                      sty.saiDiskUsedBytes);
+        }
+        else if(sortKey.equals("sai_sstable_indexes_hit"))
+        {
+            return compareDoubles(stx.saiSSTableIndexesHit, sty.saiSSTableIndexesHit);
+        }
+        else if(sortKey.equals("sai_index_segments_hit"))
+        {
+            return compareDoubles(stx.saiIndexSegmentsHit, sty.saiIndexSegmentsHit);
+        }
+        else if(sortKey.equals("sai_rows_filtered"))
+        {
+            return compareDoubles(stx.saiRowsFiltered, sty.saiRowsFiltered);
+        }
+        else if(sortKey.equals("sai_total_query_timeouts"))
+        {
+            result = sign * Long.valueOf(stx.saiTotalQueryTimeouts)
+                                .compareTo(Long.valueOf(sty.saiTotalQueryTimeouts));
+        }
+        else if(sortKey.equals("sai_queryable_total_indexes"))
+        {
+            return sign * stx.saiTotalQueryableIndexRatio.compareTo(sty.saiTotalQueryableIndexRatio);
+        }
         else
         {
             throw new IllegalStateException(String.format("Unsupported sort key: %s", sortKey));

src/java/org/apache/cassandra/tools/nodetool/stats/TableStatsHolder.java

@@ -175,6 +175,16 @@ private Map<String, Object> convertStatsTableToMap(StatsTable table)
         mpTable.put("top_tombstone_partitions", table.topTombstonePartitions);
         if (locationCheck)
             mpTable.put("sstables_in_correct_location", table.isInCorrectLocation);
+
+        mpTable.put("sai_local_query_latency",String.format("%01.3f", table.saiQueryLatencyMs));
+        mpTable.put("sai_post_filtering_read_latency",String.format("%01.3f", table.saiPostFilteringReadLatencyms));
+        mpTable.put("sai_disk_used_bytes",table.saiDiskUsedBytes);
+        mpTable.put("sai_sstable_indexes_hit",table.saiSSTableIndexesHit);
+        mpTable.put("sai_index_segments_hit",table.saiIndexSegmentsHit);
+        mpTable.put("sai_rows_filtered",table.saiRowsFiltered);
+        mpTable.put("sai_total_query_timeouts",table.saiTotalQueryTimeouts);
+        mpTable.put("sai_queryable_total_indexes", table.saiTotalQueryableIndexRatio);
+
         return mpTable;
     }
 
@@ -392,13 +402,54 @@ private void initializeKeyspaces(NodeProbe probe, boolean ignore, List<String> t
                 statsTable.topTombstonePartitions = table.getTopTombstonePartitions();
                 if (table.getTopTombstonePartitionsLastUpdate() != null)
                     statsTable.topTombstonePartitionsLastUpdate = millisToDateString(table.getTopTombstonePartitionsLastUpdate());
+                Object queryLatencyMetric = probe.getSaiMetric(keyspaceName, tableName, "QueryLatency"); // Add sai here as well.
+                double QueryLatency = getMetricMean(queryLatencyMetric);
+                statsTable.saiQueryLatencyMs = QueryLatency > 0 ? QueryLatency : Double.NaN;
+
+                Object PostFilteringReadLatency = probe.getSaiMetric(keyspaceName, tableName, "PostFilteringReadLatency");
+                double postfilteringreadlatency = getMetricMean(PostFilteringReadLatency);
+                statsTable.saiPostFilteringReadLatencyms = postfilteringreadlatency > 0 ? postfilteringreadlatency : Double.NaN;
+
+                Object diskUsedBytes = probe.getSaiMetric(keyspaceName, tableName, "DiskUsedBytes");
+                long saidiskusedbytes = (diskUsedBytes != null) ? (long) diskUsedBytes : 0L;
+                statsTable.saiDiskUsedBytes = FileUtils.stringifyFileSize(saidiskusedbytes, humanReadable);
+
+                Object SSTableIndexesHit = probe.getSaiMetric(keyspaceName, tableName, "SSTableIndexesHit");
+                statsTable.saiSSTableIndexesHit = getMetricMean(SSTableIndexesHit);
+
+                Object IndexSegmentsHit = probe.getSaiMetric(keyspaceName, tableName, "IndexSegmentsHit");
+                statsTable.saiIndexSegmentsHit = getMetricMean(IndexSegmentsHit);
+
+                Object RowsFiltered = probe.getSaiMetric(keyspaceName, tableName, "RowsFiltered");
+                statsTable.saiRowsFiltered = getMetricMean(RowsFiltered);
+
+                Object totalQueryTimeouts = probe.getSaiMetric(keyspaceName, tableName, "TotalQueryTimeouts");
+                statsTable.saiTotalQueryTimeouts = (totalQueryTimeouts != null)  ? (Long) totalQueryTimeouts : 0L;
+
+                Object totalIndexCount = probe.getSaiMetric(keyspaceName, tableName, "TotalIndexCount");
+                int saiTotalIndexCount = (totalIndexCount != null) ? (int) totalIndexCount : 0;
+
+                Object totalQueryableIndexCount = probe.getSaiMetric(keyspaceName, tableName, "TotalQueryableIndexCount");
+                int saiTotalQueryableIndexCount = (totalQueryableIndexCount != null) ? (int) totalQueryableIndexCount : 0;
+
+                statsTable.saiTotalQueryableIndexRatio = String.format("%d/%d", saiTotalIndexCount, saiTotalQueryableIndexCount);
 
                 statsKeyspace.tables.add(statsTable);
             }
             keyspaces.add(statsKeyspace);
         }
     }
 
+    private double getMetricMean(Object metricObject) {
+        if (metricObject instanceof CassandraMetricsRegistry.JmxTimerMBean) {
+            return ((CassandraMetricsRegistry.JmxTimerMBean) metricObject).getMean() / 1000;
+        }
+        if (metricObject instanceof CassandraMetricsRegistry.JmxHistogramMBean) {
+            return Math.round(((CassandraMetricsRegistry.JmxHistogramMBean) metricObject).getMean() * 100.0) / 100.0;
+        }
+        return Double.NaN;
+    }
+
     private void maybeAddTWCSWindowWithMaxDuration(StatsTable statsTable, NodeProbe probe, String keyspaceName, String tableName)
     {
         Map<String, String> compactionParameters = probe.getCfsProxy(statsTable.keyspaceName, statsTable.tableName)

src/java/org/apache/cassandra/tools/nodetool/stats/TableStatsPrinter.java

@@ -23,6 +23,7 @@
 import java.util.Map;
 
 import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.schema.SchemaConstants;
 import org.apache.cassandra.utils.FBUtilities;
 
 public class TableStatsPrinter<T extends StatsHolder>
@@ -168,9 +169,27 @@ protected void printStatsTable(StatsTable table, String tableDisplayName, String
                 for (Map.Entry<String, Long> tombstonecnt : table.topTombstonePartitions.entrySet())
                     out.printf(indent + "  %-" + maxWidth + "s %s%n", tombstonecnt.getKey(), tombstonecnt.getValue());
             }
+
+            if (!isSystemKeyspaces(table.keyspaceName))
+            {
+                out.println(indent + "SAI local query latency (mean): " + FBUtilities.prettyPrintLatency(table.saiQueryLatencyMs));
+                out.println(indent + "SAI post-filtering latency (mean): " + FBUtilities.prettyPrintLatency(table.saiPostFilteringReadLatencyms));
+                out.println(indent + "SAI space used (bytes): " + table.saiDiskUsedBytes);
+                out.println(indent + "SAI sstable indexes hit per query (mean): " + table.saiSSTableIndexesHit);
+                out.println(indent + "SAI index segments hit per query (mean): " + table.saiIndexSegmentsHit);
+                out.println(indent + "SAI rows filtered per query (mean): " + table.saiRowsFiltered);
+                out.println(indent + "SAI local query timeouts: " + table.saiTotalQueryTimeouts);
+                out.println(indent + "SAI queryable/total indexes: " + table.saiTotalQueryableIndexRatio);
+            }
+
             out.println("");
         }
 
+        private boolean isSystemKeyspaces(String keyspaceName) {
+            return SchemaConstants.isSystemKeyspace(keyspaceName);
+
+        }
+
         private String formatDataSize(long bytes, boolean humanReadable)
         {
             return humanReadable ? FileUtils.stringifyFileSize(bytes) : Long.toString(bytes);