tests: add test for prometheus exporter

apps/CMakeLists.txt

@@ -51,7 +51,6 @@ endmacro ()
 add_subdirectory (httpd)
 add_subdirectory (io_tester)
 add_subdirectory (rpc_tester)
-add_subdirectory(metrics_tester)
 add_subdirectory (iotune)
 add_subdirectory (memcached)
 add_subdirectory (seawreck)

tests/unit/CMakeLists.txt

@@ -757,3 +757,25 @@ add_test (
 set_tests_properties (Seastar.unit.json2code
   PROPERTIES
     TIMEOUT ${Seastar_TEST_TIMEOUT})
+
+add_executable (metrics_tester
+  metrics_tester.cc)
+target_link_libraries (metrics_tester
+  PRIVATE
+    seastar_private
+    yaml-cpp::yaml-cpp)
+
+add_dependencies (unit_tests metrics_tester)
+add_custom_target (test_unit_prometheus_run
+  COMMAND ${CMAKE_COMMAND} -E env ${Seastar_TEST_ENVIRONMENT}
+    ${CMAKE_CURRENT_SOURCE_DIR}/prometheus_test.py
+    --exporter $<TARGET_FILE:metrics_tester>
+    --config ${CMAKE_CURRENT_SOURCE_DIR}/conf-example.yaml
+  USES_TERMINAL)
+add_dependencies (test_unit_prometheus_run metrics_tester)
+add_test (
+  NAME Seastar.unit.prometheus
+  COMMAND ${CMAKE_COMMAND} --build ${Seastar_BINARY_DIR} --target test_unit_prometheus_run)
+set_tests_properties (Seastar.unit.prometheus
+  PROPERTIES
+    TIMEOUT ${Seastar_TEST_TIMEOUT})

tests/unit/conf-example.yaml

@@ -0,0 +1,30 @@
+metrics:
+- name: hist1
+  type: histogram
+  values: [1000,2000,3000]
+  labels:
+    private: "1"
+- name: gag1
+  type: gauge
+  values: [5]
+  labels:
+    private: "1"
+- name: count1
+  type: counter
+  labels:
+    private: "1"
+  values: [7]
+- name: counter_1
+  type: counter
+  labels:
+    private: "2"
+  values: [1]
+- name: counter_1
+  type: counter
+  labels:
+    private: "3"
+  values: [2]
+metric_family_config:
+- name: test_group_counter_1
+  aggregate_labels:
+    - private

apps/metrics_tester/metrics_tester.cc → tests/unit/metrics_tester.cc

@@ -25,7 +25,10 @@
 #include <seastar/core/metrics.hh>
 #include <seastar/core/relabel_config.hh>
 #include <seastar/core/internal/estimated_histogram.hh>
-#include "../lib/stop_signal.hh"
+#include <seastar/util/defer.hh>
+#include "../../apps/lib/stop_signal.hh"
+#include <map>
+#include <vector>
 #include <yaml-cpp/yaml.h>
 using namespace seastar;
 using namespace std::chrono_literals;
@@ -36,11 +39,14 @@ struct metric_def {
     sstring name;
     sstring type;
     std::vector<double> values;
+    std::vector<sm::label_instance> labels;
 };
 
 struct config {
     std::vector<metric_def> metrics;
+    std::vector<sm::metric_family_config> metric_family_config;
 };
+
 namespace YAML {
 template<>
 struct convert<metric_def> {
@@ -54,18 +60,45 @@ struct convert<metric_def> {
         if (node["values"]) {
             cfg.values = node["values"].as<std::vector<double>>();
         }
+        if (node["labels"]) {
+            const auto labels = node["labels"].as<std::map<std::string, std::string>>();
+            for (auto& [key, value]: labels) {
+                cfg.labels.emplace_back(key, value);
+            }
+        }
+        return true;
+    }
+};
+
+template<>
+struct convert<sm::metric_family_config> {
+    static bool decode(const Node& node, sm::metric_family_config& cfg) {
+        if (node["name"]) {
+            cfg.name = node["name"].as<std::string>();
+        }
+        if (node["regex_name"]) {
+            cfg.regex_name = node["regex_name"].as<std::string>();
+        }
+        if (node["aggregate_labels"]) {
+            cfg.aggregate_labels = node["aggregate_labels"].as<std::vector<std::string>>();
+        }
         return true;
     }
 };
+
 template<>
 struct convert<config> {
     static bool decode(const Node& node, config& cfg) {
         if (node["metrics"]) {
             cfg.metrics = node["metrics"].as<std::vector<metric_def>>();
         }
+        if (node["metric_family_config"]) {
+            cfg.metric_family_config = node["metric_family_config"].as<std::vector<sm::metric_family_config>>();
+        }
         return true;
     }
 };
+
 }
 std::function<sm::internal::time_estimated_histogram()> make_histogram_fun(const metric_def& c) {
     sm::internal::time_estimated_histogram histogram;
@@ -75,21 +108,21 @@ std::function<sm::internal::time_estimated_histogram()> make_histogram_fun(const
     return [histogram]() {return histogram;};
 }
 
-sm::impl::metric_definition_impl make_metrics_definition(const metric_def& jc, sm::label_instance private_label) {
+sm::impl::metric_definition_impl make_metrics_definition(const metric_def& jc) {
     if (jc.type == "histogram") {
         sm::internal::time_estimated_histogram histogram;
         for (const auto& v : jc.values) {
             histogram.add_micro(v);
         }
         return sm::make_histogram(jc.name, [histogram]() {return histogram.to_metrics_histogram();},
-                sm::description(jc.name),{private_label} );
+                sm::description(jc.name), jc.labels );
     }
     if (jc.type == "gauge") {
         return sm::make_gauge(jc.name, [val=jc.values[0]] { return val; },
-                sm::description(jc.name),{private_label} );
+                sm::description(jc.name), jc.labels );
     }
     return sm::make_counter(jc.name, [val=jc.values[0]] { return val; },
-            sm::description(jc.name),{private_label} );
+            sm::description(jc.name), jc.labels );
 }
 
 int main(int ac, char** av) {
@@ -110,7 +143,6 @@ int main(int ac, char** av) {
             seastar_apps_lib::stop_signal stop_signal;
             auto& opts = app.configuration();
             auto& listen = opts["listen"].as<sstring>();
-            auto private_label = sm::label_instance("private", "1");
             auto& port = opts["port"].as<uint16_t>();
             auto& conf = opts["conf"].as<sstring>();
 
@@ -119,7 +151,7 @@ int main(int ac, char** av) {
 
             for (auto&& jc : cfg.metrics) {
                 _metrics.add_group("test_group", {
-                        make_metrics_definition(jc, private_label)
+                        make_metrics_definition(jc)
                 });
             }
             smp::invoke_on_all([] {
@@ -129,24 +161,33 @@ int main(int ac, char** av) {
                     rl[0].action = metrics::relabel_config::relabel_action::drop;
 
                     rl[1].source_labels = {"private"};
-                    rl[1].expr = "1";
+                    rl[1].expr = ".*";
                     rl[1].action = metrics::relabel_config::relabel_action::keep;
                     return metrics::set_relabel_configs(rl).then([](metrics::metric_relabeling_result) {
                         return;
                     });
             }).get();
 
-            if (port) {
-                prometheus_server.start("prometheus").get();
-
-                prometheus::config pctx;
-                pctx.allow_protobuf = true;
-                prometheus::start(prometheus_server, pctx).get();
-                prometheus_server.listen(socket_address{listen, port}).handle_exception([] (auto ep) {
-                    return make_exception_future<>(ep);
-                }).get();
-                stop_signal.wait().get();
+            if (!cfg.metric_family_config.empty()) {
+                sm::set_metric_family_configs(cfg.metric_family_config);
             }
+
+            prometheus_server.start("prometheus").get();
+            auto stop_server = defer([&] () noexcept {
+                prometheus_server.stop().get();
+            });
+
+            prometheus::config pctx;
+            pctx.allow_protobuf = true;
+            prometheus::start(prometheus_server, pctx).get();
+            prometheus_server.listen(socket_address{listen, port}).handle_exception([] (auto ep) {
+                return make_exception_future<>(ep);
+            }).get();
+
+            fmt::print("{}\n", port);
+            fflush(stdout);
+
+            stop_signal.wait().get();
         });
     });
 }