diff util/statistics.lua @ 11523:5f15ab7c6ae5

Statistics: Rewrite statistics backends to use OpenMetrics The metric subsystem of Prosody has had some shortcomings from the perspective of the current state-of-the-art in metric observability. The OpenMetrics standard [0] is a formalization of the data model (and serialization format) of the well-known and widely-used Prometheus [1] software stack. The previous stats subsystem of Prosody did not map well to that format (see e.g. [2] and [3]); the key reason is that it was trying to do too much math on its own ([2]) while lacking first-class support for "families" of metrics ([3]) and structured metric metadata (despite the `extra` argument to metrics, there was no standard way of representing common things like "tags" or "labels"). Even though OpenMetrics has grown from the Prometheus world of monitoring, it maps well to other popular monitoring stacks such as: - InfluxDB (labels can be mapped to tags and fields as necessary) - Carbon/Graphite (labels can be attached to the metric name with dot-separation) - StatsD (see graphite when assuming that graphite is used as backend, which is the default) The util.statsd module has been ported to use the OpenMetrics model as a proof of concept. An implementation which exposes the util.statistics backend data as Prometheus metrics is ready for publishing in prosody-modules (most likely as mod_openmetrics_prometheus to avoid breaking existing 0.11 deployments). At the same time, the previous measure()-based API had one major advantage: It is really simple and easy to use without requiring lots of knowledge about OpenMetrics or similar concepts. For that reason as well as compatibility with existing code, it is preserved and may even be extended in the future. However, code relying on the `stats-updated` event as well as `get_stats` from `statsmanager` will break because the data model has changed completely; in case of `stats-updated`, the code will simply not run (as the event was renamed in order to avoid conflicts); the `get_stats` function has been removed completely (so it will cause a traceback when it is attempted to be used). Note that the measure_*_event methods have been removed from the module API. I was unable to find any uses or documentation and thus deemed they should not be ported. Re-implementation is possible when necessary. [0]: https://openmetrics.io/ [1]: https://prometheus.io/ [2]: #959 [3]: #960
author Jonas Schäfer <jonas@wielicki.name>
date Sun, 18 Apr 2021 11:47:41 +0200
parents d75d805c852f
children 7ba686696250
line wrap: on
line diff
--- a/util/statistics.lua	Sun Apr 25 17:32:27 2021 +0200
+++ b/util/statistics.lua	Sun Apr 18 11:47:41 2021 +0200
@@ -1,171 +1,191 @@
-local t_sort = table.sort
-local m_floor = math.floor;
 local time = require "util.time".now;
+local new_metric_registry = require "util.openmetrics".new_metric_registry;
+local render_histogram_le = require "util.openmetrics".render_histogram_le;
 
-local function nop_function() end
+-- BEGIN of Metric implementations
+
+-- Gauges
+local gauge_metric_mt = {}
+gauge_metric_mt.__index = gauge_metric_mt
+
+local function new_gauge_metric()
+	local metric = { value = 0 }
+	setmetatable(metric, gauge_metric_mt)
+	return metric
+end
+
+function gauge_metric_mt:set(value)
+	self.value = value
+end
+
+function gauge_metric_mt:add(delta)
+	self.value = self.value + delta
+end
+
+function gauge_metric_mt:reset()
+	self.value = 0
+end
 
-local function percentile(arr, length, pc)
-	local n = pc/100 * (length + 1);
-	local k, d = m_floor(n), n%1;
-	if k == 0 then
-		return arr[1] or 0;
-	elseif k >= length then
-		return arr[length];
-	end
-	return arr[k] + d*(arr[k+1] - arr[k]);
+function gauge_metric_mt:iter_samples()
+	local done = false
+	return function(_s)
+		if done then
+			return nil, true
+		end
+		done = true
+		return "", nil, _s.value
+	end, self
+end
+
+-- Counters
+local counter_metric_mt = {}
+counter_metric_mt.__index = counter_metric_mt
+
+local function new_counter_metric()
+	local metric = {
+		_created = time(),
+		value = 0,
+	}
+	setmetatable(metric, counter_metric_mt)
+	return metric
+end
+
+function counter_metric_mt:set(value)
+	self.value = value
+end
+
+function counter_metric_mt:add(value)
+	self.value = (self.value or 0) + value
+end
+
+function counter_metric_mt:iter_samples()
+	local step = 0
+	return function(_s)
+		step = step + 1
+		if step == 1 then
+			return "_created", nil, _s._created
+		elseif step == 2 then
+			return "_total", nil, _s.value
+		else
+			return nil, nil, true
+		end
+	end, self
+end
+
+function counter_metric_mt:reset()
+	self.value = 0
 end
 
-local function new_registry(config)
-	config = config or {};
-	local duration_sample_interval = config.duration_sample_interval or 5;
-	local duration_max_samples = config.duration_max_stored_samples or 5000;
+-- Histograms
+local histogram_metric_mt = {}
+histogram_metric_mt.__index = histogram_metric_mt
 
-	local function get_distribution_stats(events, n_actual_events, since, new_time, units)
-		local n_stored_events = #events;
-		t_sort(events);
-		local sum = 0;
-		for i = 1, n_stored_events do
-			sum = sum + events[i];
-		end
+local function new_histogram_metric(buckets)
+	local metric = {
+		_created = time(),
+		_sum = 0,
+		_count = 0,
+	}
+	-- the order of buckets matters unfortunately, so we cannot directly use
+	-- the thresholds as table keys
+	for i, threshold in ipairs(buckets) do
+		metric[i] = {
+			threshold = threshold,
+			threshold_s = render_histogram_le(threshold),
+			count = 0
+		}
+	end
+	setmetatable(metric, histogram_metric_mt)
+	return metric
+end
 
-		return {
-			samples = events;
-			sample_count = n_stored_events;
-			count = n_actual_events,
-			rate = n_actual_events/(new_time-since);
-			average = n_stored_events > 0 and sum/n_stored_events or 0,
-			min = events[1] or 0,
-			max = events[n_stored_events] or 0,
-			units = units,
-		};
+function histogram_metric_mt:sample(value)
+	-- According to the I-D, values must be part of all buckets
+	for i, bucket in pairs(self) do
+		if "number" == type(i) and bucket.threshold > value then
+			bucket.count = bucket.count + 1
+		end
 	end
+	self._sum = self._sum + value
+	self._count = self._count + 1
+end
 
+function histogram_metric_mt:iter_samples()
+	local key = nil
+	return function (_s)
+		local data
+		key, data = next(_s, key)
+		if key == "_created" or key == "_sum" or key == "_count" then
+			return key, nil, data
+		elseif key ~= nil then
+			return "_bucket", {["le"] = data.threshold_s}, data.count
+		else
+			return nil, nil, nil
+		end
+	end, self
+end
 
-	local registry = {};
-	local methods;
-	methods = {
-		amount = function (name, conf)
-			local v = conf and conf.initial or 0;
-			registry[name..":amount"] = function ()
-				return "amount", v, conf;
-			end
-			return function (new_v) v = new_v; end
-		end;
-		counter = function (name, conf)
-			local v = conf and conf.initial or 0;
-			registry[name..":amount"] = function ()
-				return "amount", v, conf;
-			end
-			return function (delta)
-				v = v + delta;
-			end;
-		end;
-		rate = function (name, conf)
-			local since, n, total = time(), 0, 0;
-			registry[name..":rate"] = function ()
-				total = total + n;
-				local t = time();
-				local stats = {
-					rate = n/(t-since);
-					count = n;
-					total = total;
-					units = conf and conf.units;
-					type = conf and conf.type;
-				};
-				since, n = t, 0;
-				return "rate", stats.rate, stats;
-			end;
-			return function ()
-				n = n + 1;
-			end;
-		end;
-		distribution = function (name, conf)
-			local units = conf and conf.units;
-			local type = conf and conf.type or "distribution";
-			local events, last_event = {}, 0;
-			local n_actual_events = 0;
-			local since = time();
+function histogram_metric_mt:reset()
+	self._created = time()
+	self._count = 0
+	self._sum = 0
+	for i, bucket in pairs(self) do
+		if "number" == type(i) then
+			bucket.count = 0
+		end
+	end
+end
+
+-- Summary
+local summary_metric_mt = {}
+summary_metric_mt.__index = summary_metric_mt
+
+local function new_summary_metric()
+	-- quantiles are not supported yet
+	local metric = {
+		_created = time(),
+		_sum = 0,
+		_count = 0,
+	}
+	setmetatable(metric, summary_metric_mt)
+	return metric
+end
 
-			registry[name..":"..type] = function ()
-				local new_time = time();
-				local stats = get_distribution_stats(events, n_actual_events, since, new_time, units);
-				events, last_event = {}, 0;
-				n_actual_events = 0;
-				since = new_time;
-				return type, stats.average, stats;
-			end;
+function summary_metric_mt:sample(value)
+	self._sum = self._sum + value
+	self._count = self._count + 1
+end
 
-			return function (value)
-				n_actual_events = n_actual_events + 1;
-				if n_actual_events%duration_sample_interval == 1 then
-					last_event = (last_event%duration_max_samples) + 1;
-					events[last_event] = value;
-				end
-			end;
-		end;
-		sizes = function (name, conf)
-			conf = conf or { units = "bytes", type = "size" }
-			return methods.distribution(name, conf);
-		end;
-		times = function (name, conf)
-			local units = conf and conf.units or "seconds";
-			local events, last_event = {}, 0;
-			local n_actual_events = 0;
-			local since = time();
+function summary_metric_mt:iter_samples()
+	local key = nil
+	return function (_s)
+		local data
+		key, data = next(_s, key)
+		return key, nil, data
+	end, self
+end
 
-			registry[name..":duration"] = function ()
-				local new_time = time();
-				local stats = get_distribution_stats(events, n_actual_events, since, new_time, units);
-				events, last_event = {}, 0;
-				n_actual_events = 0;
-				since = new_time;
-				return "duration", stats.average, stats;
-			end;
-
-			return function ()
-				n_actual_events = n_actual_events + 1;
-				if n_actual_events%duration_sample_interval ~= 1 then
-					return nop_function;
-				end
+function summary_metric_mt:reset()
+	self._created = time()
+	self._count = 0
+	self._sum = 0
+end
 
-				local start_time = time();
-				return function ()
-					local end_time = time();
-					local duration = end_time - start_time;
-					last_event = (last_event%duration_max_samples) + 1;
-					events[last_event] = duration;
-				end
-			end;
-		end;
+local pull_backend = {
+	gauge = new_gauge_metric,
+	counter = new_counter_metric,
+	histogram = new_histogram_metric,
+	summary = new_summary_metric,
+}
 
-		get_stats = function ()
-			return registry;
-		end;
-	};
-	return methods;
+-- END of Metric implementations
+
+local function new()
+	return {
+		metric_registry = new_metric_registry(pull_backend),
+	}
 end
 
 return {
-	new = new_registry;
-	get_histogram = function (duration, n_buckets)
-		n_buckets = n_buckets or 100;
-		local events, n_events = duration.samples, duration.sample_count;
-		if not (events and n_events) then
-			return nil, "not a valid distribution stat";
-		end
-		local histogram = {};
-
-		for i = 1, 100, 100/n_buckets do
-			histogram[i] = percentile(events, n_events, i);
-		end
-		return histogram;
-	end;
-
-	get_percentile = function (duration, pc)
-		local events, n_events = duration.samples, duration.sample_count;
-		if not (events and n_events) then
-			return nil, "not a valid distribution stat";
-		end
-		return percentile(events, n_events, pc);
-	end;
+	new = new;
 }