Mercurial > prosody-modules

comparison mod_s2s_cache_failures/mod_s2s_cache_failures.lua @ 6343:ee2ebdec1fae

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
mod_s2s_cache_failures: New module to cache failures to remote domains and control retries
author Matthew Wild <mwild1@gmail.com>
date Thu, 16 Oct 2025 19:08:20 +0100
parents
children 7ed4627ba65d
comparison
equal deleted inserted replaced
6342:4c98f4340bf1 6343:ee2ebdec1fae
1 module:set_global();
2
3 local cache = require "prosody.util.cache";
4 local st = require "prosody.util.stanza";
5 local time = require "prosody.util.time";
6
7 local measure_failed_domains = module:metric(
8 "gauge", "failed_domains", "",
9 "Number of cached domain failures"
10 );
11
12 local measure_rejected_stanzas = module:metric(
13 "counter", "failed_stanzas", "",
14 "Number of rejected stanzas to failed domains"
15 );
16
17 local failed_domains_cache_size = module:get_option_number("s2s_failure_cache_size", 128);
18
19 local domains = cache.new(failed_domains_cache_size);
20
21 -- This function returns an appropriate delay until we should retry connecting
22 -- It includes randomness, so that every Prosody server does not attempt
23 -- to reconnect to the target server at the same time (which may overload it
24 -- and cause further downtime). The logic and numbers here results in approximately
25 -- 15 retries per hour, with up to 30min between attempts.
26 local function get_holdoff_time(first_disconnect, last_disconnect)
27 -- Retry for a server that has been down for 2 hours will be the same
28 -- range as a server that has been down for 1 hour
29 local downtime = math.min(3600, last_disconnect - first_disconnect);
30 -- Range from at least 10-20s up to a range proportional to the downtime
31 return math.random(10 + downtime/4, 20 + downtime/2);
32 end
33
34 module:hook("s2sout-established", function (event)
35 -- Successfully connected, so stop tracking this domain
36 local domain = event.session.to_host;
37 domains:set(domain, nil);
38 end);
39
40 module:hook("s2sout-destroyed", function (event)
41 local domain = event.session.to_host;
42 if not event.reason or event.session.type ~= "s2sout_unauthed" then
43 return;
44 end
45 local current_time = time.now();
46 local holdoff_time = get_holdoff_time(current_time, current_time);
47 local state = domains:get(domain) or { current_time, nil, -1, current_time + holdoff_time, event.reason };
48 state[2], state[3] = current_time, state[3] + 1;
49 domains:set(domain, state);
50 module:log("info", "Preventing further connections to %s for %d seconds", holdoff_time);
51 end);
52
53 module:hook("route/remote", function (event)
54 local origin, stanza, domain = event.origin, event.stanza, event.to_host;
55 local domain_info = domains:get(domain);
56 if domain_info then
57 measure_rejected_stanzas:add(1);
58 local holdoff_until = domain_info[4];
59 if time.now() < holdoff_until then
60 return;
61 end
62 local reply = st.error_reply(
63 stanza,
64 "wait",
65 "remote-server-timeout",
66 domain_info[5] and ("Unreachable domain ("..domain_info[5]..")") or "Unreachable domain"
67 );
68 return origin.send(reply);
69 end
70 end);
71
72 module:hook("stats-update", function ()
73 measure_failed_domains:set(domains:count());
74 end);
75
76 module:add_item("shell-command", {
77 section = "s2s";
78 section_desc = "View and manage server-to-server connections";
79 name = "reset_failures";
80 desc = "Clear cache of connection failures";
81 args = {
82 { name = "remote_host", type = "string" };
83 };
84 handler = function (_, remote_host)
85 if remote_host then
86 if domains:get(remote_host) == nil then
87 return false, "No failure cached for "..remote_host;
88 end
89 domains:set(remote_host, nil);
90 return true, "Reset failure cache for "..remote_host;
91 end
92
93 -- No remote host specified, so clear all
94 domains:clear();
95 return true, "Failure cache cleared";
96 end;
97 });
98
99 module:add_item("shell-command", {
100 section = "s2s";
101 section_desc = "View and manage server-to-server connections";
102 name = "show_failures";
103 desc = "Show cache of connection failures";
104 args = {
105 { name = "remote_host", type = "string" };
106 };
107 handler = function (shell, remote_host)
108 local function show_domain_failures(domain, domain_info)
109 domain_info = domain_info or domains:get(domain);
110 if not domain_info then
111 return false;
112 end
113 local first_disconnect, last_disconnect = domain_info[1], domain_info[2];
114 local attempts, holdoff, reason = domain_info[3], domain_info[4], domain_info[5];
115
116 shell.session.print(domain..":");
117 shell.session.print(" First disconnected: "..os.date("%c", first_disconnect));
118 shell.session.print(" Last disconnected: "..os.date("%c", last_disconnect));
119 if reason then
120 shell.session.print(" because: "..reason);
121 end
122 shell.session.print(" Reconnection attempts: "..attempts);
123 shell.session.print(" No more attempts until: "..os.date("%c", last_disconnect + holdoff));
124
125 return true;
126 end
127 if remote_host then
128 if not show_domain_failures(remote_host) then
129 return false, "No failure cached for "..remote_host;
130 end
131 return true, "Showing failure record for "..remote_host;
132 end
133
134 local c = 0;
135 for domain, domain_info in domains:items() do
136 c = c + 1;
137 show_domain_failures(domain, domain_info);
138 end
139
140 return true, ("Showing %d failure records"):format(c);
141 end;
142 });