Mercurial > prosody-modules

comparison mod_muc_cache_media/mod_muc_cache_media.lua @ 6412:c25b98d42ed0

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
mod_muc_cache_media: Initial commit
author Stephen Paul Weber <singpolyma@singpolyma.net>
date Sun, 22 Feb 2026 01:08:33 -0500
parents
children 48fca0914fcf
comparison
equal deleted inserted replaced
6411:b00638d74f46 6412:c25b98d42ed0
1 local http = require "net.http"
2 local resolver = require "net.resolvers.basic"
3 local socket_url = require "socket.url"
4 local st = require "util.stanza"
5 local async = require "util.async"
6 local promise = require "util.promise"
7 local mime = require "mime"
8 local lfs = require "lfs"
9 local hashes = require "util.hashes"
10 local jid = require "util.jid"
11
12 local storage_path = module:get_option_string("muc_media_store_path", "/var/lib/prosody/muc-media")
13 local public_base_url = module:get_option_string("muc_media_public_base", "https://cache.example.com")
14 local max_size = module:get_option_number("muc_media_max_size", 10 * 1024 * 1024)
15 local MAX_REDIRECTS = 5
16
17 lfs.mkdir(storage_path)
18
19 local mime_extensions = {
20 ["image/jpeg"] = ".jpg",
21 ["image/jpg"] = ".jpg",
22 ["image/png"] = ".png",
23 ["image/gif"] = ".gif",
24 ["image/webp"] = ".webp",
25 ["image/avif"] = ".avif",
26 ["video/mp4"] = ".mp4",
27 ["video/webm"] = ".webm",
28 ["audio/mpeg"] = ".mp3",
29 ["audio/ogg"] = ".ogg",
30 }
31
32 local function extension_from_content_type(content_type)
33 if not content_type then return ".bin" end
34 local clean = content_type:match("^[^;]+")
35 clean = clean and clean:lower()
36 return mime_extensions[clean] or ".bin"
37 end
38
39 local function ipv4_to_number(ip)
40 local a,b,c,d = ip:match("^(%d+)%.(%d+)%.(%d+)%.(%d+)$")
41 if not a then return nil end
42 return (tonumber(a) << 24) | (tonumber(b) << 16) | (tonumber(c) << 8) | tonumber(d)
43 end
44
45 local function ipv4_in_range(ip, base, mask) return (ip & mask) == base end
46
47 local function is_private_ipv4(ip)
48 local n = ipv4_to_number(ip)
49 if not n then return false end
50 if ipv4_in_range(n, 0x0A000000, 0xFF000000) then return true end
51 if ipv4_in_range(n, 0xAC100000, 0xFFF00000) then return true end
52 if ipv4_in_range(n, 0xC0A80000, 0xFFFF0000) then return true end
53 if ipv4_in_range(n, 0x7F000000, 0xFF000000) then return true end
54 if ipv4_in_range(n, 0xA9FE0000, 0xFFFF0000) then return true end
55 if ipv4_in_range(n, 0x00000000, 0xFF000000) then return true end
56 if ipv4_in_range(n, 0xE0000000, 0xF0000000) then return true end
57 return false
58 end
59
60 local function is_private_ipv6(ip)
61 ip = ip:lower()
62 if ip == "::1" then return true end
63 if ip:match("^fc") or ip:match("^fd") then return true end
64 if ip:match("^fe8") or ip:match("^fe9") or ip:match("^fea") or ip:match("^feb") then return true end
65 if ip == "::" then return true end
66 return false
67 end
68
69 local function is_safe_host(host)
70 if not host then return false end
71 host = host:lower()
72 if host == "localhost" then return false end
73 local records = socket_dns.getaddrinfo(host)
74 if not records then return false end
75 for _, r in ipairs(records) do
76 local ip = r.addr
77 if ip:match("^%d+%.%d+%.%d+%.%d+$") then
78 if is_private_ipv4(ip) then return false end
79 else
80 if is_private_ipv6(ip) then return false end
81 end
82 end
83 return true
84 end
85
86 local function sha256_file(path)
87 local f = io.open(path,"rb")
88 if not f then return nil end
89 -- TODO: incremental hash
90 local digest = hashes.sha256(f:read("*all"))
91 f:close()
92 return mime.b64(digest)
93 end
94
95 local function sha3_256_file(path)
96 local f = io.open(path,"rb")
97 if not f then return nil end
98 -- TODO: incremental hash
99 local digest = hashes.sha3_256(f:read("*all"))
100 f:close()
101 return mime.b64(digest)
102 end
103
104 local function file_size(path)
105 return lfs.attributes(path,"size") or 0
106 end
107
108 local function filename_from_url(url)
109 return url:match(".*/([^/?]+)$") or "file"
110 end
111
112 local function resolve_redirects(url)
113 return promise.new(function(resolve, reject)
114 local current_url = url
115 local redirects = 0
116
117 local function step()
118 if redirects > MAX_REDIRECTS then reject("too many redirects"); return end
119 http.request(current_url,{
120 method="HEAD";
121 redirect=false;
122 }, function(_, code, response)
123 if not code or code == 0 then reject("request failed"); return end
124 if code>=300 and code<400 and response.headers.location then
125 redirects = redirects + 1
126 local location = response.headers.location
127 local parsed = socket_url.parse(current_url)
128 if not location:match("^https?://") then
129 location = parsed.scheme.."://"..parsed.host..(parsed.port and ":"..parsed.port or "")..location
130 end
131 current_url = location
132 step()
133 return
134 end
135 resolve(current_url)
136 end)
137 end
138
139 step()
140 end)
141 end
142
143 local function all_safe(resolv, didnext, net, ip)
144 if didnext and not ip then
145 return promise.resolve(true)
146 end
147
148 if not didnext or (net == "tcp4" and not is_private_ipv4(ip))
149 or (net == "tcp6" and not is_private_ipv6(ip)) then
150 return promise.new(function(resolve)
151 resolv:next(function(net,ip,port,extra)
152 all_safe(resolv, true, net, ip):next(resolve)
153 end)
154 end)
155 else
156 return promise.resolve(false)
157 end
158 end
159
160 local function secure_download_to_disk(original_url)
161 return resolve_redirects(original_url):next(function(final_url)
162 local parsed = socket_url.parse(final_url)
163 if not parsed or (parsed.scheme~="http" and parsed.scheme~="https") then return promise.reject("invalid scheme") end
164
165 return all_safe(resolver.new(parsed.host, 443, "tcp", { servername = parsed.host })):next(function(is_all_safe)
166 if not is_all_safe then return promise.reject("no safe IP") end
167
168 local tmpname = storage_path.."/tmp_"..tostring(math.random(1e9))
169 local file = io.open(tmpname,"wb")
170 if not file then return promise.reject("cannot open temp file") end
171
172 return promise.new(function(resolve, reject)
173 http.request(final_url,{
174 method = "GET";
175 redirect = false;
176 body_size_limit = max_size;
177 streaming_handler = function(r) return file, function() file:close(); file = nil end end;
178 }, function(_, code, response)
179 if code ~= 200 then
180 if file then file:close() end
181 os.remove(tmpname)
182 reject("bad status")
183 return
184 end
185 if file then
186 file:write(response.body)
187 file:close()
188 end
189 resolve({ tmpname, response.headers["content-type"] })
190 end)
191 end)
192 end)
193 end)
194 end
195
196 module:hook("muc-broadcast-message", function(event)
197 local stanza = event.stanza
198 if stanza.attr.type ~= "groupchat" then return end
199
200 local url_map = {}
201
202 -- Collect OOB URLs
203 for oob in stanza:childtags("x","jabber:x:oob") do
204 local url_tag = oob:get_child("url")
205 if url_tag then
206 url_map[url_tag:get_text()] = {}
207 end
208 end
209
210 -- Collect SIMS URLs
211 for ref in stanza:childtags("reference","urn:xmpp:reference:0") do
212 for ms in ref:childtags("media-sharing","urn:xmpp:sims:1") do
213 local sources = ms:get_child("sources")
214 if sources then
215 for sref in sources:childtags("reference","urn:xmpp:reference:0") do
216 local uri = sref.attr.uri
217 if uri then url_map[uri] = {} end
218 end
219 end
220 end
221 end
222
223 -- Cache each URL
224 for original_url,_ in pairs(url_map) do
225 local result, err = async.wait_for(secure_download_to_disk(original_url))
226 local tmp_path, content_type = result and result[1], result and result[2]
227 if not err and tmp_path then
228 local sha2 = sha256_file(tmp_path)
229 local sha3 = sha3_256_file(tmp_path)
230 local size = file_size(tmp_path)
231 local name = filename_from_url(original_url)
232 if sha2 then
233 local ext = extension_from_content_type(content_type)
234 local final_name = sha2:gsub("+","-"):gsub("/","_"):gsub("=","")..ext
235 local final_path = storage_path.."/"..final_name
236 if not lfs.attributes(final_path) then
237 os.rename(tmp_path, final_path)
238 else
239 os.remove(tmp_path)
240 end
241 url_map[original_url] = { local_url = public_base_url.."/"..final_name, path=final_path, sha2=sha2, sha3=sha3, size=size, name=name, content_type=content_type }
242 else
243 os.remove(tmp_path)
244 end
245 end
246 end
247
248 -- Rewrite URLs and enrich/create SIMS
249 local original_body = stanza:get_child_text("body")
250 for original_url,data in pairs(url_map) do
251 if data.local_url then
252
253 for oob in stanza:childtags("x","jabber:x:oob") do
254 local url_tag = oob:get_child("url")
255 if url_tag and url_tag:get_text() == original_url then
256 oob:maptags(function(el)
257 if el.name == "url" then
258 return st.stanza("url"):text(data.local_url)
259 else
260 return el
261 end
262 end)
263 end
264 end
265
266 stanza:maptags(function(el)
267 if el.name == "body" then
268 return st.stanza("body"):text(el:get_text():gsub(original_url, data.local_url))
269 elseif el.name == "fallback" and el.attr.xmlns == "urn:xmpp:fallback:0" and (el.attr["for"] == "jabber:x:oob" or el.attr["for"] == "urn:xmpp:sims:1") then
270 return el:maptags(function(subel)
271 local s, e = string.find(original_body, original_url)
272 if subel.name == "body" and math.abs(s - tonumber(subel.attr.start)) < 5 then
273 subel.attr["end"] = tostring((tonumber(subel.attr["end"]) - #original_url) + #data.local_url)
274 end
275 return subel
276 end)
277 else
278 return el
279 end
280 end)
281
282 local found = false
283 -- Enrich existing SIMS references
284 for ref in stanza:childtags("reference","urn:xmpp:reference:0") do
285 for ms in ref:childtags("media-sharing","urn:xmpp:sims:1") do
286 local sources = ms:get_child("sources")
287 if sources then
288 for sref in sources:childtags("reference","urn:xmpp:reference:0") do
289 if sref.attr.uri == original_url then
290 found = true
291 sref.attr.uri = data.local_url
292 local file_tag = ms:get_child("file","urn:xmpp:jingle:apps:file-transfer:5")
293 if not file_tag then
294 file_tag = st.stanza("file",{ xmlns="urn:xmpp:jingle:apps:file-transfer:5" })
295 ms:add_child(file_tag)
296 end
297 file_tag:maptags(function(el)
298 if el.name == "size" or el.name == "hash" then
299 return nil
300 else
301 return el
302 end
303 end)
304 -- Fill name
305 if not file_tag:get_child("name") then
306 file_tag:add_child(st.stanza("name"):text(data.name))
307 end
308 -- Fill size
309 if not file_tag:get_child("size") then
310 file_tag:add_child(st.stanza("size"):text(tostring(data.size)))
311 end
312 -- Fill SHA256 hash
313 local sha2_tag = nil
314 for htag in file_tag:childtags("hash","urn:xmpp:hashes:2") do
315 if htag.attr.algo=="sha2-256" then sha2_tag = htag end
316 end
317 if not sha2_tag then
318 file_tag:add_child(st.stanza("hash",{ xmlns="urn:xmpp:hashes:2", algo="sha2-256" }):text(data.sha2))
319 end
320 if data.sha3 then
321 -- Fill SHA3-256 hash
322 local sha3_tag = nil
323 for htag in file_tag:childtags("hash","urn:xmpp:hashes:2") do
324 if htag.attr.algo=="sha3-256" then sha3_tag = htag end
325 end
326 if not sha3_tag then
327 file_tag:add_child(st.stanza("hash",{ xmlns="urn:xmpp:hashes:2", algo="sha3-256" }):text(data.sha3))
328 end
329 end
330 end
331 end
332 end
333 end
334 end
335 -- Create new SIMS if none existed
336 if not found then
337 local ref = st.stanza("reference",{ xmlns="urn:xmpp:reference:0", type="data" })
338 local ms = st.stanza("media-sharing",{ xmlns="urn:xmpp:sims:1" })
339 local file_tag = st.stanza("file",{ xmlns="urn:xmpp:jingle:apps:file-transfer:5" })
340 file_tag:add_child(st.stanza("media-type"):text(data.content_type or "application/octet-stream"))
341 file_tag:add_child(st.stanza("name"):text(data.name))
342 file_tag:add_child(st.stanza("size"):text(tostring(data.size)))
343 file_tag:add_child(st.stanza("hash",{ xmlns="urn:xmpp:hashes:2", algo="sha2-256" }):text(data.sha2))
344 if data.sha3 then file_tag:add_child(st.stanza("hash",{ xmlns="urn:xmpp:hashes:2", algo="sha3-256" }):text(data.sha3)) end
345 ms:add_child(file_tag)
346 local sources = st.stanza("sources")
347 sources:add_child(st.stanza("reference",{ xmlns="urn:xmpp:reference:0", type="data", uri=data.local_url }))
348 ms:add_child(sources)
349 ref:add_child(ms)
350 stanza:add_child(ref)
351 end
352 end
353 end
354 end, 100)
355
356 local archive = module:open_store("muc_log", "archive");
357
358 module:hook("muc-moderate-message", function(event)
359 local room = event.room
360 local stanza_id = event.stanza_id
361 if not room or not stanza_id then return end
362
363 local room_node = jid.split(room.jid);
364
365 -- Find message by stanza_id
366 local query = {
367 with = "message<groupchat";
368 ids = { stanza_id };
369 }
370
371 local items, err = archive:find(room_node, query)
372 if not items or err then return end
373
374 for id, stanza, when in items do
375 -- Extract SIMS hashes
376 for ref in stanza:childtags("reference", "urn:xmpp:reference:0") do
377 for ms in ref:childtags("media-sharing", "urn:xmpp:sims:1") do
378 local file_tag = ms:get_child("file", "urn:xmpp:jingle:apps:file-transfer:5")
379 if file_tag then
380 for hash_tag in file_tag:childtags("hash", "urn:xmpp:hashes:2") do
381 if hash_tag.attr.algo == "sha2-256" then
382 local sha2 = hash_tag:get_text()
383 if sha2 then
384 local prefix = sha2:gsub("+","-"):gsub("/","_"):gsub("=","")
385 -- Files are named: hash + extension
386 for file in lfs.dir(storage_path) do
387 if file:match("^" .. prefix) then
388 local path = storage_path .. "/" .. file
389 if lfs.attributes(path) then
390 os.remove(path)
391 module:log("debug",
392 "Deleted cached media %s for moderated message %s",
393 file, stanza_id
394 )
395 end
396 end
397 end
398 end
399 end
400 end
401 end
402 end
403 end
404 end
405 end)