Mercurial > prosody-modules
changeset 6401:de3dc4297035
mod_xmllang_check: Initial commit
| author | Waqas Hussain <waqas20@gmail.com> |
|---|---|
| date | Mon, 09 Feb 2026 18:34:40 -0500 |
| parents | f4efb8606b92 |
| children | 4ddb5c69bc89 |
| files | mod_xmllang_check/README.md mod_xmllang_check/mod_xmllang_check.lua mod_xmllang_check/test_xmpplang.lua mod_xmllang_check/xml-lang-enforcement-findings.md mod_xmllang_check/xmpplang.lib.lua |
| diffstat | 5 files changed, 508 insertions(+), 0 deletions(-) [+] |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/mod_xmllang_check/README.md Mon Feb 09 18:34:40 2026 -0500 @@ -0,0 +1,13 @@ +# mod_xmllang_check + +This module strips invalid `xml:lang` values from S2S outgoing stanzas. This is to avoid issues with ejabberd validating these values. + +See notes on ejabberd in `xml-lang-enforcement-findings.md`. + +## Configuration + +```lua +modules_enabled = { + "xmllang_check"; +} +```
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/mod_xmllang_check/mod_xmllang_check.lua Mon Feb 09 18:34:40 2026 -0500 @@ -0,0 +1,22 @@ +-- Strip invalid xml:lang values from outgoing s2s stanzas. +-- +-- xmpplang.lib.lua must return a function that validates xml:lang strings. +local is_valid_xml_lang = module:require "xmpplang"; + +local function strip_invalid_xml_lang(element) + local xml_lang = element.attr["xml:lang"]; + if xml_lang ~= nil and not is_valid_xml_lang(xml_lang, element) then + element.attr["xml:lang"] = nil; + end + + for child in element:childtags() do + strip_invalid_xml_lang(child); + end +end + +module:hook("route/remote", function (event) + -- FIXME should we clone the stanza vs modifying in place? + -- FIXME should we replace recursively vs checking a specific subset of elements? + -- FIXME instead of outgoing, should we clean incoming stanzas from everywhere, so invalid xml:lang stanzas are never sent or stored? + strip_invalid_xml_lang(event.stanza); +end, 500);
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/mod_xmllang_check/test_xmpplang.lua Mon Feb 09 18:34:40 2026 -0500 @@ -0,0 +1,195 @@ +#!/usr/bin/env lua +-- Test suite for xmpplang.lib.lua parser + +local script_dir = arg and arg[0] and arg[0]:match("^(.*)/[^/]+$") or "."; +package.path = script_dir .. "/?.lib.lua;" .. package.path; + +local match_xmpplang = require("xmpplang") + +local passed = 0 +local failed = 0 + +local function test(input, expected, description) + local result = match_xmpplang(input) + if result == expected then + passed = passed + 1 + print(string.format("PASS: %s (%s) -> %s", description, input, tostring(result))) + else + failed = failed + 1 + print(string.format("FAIL: %s (%s) -> expected %s, got %s", description, input, tostring(expected), tostring(result))) + end +end + +print("=== Basic language tags ===") +test("en", true, "2-letter language") +test("eng", true, "3-letter language") +test("abcd", true, "4-letter language") +test("abcde", true, "5-letter language") +test("abcdefgh", true, "8-letter language") +test("abcdefghi", false, "9-letter language (invalid)") +test("e", false, "1-letter language (invalid)") + +print("\n=== Language with script ===") +test("en-Latn", true, "language-script") +test("zh-Hans", true, "zh-Hans") +test("zh-Hant", true, "zh-Hant") +test("en-Latn-US", true, "language-script-region") + +print("\n=== Language with region ===") +test("en-US", true, "language-region (2 alpha)") +test("en-GB", true, "en-GB") +test("en-123", true, "language-region (3 digit)") +-- Note: en-1234 is valid because 1234 matches variant (digit + 3 alphanum) +test("en-1234", true, "en-1234 (parsed as language + variant)") +-- Note: en-ABC is valid because ABC matches extlang (3 alpha) +test("en-ABC", true, "en-ABC (parsed as language + extlang)") +-- Actually invalid subtags +test("en-AB", true, "en-AB (language + region)") +test("en-A", false, "en-A (single alpha - invalid)") +test("en-12", false, "en-12 (2 digits - invalid)") +test("en-ABCDE", true, "en-ABCDE (language + 5 alpha variant)") + +print("\n=== Language with extlang ===") +test("zh-cmn", true, "language-extlang") +test("zh-cmn-Hans", true, "language-extlang-script") +test("zh-cmn-Hant-HK", true, "language-extlang-script-region") +test("zh-yue", true, "zh-yue") +test("zh-cmn-yue", true, "language-extlang-extlang") +test("zh-cmn-yue-gan", true, "language-extlang-extlang-extlang (max)") +test("zh-cmn-yue-gan-hak", false, "4 extlangs (too many)") + +print("\n=== Variants ===") +test("en-US-12345", true, "5-char variant") +test("en-US-12345678", true, "8-char variant") +test("en-US-1abc", true, "digit+3alphanum variant") +test("en-US-rozaj", true, "rozaj variant") +test("sl-IT-nedis", true, "sl-IT-nedis") +test("en-US-abcd", false, "4-char alpha variant (invalid)") +test("en-US-123456789", false, "9-char variant (invalid)") + +print("\n=== Extensions ===") +test("en-a-bbb", true, "extension a") +test("en-a-bbb-ccc", true, "extension with multiple subtags") +test("en-US-a-bbb", true, "language-region-extension") +test("en-US-a-bbb-b-ccc", true, "multiple extensions") +test("en-u-co-phonebk", true, "unicode extension") +test("en-a-b", false, "extension subtag too short") +test("en-a-bbbbbbbbb", false, "extension subtag too long (9)") + +print("\n=== Privateuse ===") +test("x-private", true, "privateuse standalone") +test("x-a", true, "privateuse single char subtag") +test("x-abcdefgh", true, "privateuse 8-char subtag") +test("x-abc-def", true, "privateuse multiple subtags") +test("en-x-private", true, "language-privateuse") +test("en-US-x-private", true, "language-region-privateuse") +test("X-private", true, "uppercase X privateuse") +test("x-abcdefghi", false, "privateuse subtag too long (9)") + +print("\n=== Grandfathered irregular ===") +test("en-GB-oed", true, "en-GB-oed") +test("i-ami", true, "i-ami") +test("i-bnn", true, "i-bnn") +test("i-default", true, "i-default") +test("i-enochian", true, "i-enochian") +test("i-hak", true, "i-hak") +test("i-klingon", true, "i-klingon") +test("i-lux", true, "i-lux") +test("i-mingo", true, "i-mingo") +test("i-navajo", true, "i-navajo") +test("i-pwn", true, "i-pwn") +test("i-tao", true, "i-tao") +test("i-tay", true, "i-tay") +test("i-tsu", true, "i-tsu") +test("sgn-BE-FR", true, "sgn-BE-FR") +test("sgn-BE-NL", true, "sgn-BE-NL") +test("sgn-CH-DE", true, "sgn-CH-DE") +-- Case insensitive +test("I-AMI", true, "I-AMI (uppercase)") +test("En-Gb-Oed", true, "En-Gb-Oed (mixed case)") + +print("\n=== Grandfathered regular ===") +test("art-lojban", true, "art-lojban") +test("cel-gaulish", true, "cel-gaulish") +test("no-bok", true, "no-bok") +test("no-nyn", true, "no-nyn") +test("zh-guoyu", true, "zh-guoyu") +test("zh-hakka", true, "zh-hakka") +test("zh-min", true, "zh-min") +test("zh-min-nan", true, "zh-min-nan") +test("zh-xiang", true, "zh-xiang") +-- Case insensitive +test("ZH-MIN", true, "ZH-MIN (uppercase)") +test("Art-Lojban", true, "Art-Lojban (mixed case)") + +print("\n=== Encoding ===") +test("en.UTF-8", true, "language with encoding") +test("en-US.UTF-8", true, "language-region with encoding") +test("en-Latn-US.UTF-8", true, "language-script-region with encoding") +test("en.utf-8", true, "encoding lowercase") +test("en.Utf-8", true, "encoding mixed case") +test("en.UTF-16", false, "invalid encoding") + +print("\n=== Complex valid tags ===") +test("zh-cmn-Hans-CN", true, "zh-cmn-Hans-CN") +test("sr-Latn-RS", true, "sr-Latn-RS") +test("sl-rozaj-biske", true, "sl-rozaj-biske (multiple variants)") +test("de-CH-1901", true, "de-CH-1901") +test("en-US-u-islamcal", true, "en-US-u-islamcal") +test("zh-Hans-CN-u-nu-hanidec", true, "complex with extension") + +print("\n=== Invalid cases ===") +test("", false, "empty string") +test("-en", false, "leading hyphen") +test("en-", false, "trailing hyphen") +test("en--US", false, "double hyphen") +test("123", false, "numeric only") +test("en_US", false, "underscore separator") +test("en US", false, "space separator") +test("toolonglanguage", false, "language too long") + +print("\n=== Edge cases ===") +test("qaa", true, "qaa (reserved)") +test("qaa-Qaaa-QM-x-southern", true, "all private use subtags") +test("en-Latn-GB-boont-r-extended-sequence-x-private", true, "complex tag") + +print("\n=== Additional edge cases ===") +-- Encoding edge cases +test("en.UTF-8extra", false, "encoding with extra chars") +test("en.UTF", false, "partial encoding") +test("en.", false, "dot with nothing after") +test("en-US.UTF-8.extra", false, "double dot") + +-- Extension boundary cases +test("en-0-abc", true, "extension with digit singleton") +test("en-9-abc", true, "extension with digit 9 singleton") +test("en-a-12", true, "extension with 2 char subtag") +test("en-a-12345678", true, "extension with 8 char subtag") +test("en-a-ab-cd-ef", true, "extension with multiple short subtags") + +-- Privateuse boundary cases +test("x-1", true, "privateuse with single digit") +test("x-12345678", true, "privateuse with 8 chars") +test("x-a-b-c-d-e", true, "privateuse with many 1-char subtags") + +-- Language boundary cases (no extlang possible after 4+ alpha language) +-- These are invalid because 3-alpha "ext" doesn't match script/region/variant/etc. +test("abcd-ext", false, "4-letter language with 3-char subtag (invalid)") +test("abcde-ext", false, "5-letter language with 3-char subtag (invalid)") + +-- Grandfathered should not be prefix-matched +test("i-ami-extra", false, "grandfathered with extra (invalid)") +-- Note: zh-min-foo is valid as langtag (foo is second extlang in zh-min-foo) +test("zh-min-foo", true, "zh-min-foo (valid: language + extlang + extlang)") + +-- Variants cannot repeat in a way that produces invalid structure +test("en-rozaj-rozaj", true, "repeated variant (syntactically valid)") + +-- Script + region + variant combinations +test("en-Latn-US-rozaj", true, "all optional components") +test("en-US-Latn", false, "region before script (invalid order)") + +print(string.format("\n=== Results: %d passed, %d failed ===", passed, failed)) +if failed > 0 then + os.exit(1) +end
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/mod_xmllang_check/xml-lang-enforcement-findings.md Mon Feb 09 18:34:40 2026 -0500 @@ -0,0 +1,100 @@ +# XMPP `xml:lang` Enforcement Findings (ejabberd + xmpp) + +## Scope +This documents how ProcessOne `xmpp` + `ejabberd` handle invalid `xml:lang`, including: +- where validation is applied, +- when the stream is closed vs stanza-level error, +- enforcement start date/commit, +- likely first `ejabberd` release containing it, +- exhaustive built-in element set validated with `xmpp_lang:check`. + +## Validation Mechanism +- Validator function: `xmpp_lang:check/1` + - File: `/Users/waqas/Documents/macvm/research-ejabberd/xmpp/src/xmpp_lang.erl` + - Behavior: returns input if valid, throws `{bad_lang, Value}` if invalid. +- Parser/grammar backend: + - `/Users/waqas/Documents/macvm/research-ejabberd/xmpp/c_src/xmpp_lang.rl` + - `/Users/waqas/Documents/macvm/research-ejabberd/xmpp/c_src/lang.rl` + - Grammar is enforced in NIF (`xmpp_lang`), from generated Ragel rules. + +## How Validation Is Applied +Validation is explicit per codec field, not global for every XML attribute. +The source of truth is `xmpp/specs/xmpp_codec.spec`, where `xml:lang` attrs with +`dec = {xmpp_lang, check, []}` are validated. + +### Exhaustive built-in element set validated for `xml:lang` +From `xmpp/specs/xmpp_codec.spec`: +- `stream_start` +- `iq` +- `message` +- `message_body` +- `message_subject` +- `presence` +- `presence_status` +- `error_text` +- `stream_error_text` +- `sasl_failure_text` +- `disco_identity` +- `adhoc_command` +- `jingle_reason_text` +- `jingle_ft_desc` +- `report_text` + +Notes: +- This is exhaustive for built-in generated codec elements. +- Unknown or ignored extension elements are not necessarily validated unless their decoder explicitly uses `xmpp_lang:check`. + +## Runtime Behavior: Disconnect vs Non-Disconnect +Inbound processing happens in `xmpp_stream_in`. + +### Stream start (`<stream:stream ...>`) path +- Event: `xmlstreamstart`. +- Decode failures (`{xmpp_codec, Why}`), including bad `xml:lang` on stream start, are converted to `stream:error` `invalid-xml`. +- Sending a `#stream_error{}` triggers stream close. + +Consequence: +- Invalid `xml:lang` on top stream element causes stream error + disconnect. + +### Stanza/element (`xmlstreamelement`) path +- Event: `xmlstreamelement`. +- Decode failures go through `process_invalid_xml/3`. +- If incoming element is a stanza (`iq/message/presence`), server sends stanza `bad-request` and does **not** close stream. +- For some non-stanza protocol elements (e.g., SASL auth/response/abort), server sends protocol failure (e.g., SASL malformed-request), not stream close. + +Consequence: +- Invalid `xml:lang` on validated stanza-level or validated child elements is generally stanza/protocol-level failure, not connection drop. + +## Extra Policy: Long Stream `xml:lang` +`xmpp_stream_in` has a separate rule for client stream start: +- if `size(Lang) > 35` on stream start, send `policy-violation` stream error. +- this causes disconnect (because it is a stream error). + +## Enforcement Start (Git) +### Core enforcement commit in `xmpp` +- Commit: `f6cb9a194126049fa0d562d720fd4489e7446be2` +- Date: `2018-11-23T11:50:39+03:00` +- Subject: `Validate xml:lang attribute according to RFC5234` + +### Release lineage in `xmpp` +- First tag containing this commit: `1.2.6` +- Tags containing it include: `1.2.6`, `1.2.7`, `1.2.8`, and later. + +## Likely First `ejabberd` Release Including Enforcement +Based on `ejabberd` tag -> `xmpp` dependency tag in `rebar.config`: +- `ejabberd 18.09` -> `xmpp 1.2.5` (before enforcement) +- `ejabberd 18.12` -> `xmpp 1.2.7` (after enforcement) +- `ejabberd 18.12.1` -> `xmpp 1.2.8` (after enforcement) + +Likely first `ejabberd` release with this enforcement: **`18.12`**. + +## Test Evidence in `ejabberd` +`ejabberd` suite contains explicit bad-lang stream test: +- `test_connect_bad_lang` expects `#stream_error{reason = 'invalid-xml'}` then stream end. +- File: `/Users/waqas/Documents/macvm/research-ejabberd/ejabberd/test/ejabberd_SUITE.erl` + +## Practical Compatibility Guidance for Alternate Implementations +To emulate this behavior closely: +1. Validate `xml:lang` only on known fields/elements (codec-defined), not universally on every attribute. +2. If stream-start `xml:lang` is invalid, send stream `invalid-xml` and close stream. +3. If stream-start `xml:lang` length > 35 (client stream), send stream `policy-violation` and close stream. +4. If stanza-level/child validated `xml:lang` is invalid, return stanza/protocol error without dropping the connection.
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/mod_xmllang_check/xmpplang.lib.lua Mon Feb 09 18:34:40 2026 -0500 @@ -0,0 +1,178 @@ + + +local irregular = { + "en-GB-oed", "i-ami", "i-bnn", "i-default", "i-enochian", "i-hak", "i-klingon", "i-lux", "i-mingo", "i-navajo", "i-pwn", "i-tao", "i-tay", "i-tsu", "sgn-BE-FR", "sgn-BE-NL", "sgn-CH-DE" +}; +local regular = { + "art-lojban", "cel-gaulish", "no-bok", "no-nyn", "zh-guoyu", "zh-hakka", "zh-min", "zh-min-nan", "zh-xiang" +}; +local grandfathered = {}; +for _, tag in ipairs(irregular) do + grandfathered[tag:lower()] = true; +end +for _, tag in ipairs(regular) do + grandfathered[tag:lower()] = true; +end + +local function split_subtags(lower_s) + local subtags = {}; + for subtag in lower_s:gmatch("[^%-]+") do -- split on hyphens + table.insert(subtags, subtag); + end + return subtags; +end + +local function is_privateuse(lower_s) + if lower_s:sub(1, 2) ~= "x-" then + -- print("privateuse: " .. lower_s .. " does not start with x-"); + return false; + end + + -- only allow alphanum and hyphens from here on out + if lower_s:match("[^a-z0-9%-]") then + -- print("privateuse: " .. lower_s .. " contains non-alphanum or hyphen"); + return false; + end + -- lets not allow any zero-len subtags + if lower_s:match("^%-") or lower_s:match("%-$") or lower_s:match("%-%-") then + -- print("privateuse: " .. lower_s .. " contains zero-len subtags"); + return false; + end + + local subtags = split_subtags(lower_s); + -- print("privateuse subtags: " .. table.concat(subtags, "|")); + if #subtags < 2 then return false; end + for i = 1, #subtags do + if #subtags[i] < 1 or #subtags[i] > 8 or not subtags[i]:match("^[a-z0-9]+$") then return false; end + end + -- print("privateuse: " .. lower_s .. " is valid"); + return true; +end + +local function is_langtag(lower_s) + -- print("is_langtag: " .. lower_s); + -- print(" lower_s:sub(-5): ".. lower_s:sub(-5)) + -- Remove the optional .utf8 suffix if it exists + if lower_s:sub(-6) == ".utf-8" then + -- print("removing .utf-8 suffix"); + lower_s = lower_s:sub(1, -7); + end + + -- at this point we have a hyphen-separated string of subtags, where subtags are alphanumeric + + -- only allow alphanum and hyphens from here on out + if lower_s:match("[^a-z0-9%-]") then + return false; + end + -- lets not allow any zero-len subtags + if lower_s:match("^%-") or lower_s:match("%-$") or lower_s:match("%-%-") then + return false; + end + + -- let's split on hyphens + local subtags = split_subtags(lower_s); + + -- we are now dealing with this pattern: language ( "-" script )? ( "-" region )? ( "-" variant )* ( "-" extension )* + -- we'll now process in order + + -- Let's process "language" + -- print("processing language: " .. table.concat(subtags, "|")); + local subtag = table.remove(subtags, 1); + if not subtag or not subtag:match("^[a-z]+$") then + return false; + end + if #subtag == 2 or #subtag == 3 then + -- we want to check for 0-3 instances of three-letter tags + for i = 1, 3 do + local subtag = subtags[1]; + if subtag and #subtag == 3 and subtag:match("^[a-z]+$") then + table.remove(subtags, 1); + end + end + elseif #subtag >= 4 and #subtag <= 8 then + -- passthrough + else + return false; + end + + -- Let's process optional: script = ALPHA{4}; + -- print("processing script: " .. table.concat(subtags, "|")); + subtag = subtags[1]; + if subtag and #subtag == 4 and subtag:match("^[a-z]+$") then + table.remove(subtags, 1); + end + + -- Let's process optional: region = ALPHA{2} | DIGIT{3}; + -- print("processing region: " .. table.concat(subtags, "|")); + subtag = subtags[1]; + if subtag and ((#subtag == 2 and subtag:match("^[a-z][a-z]$")) or (#subtag == 3 and subtag:match("^[0-9][0-9][0-9]$"))) then + table.remove(subtags, 1); + end + + -- Let's process zero or more: variant = alphanum{5,8} | ( DIGIT alphanum{3} ); + while true do + local variant = subtags[1]; + if not variant or (#variant < 4 or #variant > 8) then + break; + end + if #variant == 4 and not variant:match("^[0-9]") then return false; end + table.remove(subtags, 1); + end + + -- print("processing extensions: " .. table.concat(subtags, "|")); + -- Let's process zero or more: extension = singleton ( "-" alphanum{2,8} )+; + while true do + local extension = subtags[1]; + if not extension or extension == "x" then break; end + if #extension ~= 1 then return false; end + table.remove(subtags, 1); + + local part = subtags[1]; + if not part or (#part < 2 or #part > 8) then return false; end + table.remove(subtags, 1); + + while true do + part = subtags[1]; + if not part or (#part < 2 or #part > 8) then break; end + table.remove(subtags, 1); + end + end + + -- print("processing privateuse: " .. table.concat(subtags, "|")); + -- Let's process optional: privateuse = "x"i ( "-" alphanum{1,8} )+; + local privateuse = table.remove(subtags, 1); + -- print("privateuse: " .. privateuse); + if privateuse then + if privateuse ~= "x" or #subtags < 1 then return false; end + for i = 1, #subtags do + local part = subtags[i]; + if #part < 1 or #part > 8 then return false; end + end + end + + return true; +end + +local function is_xmpplang(s) + local lower_s = s:lower(); + if grandfathered[lower_s] then + return true; + end + -- print("checking privateuse: " .. lower_s); + if is_privateuse(lower_s) then + -- print("privateuse: " .. lower_s .. " is valid"); + return true; + end + -- print("checking langtag: " .. lower_s); + if is_langtag(lower_s) then + return true; + end + return false; +end + +-- print("=== x-abc-def") +-- print(is_xmpplang("x-abc-def")); +-- print("=== x-a-b-c-d-e") +-- print(is_xmpplang("x-a-b-c-d-e")); + +return is_xmpplang;
