changeset 6401:de3dc4297035

mod_xmllang_check: Initial commit
author Waqas Hussain <waqas20@gmail.com>
date Mon, 09 Feb 2026 18:34:40 -0500
parents f4efb8606b92
children 4ddb5c69bc89
files mod_xmllang_check/README.md mod_xmllang_check/mod_xmllang_check.lua mod_xmllang_check/test_xmpplang.lua mod_xmllang_check/xml-lang-enforcement-findings.md mod_xmllang_check/xmpplang.lib.lua
diffstat 5 files changed, 508 insertions(+), 0 deletions(-) [+]
line wrap: on
line diff
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/mod_xmllang_check/README.md	Mon Feb 09 18:34:40 2026 -0500
@@ -0,0 +1,13 @@
+# mod_xmllang_check
+
+This module strips invalid `xml:lang` values from S2S outgoing stanzas. This is to avoid issues with ejabberd validating these values.
+
+See notes on ejabberd in `xml-lang-enforcement-findings.md`.
+
+## Configuration
+
+```lua
+modules_enabled = {
+	"xmllang_check";
+}
+```
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/mod_xmllang_check/mod_xmllang_check.lua	Mon Feb 09 18:34:40 2026 -0500
@@ -0,0 +1,22 @@
+-- Strip invalid xml:lang values from outgoing s2s stanzas.
+--
+-- xmpplang.lib.lua must return a function that validates xml:lang strings.
+local is_valid_xml_lang = module:require "xmpplang";
+
+local function strip_invalid_xml_lang(element)
+	local xml_lang = element.attr["xml:lang"];
+	if xml_lang ~= nil and not is_valid_xml_lang(xml_lang, element) then
+		element.attr["xml:lang"] = nil;
+	end
+
+	for child in element:childtags() do
+		strip_invalid_xml_lang(child);
+	end
+end
+
+module:hook("route/remote", function (event)
+	-- FIXME should we clone the stanza vs modifying in place?
+	-- FIXME should we replace recursively vs checking a specific subset of elements?
+	-- FIXME instead of outgoing, should we clean incoming stanzas from everywhere, so invalid xml:lang stanzas are never sent or stored?
+	strip_invalid_xml_lang(event.stanza);
+end, 500);
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/mod_xmllang_check/test_xmpplang.lua	Mon Feb 09 18:34:40 2026 -0500
@@ -0,0 +1,195 @@
+#!/usr/bin/env lua
+-- Test suite for xmpplang.lib.lua parser
+
+local script_dir = arg and arg[0] and arg[0]:match("^(.*)/[^/]+$") or ".";
+package.path = script_dir .. "/?.lib.lua;" .. package.path;
+
+local match_xmpplang = require("xmpplang")
+
+local passed = 0
+local failed = 0
+
+local function test(input, expected, description)
+    local result = match_xmpplang(input)
+    if result == expected then
+        passed = passed + 1
+        print(string.format("PASS: %s (%s) -> %s", description, input, tostring(result)))
+    else
+        failed = failed + 1
+        print(string.format("FAIL: %s (%s) -> expected %s, got %s", description, input, tostring(expected), tostring(result)))
+    end
+end
+
+print("=== Basic language tags ===")
+test("en", true, "2-letter language")
+test("eng", true, "3-letter language")
+test("abcd", true, "4-letter language")
+test("abcde", true, "5-letter language")
+test("abcdefgh", true, "8-letter language")
+test("abcdefghi", false, "9-letter language (invalid)")
+test("e", false, "1-letter language (invalid)")
+
+print("\n=== Language with script ===")
+test("en-Latn", true, "language-script")
+test("zh-Hans", true, "zh-Hans")
+test("zh-Hant", true, "zh-Hant")
+test("en-Latn-US", true, "language-script-region")
+
+print("\n=== Language with region ===")
+test("en-US", true, "language-region (2 alpha)")
+test("en-GB", true, "en-GB")
+test("en-123", true, "language-region (3 digit)")
+-- Note: en-1234 is valid because 1234 matches variant (digit + 3 alphanum)
+test("en-1234", true, "en-1234 (parsed as language + variant)")
+-- Note: en-ABC is valid because ABC matches extlang (3 alpha)
+test("en-ABC", true, "en-ABC (parsed as language + extlang)")
+-- Actually invalid subtags
+test("en-AB", true, "en-AB (language + region)")
+test("en-A", false, "en-A (single alpha - invalid)")
+test("en-12", false, "en-12 (2 digits - invalid)")
+test("en-ABCDE", true, "en-ABCDE (language + 5 alpha variant)")
+
+print("\n=== Language with extlang ===")
+test("zh-cmn", true, "language-extlang")
+test("zh-cmn-Hans", true, "language-extlang-script")
+test("zh-cmn-Hant-HK", true, "language-extlang-script-region")
+test("zh-yue", true, "zh-yue")
+test("zh-cmn-yue", true, "language-extlang-extlang")
+test("zh-cmn-yue-gan", true, "language-extlang-extlang-extlang (max)")
+test("zh-cmn-yue-gan-hak", false, "4 extlangs (too many)")
+
+print("\n=== Variants ===")
+test("en-US-12345", true, "5-char variant")
+test("en-US-12345678", true, "8-char variant")
+test("en-US-1abc", true, "digit+3alphanum variant")
+test("en-US-rozaj", true, "rozaj variant")
+test("sl-IT-nedis", true, "sl-IT-nedis")
+test("en-US-abcd", false, "4-char alpha variant (invalid)")
+test("en-US-123456789", false, "9-char variant (invalid)")
+
+print("\n=== Extensions ===")
+test("en-a-bbb", true, "extension a")
+test("en-a-bbb-ccc", true, "extension with multiple subtags")
+test("en-US-a-bbb", true, "language-region-extension")
+test("en-US-a-bbb-b-ccc", true, "multiple extensions")
+test("en-u-co-phonebk", true, "unicode extension")
+test("en-a-b", false, "extension subtag too short")
+test("en-a-bbbbbbbbb", false, "extension subtag too long (9)")
+
+print("\n=== Privateuse ===")
+test("x-private", true, "privateuse standalone")
+test("x-a", true, "privateuse single char subtag")
+test("x-abcdefgh", true, "privateuse 8-char subtag")
+test("x-abc-def", true, "privateuse multiple subtags")
+test("en-x-private", true, "language-privateuse")
+test("en-US-x-private", true, "language-region-privateuse")
+test("X-private", true, "uppercase X privateuse")
+test("x-abcdefghi", false, "privateuse subtag too long (9)")
+
+print("\n=== Grandfathered irregular ===")
+test("en-GB-oed", true, "en-GB-oed")
+test("i-ami", true, "i-ami")
+test("i-bnn", true, "i-bnn")
+test("i-default", true, "i-default")
+test("i-enochian", true, "i-enochian")
+test("i-hak", true, "i-hak")
+test("i-klingon", true, "i-klingon")
+test("i-lux", true, "i-lux")
+test("i-mingo", true, "i-mingo")
+test("i-navajo", true, "i-navajo")
+test("i-pwn", true, "i-pwn")
+test("i-tao", true, "i-tao")
+test("i-tay", true, "i-tay")
+test("i-tsu", true, "i-tsu")
+test("sgn-BE-FR", true, "sgn-BE-FR")
+test("sgn-BE-NL", true, "sgn-BE-NL")
+test("sgn-CH-DE", true, "sgn-CH-DE")
+-- Case insensitive
+test("I-AMI", true, "I-AMI (uppercase)")
+test("En-Gb-Oed", true, "En-Gb-Oed (mixed case)")
+
+print("\n=== Grandfathered regular ===")
+test("art-lojban", true, "art-lojban")
+test("cel-gaulish", true, "cel-gaulish")
+test("no-bok", true, "no-bok")
+test("no-nyn", true, "no-nyn")
+test("zh-guoyu", true, "zh-guoyu")
+test("zh-hakka", true, "zh-hakka")
+test("zh-min", true, "zh-min")
+test("zh-min-nan", true, "zh-min-nan")
+test("zh-xiang", true, "zh-xiang")
+-- Case insensitive
+test("ZH-MIN", true, "ZH-MIN (uppercase)")
+test("Art-Lojban", true, "Art-Lojban (mixed case)")
+
+print("\n=== Encoding ===")
+test("en.UTF-8", true, "language with encoding")
+test("en-US.UTF-8", true, "language-region with encoding")
+test("en-Latn-US.UTF-8", true, "language-script-region with encoding")
+test("en.utf-8", true, "encoding lowercase")
+test("en.Utf-8", true, "encoding mixed case")
+test("en.UTF-16", false, "invalid encoding")
+
+print("\n=== Complex valid tags ===")
+test("zh-cmn-Hans-CN", true, "zh-cmn-Hans-CN")
+test("sr-Latn-RS", true, "sr-Latn-RS")
+test("sl-rozaj-biske", true, "sl-rozaj-biske (multiple variants)")
+test("de-CH-1901", true, "de-CH-1901")
+test("en-US-u-islamcal", true, "en-US-u-islamcal")
+test("zh-Hans-CN-u-nu-hanidec", true, "complex with extension")
+
+print("\n=== Invalid cases ===")
+test("", false, "empty string")
+test("-en", false, "leading hyphen")
+test("en-", false, "trailing hyphen")
+test("en--US", false, "double hyphen")
+test("123", false, "numeric only")
+test("en_US", false, "underscore separator")
+test("en US", false, "space separator")
+test("toolonglanguage", false, "language too long")
+
+print("\n=== Edge cases ===")
+test("qaa", true, "qaa (reserved)")
+test("qaa-Qaaa-QM-x-southern", true, "all private use subtags")
+test("en-Latn-GB-boont-r-extended-sequence-x-private", true, "complex tag")
+
+print("\n=== Additional edge cases ===")
+-- Encoding edge cases
+test("en.UTF-8extra", false, "encoding with extra chars")
+test("en.UTF", false, "partial encoding")
+test("en.", false, "dot with nothing after")
+test("en-US.UTF-8.extra", false, "double dot")
+
+-- Extension boundary cases
+test("en-0-abc", true, "extension with digit singleton")
+test("en-9-abc", true, "extension with digit 9 singleton")
+test("en-a-12", true, "extension with 2 char subtag")
+test("en-a-12345678", true, "extension with 8 char subtag")
+test("en-a-ab-cd-ef", true, "extension with multiple short subtags")
+
+-- Privateuse boundary cases
+test("x-1", true, "privateuse with single digit")
+test("x-12345678", true, "privateuse with 8 chars")
+test("x-a-b-c-d-e", true, "privateuse with many 1-char subtags")
+
+-- Language boundary cases (no extlang possible after 4+ alpha language)
+-- These are invalid because 3-alpha "ext" doesn't match script/region/variant/etc.
+test("abcd-ext", false, "4-letter language with 3-char subtag (invalid)")
+test("abcde-ext", false, "5-letter language with 3-char subtag (invalid)")
+
+-- Grandfathered should not be prefix-matched
+test("i-ami-extra", false, "grandfathered with extra (invalid)")
+-- Note: zh-min-foo is valid as langtag (foo is second extlang in zh-min-foo)
+test("zh-min-foo", true, "zh-min-foo (valid: language + extlang + extlang)")
+
+-- Variants cannot repeat in a way that produces invalid structure
+test("en-rozaj-rozaj", true, "repeated variant (syntactically valid)")
+
+-- Script + region + variant combinations
+test("en-Latn-US-rozaj", true, "all optional components")
+test("en-US-Latn", false, "region before script (invalid order)")
+
+print(string.format("\n=== Results: %d passed, %d failed ===", passed, failed))
+if failed > 0 then
+    os.exit(1)
+end
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/mod_xmllang_check/xml-lang-enforcement-findings.md	Mon Feb 09 18:34:40 2026 -0500
@@ -0,0 +1,100 @@
+# XMPP `xml:lang` Enforcement Findings (ejabberd + xmpp)
+
+## Scope
+This documents how ProcessOne `xmpp` + `ejabberd` handle invalid `xml:lang`, including:
+- where validation is applied,
+- when the stream is closed vs stanza-level error,
+- enforcement start date/commit,
+- likely first `ejabberd` release containing it,
+- exhaustive built-in element set validated with `xmpp_lang:check`.
+
+## Validation Mechanism
+- Validator function: `xmpp_lang:check/1`
+  - File: `/Users/waqas/Documents/macvm/research-ejabberd/xmpp/src/xmpp_lang.erl`
+  - Behavior: returns input if valid, throws `{bad_lang, Value}` if invalid.
+- Parser/grammar backend:
+  - `/Users/waqas/Documents/macvm/research-ejabberd/xmpp/c_src/xmpp_lang.rl`
+  - `/Users/waqas/Documents/macvm/research-ejabberd/xmpp/c_src/lang.rl`
+  - Grammar is enforced in NIF (`xmpp_lang`), from generated Ragel rules.
+
+## How Validation Is Applied
+Validation is explicit per codec field, not global for every XML attribute.
+The source of truth is `xmpp/specs/xmpp_codec.spec`, where `xml:lang` attrs with
+`dec = {xmpp_lang, check, []}` are validated.
+
+### Exhaustive built-in element set validated for `xml:lang`
+From `xmpp/specs/xmpp_codec.spec`:
+- `stream_start`
+- `iq`
+- `message`
+- `message_body`
+- `message_subject`
+- `presence`
+- `presence_status`
+- `error_text`
+- `stream_error_text`
+- `sasl_failure_text`
+- `disco_identity`
+- `adhoc_command`
+- `jingle_reason_text`
+- `jingle_ft_desc`
+- `report_text`
+
+Notes:
+- This is exhaustive for built-in generated codec elements.
+- Unknown or ignored extension elements are not necessarily validated unless their decoder explicitly uses `xmpp_lang:check`.
+
+## Runtime Behavior: Disconnect vs Non-Disconnect
+Inbound processing happens in `xmpp_stream_in`.
+
+### Stream start (`<stream:stream ...>`) path
+- Event: `xmlstreamstart`.
+- Decode failures (`{xmpp_codec, Why}`), including bad `xml:lang` on stream start, are converted to `stream:error` `invalid-xml`.
+- Sending a `#stream_error{}` triggers stream close.
+
+Consequence:
+- Invalid `xml:lang` on top stream element causes stream error + disconnect.
+
+### Stanza/element (`xmlstreamelement`) path
+- Event: `xmlstreamelement`.
+- Decode failures go through `process_invalid_xml/3`.
+- If incoming element is a stanza (`iq/message/presence`), server sends stanza `bad-request` and does **not** close stream.
+- For some non-stanza protocol elements (e.g., SASL auth/response/abort), server sends protocol failure (e.g., SASL malformed-request), not stream close.
+
+Consequence:
+- Invalid `xml:lang` on validated stanza-level or validated child elements is generally stanza/protocol-level failure, not connection drop.
+
+## Extra Policy: Long Stream `xml:lang`
+`xmpp_stream_in` has a separate rule for client stream start:
+- if `size(Lang) > 35` on stream start, send `policy-violation` stream error.
+- this causes disconnect (because it is a stream error).
+
+## Enforcement Start (Git)
+### Core enforcement commit in `xmpp`
+- Commit: `f6cb9a194126049fa0d562d720fd4489e7446be2`
+- Date: `2018-11-23T11:50:39+03:00`
+- Subject: `Validate xml:lang attribute according to RFC5234`
+
+### Release lineage in `xmpp`
+- First tag containing this commit: `1.2.6`
+- Tags containing it include: `1.2.6`, `1.2.7`, `1.2.8`, and later.
+
+## Likely First `ejabberd` Release Including Enforcement
+Based on `ejabberd` tag -> `xmpp` dependency tag in `rebar.config`:
+- `ejabberd 18.09` -> `xmpp 1.2.5` (before enforcement)
+- `ejabberd 18.12` -> `xmpp 1.2.7` (after enforcement)
+- `ejabberd 18.12.1` -> `xmpp 1.2.8` (after enforcement)
+
+Likely first `ejabberd` release with this enforcement: **`18.12`**.
+
+## Test Evidence in `ejabberd`
+`ejabberd` suite contains explicit bad-lang stream test:
+- `test_connect_bad_lang` expects `#stream_error{reason = 'invalid-xml'}` then stream end.
+- File: `/Users/waqas/Documents/macvm/research-ejabberd/ejabberd/test/ejabberd_SUITE.erl`
+
+## Practical Compatibility Guidance for Alternate Implementations
+To emulate this behavior closely:
+1. Validate `xml:lang` only on known fields/elements (codec-defined), not universally on every attribute.
+2. If stream-start `xml:lang` is invalid, send stream `invalid-xml` and close stream.
+3. If stream-start `xml:lang` length > 35 (client stream), send stream `policy-violation` and close stream.
+4. If stanza-level/child validated `xml:lang` is invalid, return stanza/protocol error without dropping the connection.
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/mod_xmllang_check/xmpplang.lib.lua	Mon Feb 09 18:34:40 2026 -0500
@@ -0,0 +1,178 @@
+
+
+local irregular = {
+  "en-GB-oed", "i-ami", "i-bnn", "i-default", "i-enochian", "i-hak", "i-klingon", "i-lux", "i-mingo", "i-navajo", "i-pwn", "i-tao", "i-tay", "i-tsu", "sgn-BE-FR", "sgn-BE-NL", "sgn-CH-DE"
+};
+local regular = {
+  "art-lojban", "cel-gaulish", "no-bok", "no-nyn", "zh-guoyu", "zh-hakka", "zh-min", "zh-min-nan", "zh-xiang"
+};
+local grandfathered = {};
+for _, tag in ipairs(irregular) do
+  grandfathered[tag:lower()] = true;
+end
+for _, tag in ipairs(regular) do
+  grandfathered[tag:lower()] = true;
+end
+
+local function split_subtags(lower_s)
+  local subtags = {};
+  for subtag in lower_s:gmatch("[^%-]+") do -- split on hyphens
+    table.insert(subtags, subtag);
+  end
+  return subtags;
+end
+
+local function is_privateuse(lower_s)
+  if lower_s:sub(1, 2) ~= "x-" then
+    -- print("privateuse: " .. lower_s .. " does not start with x-");
+    return false;
+  end
+
+  -- only allow alphanum and hyphens from here on out
+  if lower_s:match("[^a-z0-9%-]") then
+    -- print("privateuse: " .. lower_s .. " contains non-alphanum or hyphen");
+    return false;
+  end
+  -- lets not allow any zero-len subtags
+  if lower_s:match("^%-") or lower_s:match("%-$") or lower_s:match("%-%-") then
+    -- print("privateuse: " .. lower_s .. " contains zero-len subtags");
+    return false;
+  end
+
+  local subtags = split_subtags(lower_s);
+  -- print("privateuse subtags: " .. table.concat(subtags, "|"));
+  if #subtags < 2 then return false; end
+  for i = 1, #subtags do
+    if #subtags[i] < 1 or #subtags[i] > 8 or not subtags[i]:match("^[a-z0-9]+$") then return false; end
+  end
+  -- print("privateuse: " .. lower_s .. " is valid");
+  return true;
+end
+
+local function is_langtag(lower_s)
+  -- print("is_langtag: " .. lower_s);
+  -- print(" lower_s:sub(-5): ".. lower_s:sub(-5))
+  -- Remove the optional .utf8 suffix if it exists
+  if lower_s:sub(-6) == ".utf-8" then
+    -- print("removing .utf-8 suffix");
+    lower_s = lower_s:sub(1, -7);
+  end
+
+  -- at this point we have a hyphen-separated string of subtags, where subtags are alphanumeric
+
+  -- only allow alphanum and hyphens from here on out
+  if lower_s:match("[^a-z0-9%-]") then
+    return false;
+  end
+  -- lets not allow any zero-len subtags
+  if lower_s:match("^%-") or lower_s:match("%-$") or lower_s:match("%-%-") then
+    return false;
+  end
+
+  -- let's split on hyphens
+  local subtags = split_subtags(lower_s);
+
+  -- we are now dealing with this pattern: language ( "-" script )? ( "-" region )? ( "-" variant )* ( "-" extension )*
+  -- we'll now process in order
+
+  -- Let's process "language"
+  -- print("processing language: " .. table.concat(subtags, "|"));
+  local subtag = table.remove(subtags, 1);
+  if not subtag or not subtag:match("^[a-z]+$") then
+    return false;
+  end
+  if #subtag == 2 or #subtag == 3 then
+    -- we want to check for 0-3 instances of three-letter tags
+    for i = 1, 3 do
+      local subtag = subtags[1];
+      if subtag and #subtag == 3 and subtag:match("^[a-z]+$") then
+        table.remove(subtags, 1);
+      end
+    end
+  elseif #subtag >= 4 and #subtag <= 8 then
+    -- passthrough
+  else
+    return false;
+  end
+
+  -- Let's process optional: script = ALPHA{4};
+  -- print("processing script: " .. table.concat(subtags, "|"));
+  subtag = subtags[1];
+  if subtag and #subtag == 4 and subtag:match("^[a-z]+$") then
+    table.remove(subtags, 1);
+  end
+
+  -- Let's process optional: region = ALPHA{2} | DIGIT{3};
+  -- print("processing region: " .. table.concat(subtags, "|"));
+  subtag = subtags[1];
+  if subtag and ((#subtag == 2 and subtag:match("^[a-z][a-z]$")) or (#subtag == 3 and subtag:match("^[0-9][0-9][0-9]$"))) then
+    table.remove(subtags, 1);
+  end
+
+  -- Let's process zero or more: variant = alphanum{5,8} | ( DIGIT alphanum{3} );
+  while true do
+    local variant = subtags[1];
+    if not variant or (#variant < 4 or #variant > 8) then
+      break;
+    end
+    if #variant == 4 and not variant:match("^[0-9]") then return false; end
+    table.remove(subtags, 1);
+  end
+
+  -- print("processing extensions: " .. table.concat(subtags, "|"));
+  -- Let's process zero or more: extension = singleton ( "-" alphanum{2,8} )+;
+  while true do
+    local extension = subtags[1];
+    if not extension or extension == "x" then break; end
+    if #extension ~= 1 then return false; end
+    table.remove(subtags, 1);
+
+    local part = subtags[1];
+    if not part or (#part < 2 or #part > 8) then return false; end
+    table.remove(subtags, 1);
+
+    while true do
+      part = subtags[1];
+      if not part or (#part < 2 or #part > 8) then break; end
+      table.remove(subtags, 1);
+    end
+  end
+
+  -- print("processing privateuse: " .. table.concat(subtags, "|"));
+  -- Let's process optional: privateuse = "x"i ( "-" alphanum{1,8} )+;
+  local privateuse = table.remove(subtags, 1);
+  -- print("privateuse: " .. privateuse);
+  if privateuse then
+    if privateuse ~= "x" or #subtags < 1 then return false; end
+    for i = 1, #subtags do
+      local part = subtags[i];
+      if #part < 1 or #part > 8 then return false; end
+    end
+  end
+
+  return true;
+end
+
+local function is_xmpplang(s)
+  local lower_s = s:lower();
+  if grandfathered[lower_s] then
+    return true;
+  end
+  -- print("checking privateuse: " .. lower_s);
+  if is_privateuse(lower_s) then
+    -- print("privateuse: " .. lower_s .. " is valid");
+    return true;
+  end
+  -- print("checking langtag: " .. lower_s);
+  if is_langtag(lower_s) then
+    return true;
+  end
+  return false;
+end
+
+-- print("=== x-abc-def")
+-- print(is_xmpplang("x-abc-def"));
+-- print("=== x-a-b-c-d-e")
+-- print(is_xmpplang("x-a-b-c-d-e"));
+
+return is_xmpplang;