Mercurial > prosody-hg

annotate net/http/parser.lua @ 14217:7eb0e47418ab 13.0

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
util.human.io: Add simple version comparison This is a small helper function that compares version string (it can be passed to table.sort() as a comparator). Full version sorts (e.g. semver, and natural/alphanumeric sort in general) are more complex, and I would like to eventually support those. But this is a stable branch and we don't need it for anything just yet. Regarding justification for the stable branch: The function is being committed with extensive tests, and isn't currently used anywhere. The plan is to use it in an upcoming update of how TLS profiles are configured, but the inputs in that case will be fixed (meaning that the risk of breakage is lower than if we were running it against arbitrary data).
author Matthew Wild <mwild1@gmail.com>
date Fri, 12 Jun 2026 11:38:39 +0100
parents 190d172ab021
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
4631
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
1 local tonumber = tonumber;
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
2 local assert = assert;
4866
d54999db3aa1 net.http.parser: Do full URL decoding and parsing (e.g. adds request.url.query when present)
Matthew Wild <mwild1@gmail.com>
parents: 4716
diff changeset
3 local url_parse = require "socket.url".parse;
12974
ba409c67353b net: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents: 12889
diff changeset
4 local urldecode = require "prosody.util.http".urldecode;
ba409c67353b net: Prefix module imports with prosody namespace
Kim Alvefur <zash@zash.se>
parents: 12889
diff changeset
5 local dbuffer = require "prosody.util.dbuffer";
4631
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
6
14171
190d172ab021 net.http.parser: Add explanatory comment about scope of preprocess_path
Matthew Wild <mwild1@gmail.com>
parents: 14170
diff changeset
7
190d172ab021 net.http.parser: Add explanatory comment about scope of preprocess_path
Matthew Wild <mwild1@gmail.com>
parents: 14170
diff changeset
8 -- This is a basic check, and does *not* fully prevent path traversal (CWE-22).
190d172ab021 net.http.parser: Add explanatory comment about scope of preprocess_path
Matthew Wild <mwild1@gmail.com>
parents: 14170
diff changeset
9 -- If the path will be mapped to a filesystem, further processing and checks
190d172ab021 net.http.parser: Add explanatory comment about scope of preprocess_path
Matthew Wild <mwild1@gmail.com>
parents: 14170
diff changeset
10 -- are needed for safety and compliance. This is handled by net.http.files.
4716
6eeb142a8073 mod_http_files, net.http.parser: Move path normalization to net.http.parser so that all modules can benefit
Matthew Wild <mwild1@gmail.com>
parents: 4712
diff changeset
11 local function preprocess_path(path)
5222
61c47d26481d net.http.parser: Fix syntax error introduced in c5edb08fc7cb.
Waqas Hussain <waqas20@gmail.com>
parents: 5207
diff changeset
12 path = urldecode((path:gsub("//+", "/")));
4716
6eeb142a8073 mod_http_files, net.http.parser: Move path normalization to net.http.parser so that all modules can benefit
Matthew Wild <mwild1@gmail.com>
parents: 4712
diff changeset
13 if path:sub(1,1) ~= "/" then
6eeb142a8073 mod_http_files, net.http.parser: Move path normalization to net.http.parser so that all modules can benefit
Matthew Wild <mwild1@gmail.com>
parents: 4712
diff changeset
14 path = "/"..path;
6eeb142a8073 mod_http_files, net.http.parser: Move path normalization to net.http.parser so that all modules can benefit
Matthew Wild <mwild1@gmail.com>
parents: 4712
diff changeset
15 end
6eeb142a8073 mod_http_files, net.http.parser: Move path normalization to net.http.parser so that all modules can benefit
Matthew Wild <mwild1@gmail.com>
parents: 4712
diff changeset
16 local level = 0;
14170
948bda5ed45f net.http.parser: Include final component in path normalization check
Matthew Wild <mwild1@gmail.com>
parents: 14086
diff changeset
17 for component in path:gmatch("([^/]+)") do
4716
6eeb142a8073 mod_http_files, net.http.parser: Move path normalization to net.http.parser so that all modules can benefit
Matthew Wild <mwild1@gmail.com>
parents: 4712
diff changeset
18 if component == ".." then
6eeb142a8073 mod_http_files, net.http.parser: Move path normalization to net.http.parser so that all modules can benefit
Matthew Wild <mwild1@gmail.com>
parents: 4712
diff changeset
19 level = level - 1;
6eeb142a8073 mod_http_files, net.http.parser: Move path normalization to net.http.parser so that all modules can benefit
Matthew Wild <mwild1@gmail.com>
parents: 4712
diff changeset
20 elseif component ~= "." then
6eeb142a8073 mod_http_files, net.http.parser: Move path normalization to net.http.parser so that all modules can benefit
Matthew Wild <mwild1@gmail.com>
parents: 4712
diff changeset
21 level = level + 1;
6eeb142a8073 mod_http_files, net.http.parser: Move path normalization to net.http.parser so that all modules can benefit
Matthew Wild <mwild1@gmail.com>
parents: 4712
diff changeset
22 end
6eeb142a8073 mod_http_files, net.http.parser: Move path normalization to net.http.parser so that all modules can benefit
Matthew Wild <mwild1@gmail.com>
parents: 4712
diff changeset
23 if level < 0 then
6eeb142a8073 mod_http_files, net.http.parser: Move path normalization to net.http.parser so that all modules can benefit
Matthew Wild <mwild1@gmail.com>
parents: 4712
diff changeset
24 return nil;
6eeb142a8073 mod_http_files, net.http.parser: Move path normalization to net.http.parser so that all modules can benefit
Matthew Wild <mwild1@gmail.com>
parents: 4712
diff changeset
25 end
6eeb142a8073 mod_http_files, net.http.parser: Move path normalization to net.http.parser so that all modules can benefit
Matthew Wild <mwild1@gmail.com>
parents: 4712
diff changeset
26 end
6eeb142a8073 mod_http_files, net.http.parser: Move path normalization to net.http.parser so that all modules can benefit
Matthew Wild <mwild1@gmail.com>
parents: 4712
diff changeset
27 return path;
6eeb142a8073 mod_http_files, net.http.parser: Move path normalization to net.http.parser so that all modules can benefit
Matthew Wild <mwild1@gmail.com>
parents: 4712
diff changeset
28 end
6eeb142a8073 mod_http_files, net.http.parser: Move path normalization to net.http.parser so that all modules can benefit
Matthew Wild <mwild1@gmail.com>
parents: 4712
diff changeset
29
4631
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
30 local httpstream = {};
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
31
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
32 function httpstream.new(success_cb, error_cb, parser_type, options_cb)
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
33 local client = true;
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
34 if not parser_type or parser_type == "server" then client = false; else assert(parser_type == "client", "Invalid parser type"); end
7578
65bf55fdf971 net.http.parser: Allow limits to be configurable via options callback
Kim Alvefur <zash@zash.se>
parents: 7577
diff changeset
35 local bodylimit = tonumber(options_cb and options_cb().body_size_limit) or 10*1024*1024;
11020
7076ed654ac9 net.http.parser: Switch to util.dbuffer for buffering incoming data
Kim Alvefur <zash@zash.se>
parents: 10580
diff changeset
36 -- https://stackoverflow.com/a/686243
11727
f3aee8a825cc Fix various spelling errors (thanks codespell)
Kim Alvefur <zash@zash.se>
parents: 11184
diff changeset
37 -- Individual headers can be up to 16k? What madness?
11020
7076ed654ac9 net.http.parser: Switch to util.dbuffer for buffering incoming data
Kim Alvefur <zash@zash.se>
parents: 10580
diff changeset
38 local headlimit = tonumber(options_cb and options_cb().head_size_limit) or 10*1024;
7578
65bf55fdf971 net.http.parser: Allow limits to be configurable via options callback
Kim Alvefur <zash@zash.se>
parents: 7577
diff changeset
39 local buflimit = tonumber(options_cb and options_cb().buffer_size_limit) or bodylimit * 2;
11020
7076ed654ac9 net.http.parser: Switch to util.dbuffer for buffering incoming data
Kim Alvefur <zash@zash.se>
parents: 10580
diff changeset
40 local buffer = dbuffer.new(buflimit);
7076ed654ac9 net.http.parser: Switch to util.dbuffer for buffering incoming data
Kim Alvefur <zash@zash.se>
parents: 10580
diff changeset
41 local chunked;
4631
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
42 local state = nil;
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
43 local packet;
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
44 local len;
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
45 local have_body;
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
46 local error;
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
47 return {
7569
a15ce0014ac9 net.http.parser: Remove unused argument [luacheck]
Kim Alvefur <zash@zash.se>
parents: 6523
diff changeset
48 feed = function(_, data)
4631
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
49 if error then return nil, "parse has failed"; end
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
50 if not data then -- EOF
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
51 if state and client and not len then -- reading client body until EOF
11020
7076ed654ac9 net.http.parser: Switch to util.dbuffer for buffering incoming data
Kim Alvefur <zash@zash.se>
parents: 10580
diff changeset
52 buffer:collapse();
7076ed654ac9 net.http.parser: Switch to util.dbuffer for buffering incoming data
Kim Alvefur <zash@zash.se>
parents: 10580
diff changeset
53 packet.body = buffer:read_chunk() or "";
11184
2ede7f43ccfe net.http.parser: Expose 'partial', 'chunked' and 'body_length' on packets
Matthew Wild <mwild1@gmail.com>
parents: 11029
diff changeset
54 packet.partial = nil;
4631
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
55 success_cb(packet);
11020
7076ed654ac9 net.http.parser: Switch to util.dbuffer for buffering incoming data
Kim Alvefur <zash@zash.se>
parents: 10580
diff changeset
56 state = nil;
7076ed654ac9 net.http.parser: Switch to util.dbuffer for buffering incoming data
Kim Alvefur <zash@zash.se>
parents: 10580
diff changeset
57 elseif buffer:length() ~= 0 then -- unexpected EOF
8045
55a56dc935f2 net.http: Pass error all the way to callback
Kim Alvefur <zash@zash.se>
parents: 7635
diff changeset
58 error = true; return error_cb("unexpected-eof");
4631
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
59 end
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
60 return;
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
61 end
11020
7076ed654ac9 net.http.parser: Switch to util.dbuffer for buffering incoming data
Kim Alvefur <zash@zash.se>
parents: 10580
diff changeset
62 if not buffer:write(data) then error = true; return error_cb("max-buffer-size-exceeded"); end
7076ed654ac9 net.http.parser: Switch to util.dbuffer for buffering incoming data
Kim Alvefur <zash@zash.se>
parents: 10580
diff changeset
63 while buffer:length() > 0 do
4631
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
64 if state == nil then -- read request
11020
7076ed654ac9 net.http.parser: Switch to util.dbuffer for buffering incoming data
Kim Alvefur <zash@zash.se>
parents: 10580
diff changeset
65 local index = buffer:sub(1, headlimit):find("\r\n\r\n", nil, true);
13378
db30ffbf2090 net.http.parser: Reject overlarge header section earlier
Kim Alvefur <zash@zash.se>
parents: 12889
diff changeset
66 if not index then
db30ffbf2090 net.http.parser: Reject overlarge header section earlier
Kim Alvefur <zash@zash.se>
parents: 12889
diff changeset
67 if buffer:length() > headlimit then
db30ffbf2090 net.http.parser: Reject overlarge header section earlier
Kim Alvefur <zash@zash.se>
parents: 12889
diff changeset
68 return error_cb("header-too-large");
db30ffbf2090 net.http.parser: Reject overlarge header section earlier
Kim Alvefur <zash@zash.se>
parents: 12889
diff changeset
69 end
db30ffbf2090 net.http.parser: Reject overlarge header section earlier
Kim Alvefur <zash@zash.se>
parents: 12889
diff changeset
70 -- not enough data
db30ffbf2090 net.http.parser: Reject overlarge header section earlier
Kim Alvefur <zash@zash.se>
parents: 12889
diff changeset
71 return;
db30ffbf2090 net.http.parser: Reject overlarge header section earlier
Kim Alvefur <zash@zash.se>
parents: 12889
diff changeset
72 end
10540
375d31225d53 net.http.parser: Silence warning about unused variable [luacheck]
Kim Alvefur <zash@zash.se>
parents: 8045
diff changeset
73 -- FIXME was reason_phrase meant to be passed on somewhere?
375d31225d53 net.http.parser: Silence warning about unused variable [luacheck]
Kim Alvefur <zash@zash.se>
parents: 8045
diff changeset
74 local method, path, httpversion, status_code, reason_phrase; -- luacheck: ignore reason_phrase
4631
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
75 local first_line;
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
76 local headers = {};
11020
7076ed654ac9 net.http.parser: Switch to util.dbuffer for buffering incoming data
Kim Alvefur <zash@zash.se>
parents: 10580
diff changeset
77 for line in buffer:read(index+3):gmatch("([^\r\n]+)\r\n") do -- parse request
4631
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
78 if first_line then
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
79 local key, val = line:match("^([^%s:]+): *(.*)$");
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
80 if not key then error = true; return error_cb("invalid-header-line"); end -- TODO handle multi-line and invalid headers
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
81 key = key:lower();
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
82 headers[key] = headers[key] and headers[key]..","..val or val;
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
83 else
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
84 first_line = line;
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
85 if client then
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
86 httpversion, status_code, reason_phrase = line:match("^HTTP/(1%.[01]) (%d%d%d) (.*)$");
5462
3ecae471d9dd net.http.parser: Convert status_code to a number before trying to compare it to numbers
Matthew Wild <mwild1@gmail.com>
parents: 5461
diff changeset
87 status_code = tonumber(status_code);
4631
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
88 if not status_code then error = true; return error_cb("invalid-status-line"); end
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
89 have_body = not
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
90 ( (options_cb and options_cb().method == "HEAD")
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
91 or (status_code == 204 or status_code == 304 or status_code == 301)
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
92 or (status_code >= 100 and status_code < 200) );
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
93 else
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
94 method, path, httpversion = line:match("^(%w+) (%S+) HTTP/(1%.[01])$");
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
95 if not method then error = true; return error_cb("invalid-status-line"); end
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
96 end
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
97 end
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
98 end
5291
01f7522049fb net.http.parser: Abort if no status line is received.
Kim Alvefur <zash@zash.se>
parents: 5259
diff changeset
99 if not first_line then error = true; return error_cb("invalid-status-line"); end
4631
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
100 len = tonumber(headers["content-length"]); -- TODO check for invalid len
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
101 if client then
14086
79a6598cef0b net.http.parser: Fix handling of chunked request
Kim Alvefur <zash@zash.se>
parents: 13379
diff changeset
102 -- The presence of a message body in a response, as detailed in Section 6.3, depends on both the request method to which it is responding
79a6598cef0b net.http.parser: Fix handling of chunked request
Kim Alvefur <zash@zash.se>
parents: 13379
diff changeset
103 -- and the response status code.
79a6598cef0b net.http.parser: Fix handling of chunked request
Kim Alvefur <zash@zash.se>
parents: 13379
diff changeset
104 -- -- RFC9112 Section 6
79a6598cef0b net.http.parser: Fix handling of chunked request
Kim Alvefur <zash@zash.se>
parents: 13379
diff changeset
105 chunked = have_body and headers["transfer-encoding"] == "chunked";
4631
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
106 -- FIXME handle '100 Continue' response (by skipping it)
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
107 if not have_body then len = 0; end
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
108 packet = {
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
109 code = status_code;
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
110 httpversion = httpversion;
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
111 headers = headers;
11020
7076ed654ac9 net.http.parser: Switch to util.dbuffer for buffering incoming data
Kim Alvefur <zash@zash.se>
parents: 10580
diff changeset
112 body = false;
11184
2ede7f43ccfe net.http.parser: Expose 'partial', 'chunked' and 'body_length' on packets
Matthew Wild <mwild1@gmail.com>
parents: 11029
diff changeset
113 body_length = len;
2ede7f43ccfe net.http.parser: Expose 'partial', 'chunked' and 'body_length' on packets
Matthew Wild <mwild1@gmail.com>
parents: 11029
diff changeset
114 chunked = chunked;
2ede7f43ccfe net.http.parser: Expose 'partial', 'chunked' and 'body_length' on packets
Matthew Wild <mwild1@gmail.com>
parents: 11029
diff changeset
115 partial = true;
4631
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
116 -- COMPAT the properties below are deprecated
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
117 responseversion = httpversion;
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
118 responseheaders = headers;
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
119 };
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
120 else
14086
79a6598cef0b net.http.parser: Fix handling of chunked request
Kim Alvefur <zash@zash.se>
parents: 13379
diff changeset
121 -- The presence of a message body in a request is signaled by a Content-Length or Transfer-Encoding header field. Request message framing
79a6598cef0b net.http.parser: Fix handling of chunked request
Kim Alvefur <zash@zash.se>
parents: 13379
diff changeset
122 -- is independent of method semantics.
79a6598cef0b net.http.parser: Fix handling of chunked request
Kim Alvefur <zash@zash.se>
parents: 13379
diff changeset
123 -- -- RFC9112 Section 6
79a6598cef0b net.http.parser: Fix handling of chunked request
Kim Alvefur <zash@zash.se>
parents: 13379
diff changeset
124 chunked = headers["transfer-encoding"] == "chunked";
79a6598cef0b net.http.parser: Fix handling of chunked request
Kim Alvefur <zash@zash.se>
parents: 13379
diff changeset
125 have_body = chunked or len;
5259
c85c348253bd net.http.parser: Skip url.parse when we don't have a full URL (also fixes traceback on paths starting with '//').
Waqas Hussain <waqas20@gmail.com>
parents: 5222
diff changeset
126 local parsed_url;
c85c348253bd net.http.parser: Skip url.parse when we don't have a full URL (also fixes traceback on paths starting with '//').
Waqas Hussain <waqas20@gmail.com>
parents: 5222
diff changeset
127 if path:byte() == 47 then -- starts with /
c85c348253bd net.http.parser: Skip url.parse when we don't have a full URL (also fixes traceback on paths starting with '//').
Waqas Hussain <waqas20@gmail.com>
parents: 5222
diff changeset
128 local _path, _query = path:match("([^?]*).?(.*)");
c85c348253bd net.http.parser: Skip url.parse when we don't have a full URL (also fixes traceback on paths starting with '//').
Waqas Hussain <waqas20@gmail.com>
parents: 5222
diff changeset
129 if _query == "" then _query = nil; end
c85c348253bd net.http.parser: Skip url.parse when we don't have a full URL (also fixes traceback on paths starting with '//').
Waqas Hussain <waqas20@gmail.com>
parents: 5222
diff changeset
130 parsed_url = { path = _path, query = _query };
c85c348253bd net.http.parser: Skip url.parse when we don't have a full URL (also fixes traceback on paths starting with '//').
Waqas Hussain <waqas20@gmail.com>
parents: 5222
diff changeset
131 else
c85c348253bd net.http.parser: Skip url.parse when we don't have a full URL (also fixes traceback on paths starting with '//').
Waqas Hussain <waqas20@gmail.com>
parents: 5222
diff changeset
132 parsed_url = url_parse(path);
5323
4c30f638ff55 net.http.parser: Ensure full URL in status line contains a path.
Waqas Hussain <waqas20@gmail.com>
parents: 5322
diff changeset
133 if not(parsed_url and parsed_url.path) then error = true; return error_cb("invalid-url"); end
5259
c85c348253bd net.http.parser: Skip url.parse when we don't have a full URL (also fixes traceback on paths starting with '//').
Waqas Hussain <waqas20@gmail.com>
parents: 5222
diff changeset
134 end
4866
d54999db3aa1 net.http.parser: Do full URL decoding and parsing (e.g. adds request.url.query when present)
Matthew Wild <mwild1@gmail.com>
parents: 4716
diff changeset
135 path = preprocess_path(parsed_url.path);
4879
45bb378a4a98 net.http.parser: Keep the Host header no host is present in the URI
Kim Alvefur <zash@zash.se>
parents: 4866
diff changeset
136 headers.host = parsed_url.host or headers.host;
4712
4fc99f1b7570 net.http.parser: Handle full URLs in status line.
Waqas Hussain <waqas20@gmail.com>
parents: 4631
diff changeset
137
4631
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
138 len = len or 0;
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
139 packet = {
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
140 method = method;
4866
d54999db3aa1 net.http.parser: Do full URL decoding and parsing (e.g. adds request.url.query when present)
Matthew Wild <mwild1@gmail.com>
parents: 4716
diff changeset
141 url = parsed_url;
4631
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
142 path = path;
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
143 httpversion = httpversion;
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
144 headers = headers;
11020
7076ed654ac9 net.http.parser: Switch to util.dbuffer for buffering incoming data
Kim Alvefur <zash@zash.se>
parents: 10580
diff changeset
145 body = false;
7076ed654ac9 net.http.parser: Switch to util.dbuffer for buffering incoming data
Kim Alvefur <zash@zash.se>
parents: 10580
diff changeset
146 body_sink = nil;
11184
2ede7f43ccfe net.http.parser: Expose 'partial', 'chunked' and 'body_length' on packets
Matthew Wild <mwild1@gmail.com>
parents: 11029
diff changeset
147 chunked = chunked;
2ede7f43ccfe net.http.parser: Expose 'partial', 'chunked' and 'body_length' on packets
Matthew Wild <mwild1@gmail.com>
parents: 11029
diff changeset
148 partial = true;
4631
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
149 };
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
150 end
12882
9ed628635dc6 net.http.parser: Improve handling of responses without content-length
Matthew Wild <mwild1@gmail.com>
parents: 11727
diff changeset
151 if not len or len > bodylimit then
11021
9673c95895fb net.http.parser: Allow specifying sink for large request bodies
Kim Alvefur <zash@zash.se>
parents: 11020
diff changeset
152 -- Early notification, for redirection
9673c95895fb net.http.parser: Allow specifying sink for large request bodies
Kim Alvefur <zash@zash.se>
parents: 11020
diff changeset
153 success_cb(packet);
12882
9ed628635dc6 net.http.parser: Improve handling of responses without content-length
Matthew Wild <mwild1@gmail.com>
parents: 11727
diff changeset
154 if not packet.body_sink and (len and len > bodylimit) then
9ed628635dc6 net.http.parser: Improve handling of responses without content-length
Matthew Wild <mwild1@gmail.com>
parents: 11727
diff changeset
155 error = true;
9ed628635dc6 net.http.parser: Improve handling of responses without content-length
Matthew Wild <mwild1@gmail.com>
parents: 11727
diff changeset
156 return error_cb("content-length-limit-exceeded");
9ed628635dc6 net.http.parser: Improve handling of responses without content-length
Matthew Wild <mwild1@gmail.com>
parents: 11727
diff changeset
157 end
11021
9673c95895fb net.http.parser: Allow specifying sink for large request bodies
Kim Alvefur <zash@zash.se>
parents: 11020
diff changeset
158 end
9673c95895fb net.http.parser: Allow specifying sink for large request bodies
Kim Alvefur <zash@zash.se>
parents: 11020
diff changeset
159 if chunked and not packet.body_sink then
9673c95895fb net.http.parser: Allow specifying sink for large request bodies
Kim Alvefur <zash@zash.se>
parents: 11020
diff changeset
160 success_cb(packet);
9673c95895fb net.http.parser: Allow specifying sink for large request bodies
Kim Alvefur <zash@zash.se>
parents: 11020
diff changeset
161 if not packet.body_sink then
11029
5550fc5e83f3 net.http.parser: Fix indentation
Kim Alvefur <zash@zash.se>
parents: 11021
diff changeset
162 packet.body_buffer = dbuffer.new(buflimit);
5550fc5e83f3 net.http.parser: Fix indentation
Kim Alvefur <zash@zash.se>
parents: 11021
diff changeset
163 end
11021
9673c95895fb net.http.parser: Allow specifying sink for large request bodies
Kim Alvefur <zash@zash.se>
parents: 11020
diff changeset
164 end
4631
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
165 state = true;
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
166 end
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
167 if state then -- read body
11020
7076ed654ac9 net.http.parser: Switch to util.dbuffer for buffering incoming data
Kim Alvefur <zash@zash.se>
parents: 10580
diff changeset
168 if chunked then
7076ed654ac9 net.http.parser: Switch to util.dbuffer for buffering incoming data
Kim Alvefur <zash@zash.se>
parents: 10580
diff changeset
169 local chunk_header = buffer:sub(1, 512); -- XXX How large do chunk headers grow?
7076ed654ac9 net.http.parser: Switch to util.dbuffer for buffering incoming data
Kim Alvefur <zash@zash.se>
parents: 10580
diff changeset
170 local chunk_size, chunk_start = chunk_header:match("^(%x+)[^\r\n]*\r\n()");
7076ed654ac9 net.http.parser: Switch to util.dbuffer for buffering incoming data
Kim Alvefur <zash@zash.se>
parents: 10580
diff changeset
171 if not chunk_size then return; end
7076ed654ac9 net.http.parser: Switch to util.dbuffer for buffering incoming data
Kim Alvefur <zash@zash.se>
parents: 10580
diff changeset
172 chunk_size = chunk_size and tonumber(chunk_size, 16);
7076ed654ac9 net.http.parser: Switch to util.dbuffer for buffering incoming data
Kim Alvefur <zash@zash.se>
parents: 10580
diff changeset
173 if not chunk_size then error = true; return error_cb("invalid-chunk-size"); end
12889
94a99330ce87 net.http.parser: Fix off-by-one error in chunk parser
Matthew Wild <mwild1@gmail.com>
parents: 12882
diff changeset
174
11020
7076ed654ac9 net.http.parser: Switch to util.dbuffer for buffering incoming data
Kim Alvefur <zash@zash.se>
parents: 10580
diff changeset
175 if chunk_size == 0 and chunk_header:find("\r\n\r\n", chunk_start-2, true) then
7076ed654ac9 net.http.parser: Switch to util.dbuffer for buffering incoming data
Kim Alvefur <zash@zash.se>
parents: 10580
diff changeset
176 local body_buffer = packet.body_buffer;
7076ed654ac9 net.http.parser: Switch to util.dbuffer for buffering incoming data
Kim Alvefur <zash@zash.se>
parents: 10580
diff changeset
177 if body_buffer then
7076ed654ac9 net.http.parser: Switch to util.dbuffer for buffering incoming data
Kim Alvefur <zash@zash.se>
parents: 10580
diff changeset
178 packet.body_buffer = nil;
7076ed654ac9 net.http.parser: Switch to util.dbuffer for buffering incoming data
Kim Alvefur <zash@zash.se>
parents: 10580
diff changeset
179 body_buffer:collapse();
7076ed654ac9 net.http.parser: Switch to util.dbuffer for buffering incoming data
Kim Alvefur <zash@zash.se>
parents: 10580
diff changeset
180 packet.body = body_buffer:read_chunk() or "";
4631
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
181 end
11020
7076ed654ac9 net.http.parser: Switch to util.dbuffer for buffering incoming data
Kim Alvefur <zash@zash.se>
parents: 10580
diff changeset
182
7076ed654ac9 net.http.parser: Switch to util.dbuffer for buffering incoming data
Kim Alvefur <zash@zash.se>
parents: 10580
diff changeset
183 buffer:collapse();
7076ed654ac9 net.http.parser: Switch to util.dbuffer for buffering incoming data
Kim Alvefur <zash@zash.se>
parents: 10580
diff changeset
184 local buf = buffer:read_chunk();
7076ed654ac9 net.http.parser: Switch to util.dbuffer for buffering incoming data
Kim Alvefur <zash@zash.se>
parents: 10580
diff changeset
185 buf = buf:gsub("^.-\r\n\r\n", ""); -- This ensure extensions and trailers are stripped
7076ed654ac9 net.http.parser: Switch to util.dbuffer for buffering incoming data
Kim Alvefur <zash@zash.se>
parents: 10580
diff changeset
186 buffer:write(buf);
7076ed654ac9 net.http.parser: Switch to util.dbuffer for buffering incoming data
Kim Alvefur <zash@zash.se>
parents: 10580
diff changeset
187 state, chunked = nil, nil;
11184
2ede7f43ccfe net.http.parser: Expose 'partial', 'chunked' and 'body_length' on packets
Matthew Wild <mwild1@gmail.com>
parents: 11029
diff changeset
188 packet.partial = nil;
11020
7076ed654ac9 net.http.parser: Switch to util.dbuffer for buffering incoming data
Kim Alvefur <zash@zash.se>
parents: 10580
diff changeset
189 success_cb(packet);
12889
94a99330ce87 net.http.parser: Fix off-by-one error in chunk parser
Matthew Wild <mwild1@gmail.com>
parents: 12882
diff changeset
190 elseif buffer:length() - chunk_start - 1 >= chunk_size then -- we have a chunk
94a99330ce87 net.http.parser: Fix off-by-one error in chunk parser
Matthew Wild <mwild1@gmail.com>
parents: 12882
diff changeset
191 buffer:discard(chunk_start - 1);
11021
9673c95895fb net.http.parser: Allow specifying sink for large request bodies
Kim Alvefur <zash@zash.se>
parents: 11020
diff changeset
192 (packet.body_sink or packet.body_buffer):write(buffer:read(chunk_size));
11020
7076ed654ac9 net.http.parser: Switch to util.dbuffer for buffering incoming data
Kim Alvefur <zash@zash.se>
parents: 10580
diff changeset
193 buffer:discard(2); -- CRLF
7076ed654ac9 net.http.parser: Switch to util.dbuffer for buffering incoming data
Kim Alvefur <zash@zash.se>
parents: 10580
diff changeset
194 else -- Partial chunk remaining
5461
67b674f6a299 net.http.parser: Break when no more usable data in buffer (client part of e5ec60dfb202)
Matthew Wild <mwild1@gmail.com>
parents: 5460
diff changeset
195 break;
4631
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
196 end
11021
9673c95895fb net.http.parser: Allow specifying sink for large request bodies
Kim Alvefur <zash@zash.se>
parents: 11020
diff changeset
197 elseif packet.body_sink then
9673c95895fb net.http.parser: Allow specifying sink for large request bodies
Kim Alvefur <zash@zash.se>
parents: 11020
diff changeset
198 local chunk = buffer:read_chunk(len);
12882
9ed628635dc6 net.http.parser: Improve handling of responses without content-length
Matthew Wild <mwild1@gmail.com>
parents: 11727
diff changeset
199 while chunk and (not len or len > 0) do
11021
9673c95895fb net.http.parser: Allow specifying sink for large request bodies
Kim Alvefur <zash@zash.se>
parents: 11020
diff changeset
200 if packet.body_sink:write(chunk) then
12882
9ed628635dc6 net.http.parser: Improve handling of responses without content-length
Matthew Wild <mwild1@gmail.com>
parents: 11727
diff changeset
201 if len then
9ed628635dc6 net.http.parser: Improve handling of responses without content-length
Matthew Wild <mwild1@gmail.com>
parents: 11727
diff changeset
202 len = len - #chunk;
9ed628635dc6 net.http.parser: Improve handling of responses without content-length
Matthew Wild <mwild1@gmail.com>
parents: 11727
diff changeset
203 end
11021
9673c95895fb net.http.parser: Allow specifying sink for large request bodies
Kim Alvefur <zash@zash.se>
parents: 11020
diff changeset
204 chunk = buffer:read_chunk(len);
9673c95895fb net.http.parser: Allow specifying sink for large request bodies
Kim Alvefur <zash@zash.se>
parents: 11020
diff changeset
205 else
9673c95895fb net.http.parser: Allow specifying sink for large request bodies
Kim Alvefur <zash@zash.se>
parents: 11020
diff changeset
206 error = true;
9673c95895fb net.http.parser: Allow specifying sink for large request bodies
Kim Alvefur <zash@zash.se>
parents: 11020
diff changeset
207 return error_cb("body-sink-write-failure");
9673c95895fb net.http.parser: Allow specifying sink for large request bodies
Kim Alvefur <zash@zash.se>
parents: 11020
diff changeset
208 end
9673c95895fb net.http.parser: Allow specifying sink for large request bodies
Kim Alvefur <zash@zash.se>
parents: 11020
diff changeset
209 end
11184
2ede7f43ccfe net.http.parser: Expose 'partial', 'chunked' and 'body_length' on packets
Matthew Wild <mwild1@gmail.com>
parents: 11029
diff changeset
210 if len == 0 then
2ede7f43ccfe net.http.parser: Expose 'partial', 'chunked' and 'body_length' on packets
Matthew Wild <mwild1@gmail.com>
parents: 11029
diff changeset
211 state = nil;
2ede7f43ccfe net.http.parser: Expose 'partial', 'chunked' and 'body_length' on packets
Matthew Wild <mwild1@gmail.com>
parents: 11029
diff changeset
212 packet.partial = nil;
2ede7f43ccfe net.http.parser: Expose 'partial', 'chunked' and 'body_length' on packets
Matthew Wild <mwild1@gmail.com>
parents: 11029
diff changeset
213 success_cb(packet);
2ede7f43ccfe net.http.parser: Expose 'partial', 'chunked' and 'body_length' on packets
Matthew Wild <mwild1@gmail.com>
parents: 11029
diff changeset
214 end
12882
9ed628635dc6 net.http.parser: Improve handling of responses without content-length
Matthew Wild <mwild1@gmail.com>
parents: 11727
diff changeset
215 elseif not len or buffer:length() >= len then -- or not len
11020
7076ed654ac9 net.http.parser: Switch to util.dbuffer for buffering incoming data
Kim Alvefur <zash@zash.se>
parents: 10580
diff changeset
216 assert(not chunked)
12882
9ed628635dc6 net.http.parser: Improve handling of responses without content-length
Matthew Wild <mwild1@gmail.com>
parents: 11727
diff changeset
217 packet.body = len and buffer:read(len) or buffer:read_chunk() or "";
11184
2ede7f43ccfe net.http.parser: Expose 'partial', 'chunked' and 'body_length' on packets
Matthew Wild <mwild1@gmail.com>
parents: 11029
diff changeset
218 state = nil;
2ede7f43ccfe net.http.parser: Expose 'partial', 'chunked' and 'body_length' on packets
Matthew Wild <mwild1@gmail.com>
parents: 11029
diff changeset
219 packet.partial = nil;
2ede7f43ccfe net.http.parser: Expose 'partial', 'chunked' and 'body_length' on packets
Matthew Wild <mwild1@gmail.com>
parents: 11029
diff changeset
220 success_cb(packet);
4910
e5ec60dfb202 net.http.parser: Break loop when no more usable data in buffer
Matthew Wild <mwild1@gmail.com>
parents: 4879
diff changeset
221 else
e5ec60dfb202 net.http.parser: Break loop when no more usable data in buffer
Matthew Wild <mwild1@gmail.com>
parents: 4879
diff changeset
222 break;
4631
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
223 end
11020
7076ed654ac9 net.http.parser: Switch to util.dbuffer for buffering incoming data
Kim Alvefur <zash@zash.se>
parents: 10580
diff changeset
224 else
7076ed654ac9 net.http.parser: Switch to util.dbuffer for buffering incoming data
Kim Alvefur <zash@zash.se>
parents: 10580
diff changeset
225 break;
4631
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
226 end
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
227 end
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
228 end;
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
229 };
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
230 end
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
231
fc5d3b053454 net.http.{server|codes|parser}: Initial commit.
Waqas Hussain <waqas20@gmail.com>
parents:
diff changeset
232 return httpstream;