moonbridge
view moonbridge_http.lua @ 159:bd7225b30391
Further work on new HTTP layer (not finished)
| author | jbe |
|---|---|
| date | Fri Jun 05 19:53:41 2015 +0200 (2015-06-05) |
| parents | 99a70d18e47c |
| children | 573995950b0b |
line source
1 #!/usr/bin/env lua
3 -- module preamble
4 local _G, _M = _ENV, {}
5 _ENV = setmetatable({}, {
6 __index = function(self, key)
7 local value = _M[key]; if value ~= nil then return value end
8 return _G[key]
9 end,
10 __newindex = _M
11 })
13 -- function that encodes certain HTML entities:
14 -- (not used by the library itself)
15 function encode_html(text)
16 return (
17 string.gsub(
18 text, '[<>&"]',
19 function(char)
20 if char == '<' then
21 return "<"
22 elseif char == '>' then
23 return ">"
24 elseif char == '&' then
25 return "&"
26 elseif char == '"' then
27 return """
28 end
29 end
30 )
31 )
33 end
35 -- function that encodes special characters for URIs:
36 -- (not used by the library itself)
37 function encode_uri(text)
38 return (
39 string.gsub(text, "[^0-9A-Za-z_%.~-]",
40 function (char)
41 return string.format("%%%02x", string.byte(char))
42 end
43 )
44 )
45 end
47 -- function undoing URL encoding:
48 do
49 local b0 = string.byte("0")
50 local b9 = string.byte("9")
51 local bA = string.byte("A")
52 local bF = string.byte("F")
53 local ba = string.byte("a")
54 local bf = string.byte("f")
55 function decode_uri(str)
56 return (
57 string.gsub(
58 string.gsub(str, "%+", " "),
59 "%%([0-9A-Fa-f][0-9A-Fa-f])",
60 function(hex)
61 local n1, n2 = string.byte(hex, 1, 2)
62 if n1 >= b0 and n1 <= b9 then n1 = n1 - b0
63 elseif n1 >= bA and n1 <= bF then n1 = n1 - bA + 10
64 elseif n1 >= ba and n1 <= bf then n1 = n1 - ba + 10
65 else error("Assertion failed") end
66 if n2 >= b0 and n2 <= b9 then n2 = n2 - b0
67 elseif n2 >= bA and n2 <= bF then n2 = n2 - bA + 10
68 elseif n2 >= ba and n2 <= bf then n2 = n2 - ba + 10
69 else error("Assertion failed") end
70 return string.char(n1 * 16 + n2)
71 end
72 )
73 )
74 end
75 end
77 -- status codes that carry no response body (in addition to 1xx):
78 -- (set to "zero_content_length" if Content-Length header is required)
79 status_without_response_body = {
80 ["101"] = true, -- list 101 to allow protocol switch
81 ["204"] = true,
82 ["205"] = "zero_content_length",
83 ["304"] = true
84 }
86 -- parses URL encoded form data:
87 local function read_urlencoded_form(data)
88 local tbl = {}
89 for rawkey, rawvalue in string.gmatch(data, "([^?=&]*)=([^?=&]*)") do
90 local key = decode_uri(rawkey)
91 local value = decode_uri(rawvalue)
92 local subtbl = tbl[key]
93 if subtbl then
94 subtbl[#subtbl+1] = value
95 else
96 tbl[key] = {value}
97 end
98 end
99 return tbl
100 end
102 -- extracts first value from each subtable:
103 local function get_first_values(tbl)
104 local newtbl = {}
105 for key, subtbl in pairs(tbl) do
106 newtbl[key] = subtbl[1]
107 end
108 return newtbl
109 end
111 local headers_mt_self = setmetatable({}, {__mode="k"})
113 local headers_mts = {
114 headers_mt = {
115 __index = function(tbl, key)
116 local self = headers_mt_self[tbl]
117 local lowerkey = string.lower(key)
118 local result = self._headers[lowerkey]
119 if result == nil then
120 result = {}
121 end
122 tbl[lowerkey] = result
123 tbl[key] = result
124 return result
125 end
126 },
127 -- table mapping header field names to value-lists
128 -- (for headers with comma separated values):
129 headers_csv_table = {
130 __index = function(tbl, key)
131 local self = headers_mt_self[tbl]
132 local result = {}
133 for i, line in ipairs(self.headers[key]) do
134 for entry in string.gmatch(line, "[^,]+") do
135 local value = string.match(entry, "^[ \t]*(..-)[ \t]*$")
136 if value then
137 result[#result+1] = value
138 end
139 end
140 end
141 tbl[key] = result
142 return result
143 end
144 },
145 -- table mapping header field names to a comma separated string
146 -- (for headers with comma separated values):
147 headers_csv_string = {
148 __index = function(tbl, key)
149 local self = headers_mt_self[tbl]
150 local result = {}
151 for i, line in ipairs(self.headers[key]) do
152 result[#result+1] = line
153 end
154 result = string.concat(result, ", ")
155 tbl[key] = result
156 return result
157 end
158 },
159 -- table mapping header field names to a single string value
160 -- (or false if header has been sent multiple times):
161 headers_value = {
162 __index = function(tbl, key)
163 local self = headers_mt_self[tbl]
164 if self._headers_value_nil[key] then
165 return nil
166 end
167 local result = nil
168 local values = self.headers_csv_table[key]
169 if #values == 0 then
170 self._headers_value_nil[key] = true
171 elseif #values == 1 then
172 result = values[1]
173 else
174 result = false
175 end
176 tbl[key] = result
177 return result
178 end
179 },
180 -- table mapping header field names to a flag table,
181 -- indicating if the comma separated value contains certain entries:
182 headers_flags = {
183 __index = function(tbl, key)
184 local self = headers_mt_self[tbl]
185 local result = setmetatable({}, {
186 __index = function(tbl, key)
187 local lowerkey = string.lower(key)
188 local result = rawget(tbl, lowerkey) or false
189 tbl[lowerkey] = result
190 tbl[key] = result
191 return result
192 end
193 })
194 for i, value in ipairs(self.headers_csv_table[key]) do
195 result[string.lower(value)] = true
196 end
197 tbl[key] = result
198 return result
199 end
200 }
201 }
203 request_pt = {}
204 request_mt = { __index = request_pt }
206 function request_pt:_init(handler, options)
207 self._application_handler = handler
208 -- process options:
209 options = options or {}
210 do
211 -- named arg "static_headers" is used to create the preamble:
212 local s = options.static_headers
213 local t = {}
214 if s then
215 if type(s) == "string" then
216 for line in string.gmatch(s, "[^\r\n]+") do
217 t[#t+1] = line
218 end
219 else
220 for i, kv in ipairs(options.static_headers) do
221 if type(kv) == "string" then
222 t[#t+1] = kv
223 else
224 t[#t+1] = kv[1] .. ": " .. kv[2]
225 end
226 end
227 end
228 end
229 t[#t+1] = ""
230 self._preamble = table.concat(t, "\r\n") -- preamble sent with every(!) HTTP response
231 end
232 self._input_chunk_size = options.maximum_input_chunk_size or options.chunk_size or 16384
233 self._output_chunk_size = options.minimum_output_chunk_size or options.chunk_size or 1024
234 self._header_size_limit = options.header_size_limit or 1024*1024
235 self._body_size_limit = options.body_size_limit or 64*1024*1024
236 local function init_timeout(name, default)
237 local value = options[name]
238 if value == nil then
239 self["_"..name] = default
240 else
241 self["_"..name] = value
242 end
243 end
244 init_timeout("request_idle_timeout", 330)
245 init_timeout("request_header_timeout", 30)
246 init_timeout("request_body_timeout", 1800)
247 init_timeout("response_timeout", 1830)
248 self._poll = options.poll_function or moonbridge_io.poll
249 self:_create_closure("_write_yield")
250 self:_create_closure("_handler")
251 self:_create_header_metatables()
252 end
254 function request_pt:_create_closure(name)
255 self[name.."_closure"] = function(...)
256 return self[name](self, ...)
257 end
258 end
260 function request_pt:_handler(socket)
261 self._socket = socket
262 self._survive = true
263 self._socket_set = {[socket] = true}
264 self._faulty = false
265 self._state = "config"
266 self._connection_close_requested = false
267 self._connection_close_responded = false
268 for name, mt in pairs(headers_mts) do
269 local tbl = setmetatable({}, mt)
270 headers_mt_self[tbl] = self
271 self[name] = tbl
272 end
273 repeat
274 -- wait for input:
275 if not self._poll(self._socket_set, nil, self._request_idle_timeout) then
276 self:_error("408 Request Timeout", "Idle connection timed out")
277 return self._survive
278 end
279 -- read headers (with timeout):
280 do
281 local coro = coroutine.wrap(self._read_headers)
282 local timeout = self._request_header_timeout
283 local starttime = timeout and moonbridge_io.timeref()
284 while true do
285 local status = coro(self)
286 if status == nil then
287 local remaining
288 if timeout then
289 remaining = timeout - moonbridge_io.timeref(starttime)
290 end
291 if not self._poll(self._socket_set, nil, remaining) then
292 self:_error("408 Request Timeout", "Timeout while receiving headers")
293 return self._survive
294 end
295 elseif status == false then
296 return self._survive
297 elseif status == true then
298 break
299 else
300 error("Unexpected yield value")
301 end
302 end
303 end
304 -- prepare reading of body:
305 self._read_body_coro = coroutine.wrap(self._read_body) --TODO?
306 -- set timeout for application handler:
307 timeout(self._response_timeout or 0)
308 -- call application handler:
309 if self._application_handler(self) ~= true then
310 self._survive = false
311 end
312 -- enforce request:finish()
313 request:finish()
314 -- reset timeout of application handler
315 timeout(0)
316 until self._connection_close_responded
317 return self._survive
318 end
320 function request_pt:_prepare_body()
321 self:_assert_not_faulty()
322 if self._state == "prepare" then
323 error("Unexpected state in HTTP module")
324 elseif self._state ~= "config" then
325 return
326 end
327 self._state = "prepare"
328 local content_type = self.headers_value["Content-Type"]
329 if content_type then
330 if
331 content_type == "application/x-www-form-urlencoded" or
332 string.match(content_type, "^application/x%-www%-form%-urlencoded *;")
333 then
334 self._consume_all_input()
335 self.post_params_list = read_urlencoded_form(self.body)
336 else
337 local boundary = string.match(
338 content_type,
339 '^multipart/form%-data[ \t]*[;,][ \t]*boundary="([^"]+)"$'
340 ) or string.match(
341 content_type,
342 '^multipart/form%-data[ \t]*[;,][ \t]*boundary=([^"; \t]+)$'
343 )
344 if boundary then
345 self.post_metadata_list = {}
346 boundary = "--" .. boundary
347 local headerdata = ""
348 local streamer
349 local field_name
350 local metadata = {}
351 local value_parts
352 local function default_streamer(chunk)
353 value_parts[#value_parts+1] = chunk
354 end
355 local function stream_part_finish()
356 if streamer == default_streamer then
357 local value = table.concat(value_parts)
358 value_parts = nil
359 if field_name then
360 local values = self.post_params_list[field_name]
361 values[#values+1] = value
362 local metadata_entries = post_metadata_list[field_name]
363 metadata_entries[#metadata_entries+1] = metadata
364 end
365 else
366 streamer()
367 end
368 headerdata = ""
369 streamer = nil
370 field_name = nil
371 metadata = {}
372 end
373 local function stream_part_chunk(chunk)
374 if streamer then
375 streamer(chunk)
376 else
377 headerdata = headerdata .. chunk
378 while true do
379 local line, remaining = string.match(headerdata, "^(.-)\r?\n(.*)$")
380 if not line then
381 break
382 end
383 if line == "" then
384 streamer = streamed_post_params[field_name]
385 if not streamer then
386 for i, rule in ipairs(streamed_post_param_patterns) do
387 if string.match(field_name, rule[1]) then
388 streamer = rule[2]
389 break
390 end
391 end
392 end
393 if not streamer then
394 value_parts = {}
395 streamer = default_streamer
396 end
397 streamer(remaining, field_name, metadata)
398 return
399 end
400 headerdata = remaining
401 local header_key, header_value = string.match(line, "^([^:]*):[ \t]*(.-)[ \t]*$")
402 if not header_key then
403 request_error(true, "400 Bad Request", "Invalid header in multipart/form-data part")
404 end
405 header_key = string.lower(header_key)
406 if header_key == "content-disposition" then
407 local escaped_header_value = string.gsub(header_value, '"[^"]*"', function(str)
408 return string.gsub(str, "=", "==")
409 end)
410 field_name = string.match(escaped_header_value, ';[ \t]*name="([^"]*)"')
411 if field_name then
412 field_name = string.gsub(field_name, "==", "=")
413 else
414 field_name = string.match(header_value, ';[ \t]*name=([^"; \t]+)')
415 end
416 metadata.file_name = string.match(escaped_header_value, ';[ \t]*filename="([^"]*)"')
417 if metadata.file_name then
418 metadata.file_name = string.gsub(metadata.file_name, "==", "=")
419 else
420 string.match(header_value, ';[ \t]*filename=([^"; \t]+)')
421 end
422 elseif header_key == "content-type" then
423 metadata.content_type = header_value
424 elseif header_key == "content-transfer-encoding" then
425 request_error(true, "400 Bad Request", "Content-transfer-encoding not supported by multipart/form-data parser")
426 end
427 end
428 end
429 end
430 local skippart = true -- ignore data until first boundary
431 local afterbound = false -- interpret 2 bytes after boundary ("\r\n" or "--")
432 local terminated = false -- final boundary read
433 local bigchunk = ""
434 request:stream_request_body(function(chunk)
435 if terminated then
436 return
437 end
438 bigchunk = bigchunk .. chunk
439 while true do
440 if afterbound then
441 if #bigchunk <= 2 then
442 return
443 end
444 local terminator = string.sub(bigchunk, 1, 2)
445 if terminator == "\r\n" then
446 afterbound = false
447 bigchunk = string.sub(bigchunk, 3)
448 elseif terminator == "--" then
449 terminated = true
450 bigchunk = nil
451 return
452 else
453 request_error(true, "400 Bad Request", "Error while parsing multipart body (expected CRLF or double minus)")
454 end
455 end
456 local pos1, pos2 = string.find(bigchunk, boundary, 1, true)
457 if not pos1 then
458 if not skippart then
459 local safe = #bigchunk-#boundary
460 if safe > 0 then
461 stream_part_chunk(string.sub(bigchunk, 1, safe))
462 bigchunk = string.sub(bigchunk, safe+1)
463 end
464 end
465 return
466 end
467 if not skippart then
468 stream_part_chunk(string.sub(bigchunk, 1, pos1 - 1))
469 stream_part_finish()
470 else
471 boundary = "\r\n" .. boundary
472 skippart = false
473 end
474 bigchunk = string.sub(bigchunk, pos2 + 1)
475 afterbound = true
476 end
477 end)
478 if not terminated then
479 request_error(true, "400 Bad Request", "Premature end of multipart/form-data request body")
480 end
481 request.post_metadata_list, request.post_metadata = post_metadata_list, post_metadata
482 else
483 request_error(true, "415 Unsupported Media Type", "Unknown Content-Type of request body")
484 end
485 end
486 end
487 self.post_params = get_first_values(self.post_params_list)
488 self._state = "no_status_sent"
489 end
491 function request_pt:_drain_input()
492 self._read_body_coro = "drain"
493 end
495 function request_pt:_consume_some_input()
496 local coro = self._read_body_coro
497 if coro == "drain" then
498 local bytes, status = self._socket:drain_nb(self._input_chunk_size)
499 if status == "eof" then
500 coro = nil
501 end
502 elseif coro then
503 local retval = coro(self)
504 if retval ~= nil then
505 coro = nil -- can't consume more data
506 end
507 end
508 end
510 function request_pt:_consume_all_input()
511 while self._read_body_coro do
512 self._poll(socket_set)
513 self:_consume_some_input()
514 end
515 end
517 function request_pt:_error(status, explanation)
518 end
520 function request_pt:_read(...)
521 local line, status = self._socket:read_yield(...)
522 if line == nil then
523 self._faulty = true
524 error(status)
525 else
526 return line, status
527 end
528 end
530 function request_pt:_read_headers()
531 local remaining = self._header_size_limit
532 -- read and parse request line:
533 local target, proto
534 do
535 local line, status = self:_read(remaining-2, "\n")
536 if status == "maxlen" then
537 self:_error("414 Request-URI Too Long")
538 return false
539 elseif status == "eof" then
540 if line ~= "" then
541 self:_error("400 Bad Request", "Unexpected EOF in request-URI line")
542 end
543 return false
544 end
545 remaining = remaining - #line
546 self.method, target, proto =
547 line:match("^([^ \t\r]+)[ \t]+([^ \t\r]+)[ \t]*([^ \t\r]*)[ \t]*\r?\n$")
548 if not request.method then
549 self:_error("400 Bad Request", "Invalid request-URI line")
550 return false
551 elseif proto ~= "HTTP/1.1" then
552 self:_error("505 HTTP Version Not Supported")
553 return false
554 end
555 end
556 -- read and parse headers:
557 self._headers = {}
558 self._headers_value_nil = {}
559 while true do
560 local line, status = self:_read(remaining, "\n");
561 if status == "maxlen" then
562 self:_error("431 Request Header Fields Too Large")
563 return false
564 elseif status == "eof" then
565 self:_error("400 Bad Request", "Unexpected EOF in request headers")
566 return false
567 end
568 remaining = remaining - #line
569 if line == "\r\n" or line == "\n" then
570 break
571 end
572 local key, value = string.match(line, "^([^ \t\r]+):[ \t]*(.-)[ \t]*\r?\n$")
573 if not key then
574 self:_error("400 Bad Request", "Invalid header line")
575 return false
576 end
577 local lowerkey = key:lower()
578 local values = self._headers[lowerkey]
579 if values then
580 values[#values+1] = value
581 else
582 self._headers[lowerkey] = {value}
583 end
584 end
585 -- process "Connection: close" header if existent:
586 self._connection_close_requested = self.headers_flags["Connection"]["close"]
587 -- process "Content-Length" header if existent:
588 do
589 local values = self.headers_csv_table["Content-Length"]
590 if #values > 0 then
591 self._request_body_content_length = tonumber(values[1])
592 local proper_value = tostring(request_body_content_length)
593 for i, value in ipairs(values) do
594 value = string.match(value, "^0*(.*)")
595 if value ~= proper_value then
596 self:_error("400 Bad Request", "Content-Length header(s) invalid")
597 return false
598 end
599 end
600 if request_body_content_length > self._body_size_limit then
601 self:_error("413 Request Entity Too Large", "Announced request body size is too big")
602 return false
603 end
604 end
605 end
606 -- process "Transfer-Encoding" header if existent:
607 do
608 local flag = self.headers_flags["Transfer-Encoding"]["chunked"]
609 local list = self.headers_csv_table["Transfer-Encoding"]
610 if (flag and #list ~= 1) or (not flag and #list ~= 0) then
611 self:_error("400 Bad Request", "Unexpected Transfer-Encoding")
612 return false
613 end
614 end
615 -- process "Expect" header if existent:
616 for i, value in ipairs(self.headers_csv_table["Expect"]) do
617 if string.lower(value) ~= "100-continue" then
618 self:_error("417 Expectation Failed", "Unexpected Expect header")
619 return false
620 end
621 end
622 -- get mandatory Host header according to RFC 7230:
623 self.host = self.headers_value["Host"]
624 if not self.host then
625 self:_error("400 Bad Request", "No valid host header")
626 return false
627 end
628 -- parse request target:
629 self.path, self.query = string.match(target, "^/([^?]*)(.*)$")
630 if not self.path then
631 local host2
632 host2, self.path, self.query = string.match(target, "^[Hh][Tt][Tt][Pp]://([^/?]+)/?([^?]*)(.*)$")
633 if host2 then
634 if self.host ~= host2 then
635 self:_error("400 Bad Request", "No valid host header")
636 return false
637 end
638 elseif not (target == "*" and self.method == "OPTIONS") then
639 self:_error("400 Bad Request", "Invalid request target")
640 return false
641 end
642 end
643 -- parse GET params:
644 if self.query then
645 self.get_params_list = read_urlencoded_form(request.query)
646 self.get_params = get_first_values(self.get_params_list)
647 end
648 -- parse cookies:
649 self.cookies = {}
650 for i, line in ipairs(self.headers["Cookie"]) do
651 for rawkey, rawvalue in
652 string.gmatch(line, "([^=; ]*)=([^=; ]*)")
653 do
654 self.cookies[decode_uri(rawkey)] = decode_uri(rawvalue)
655 end
656 end
657 -- indicate success:
658 return true
659 end
661 function request_pt:_read_body()
662 local remaining = self._body_size_limit
663 if request.headers_flags["Transfer-Encoding"]["chunked"] then
664 while true do
665 local line, status = self:_read(32 + remaining, "\n")
666 if status == "maxlen" then
667 self:_error("400 Bad Request", "Request body size limit exceeded")
668 return false
669 elseif status == "eof" then
670 self:_error("400 Bad Request", "Encoding error or unexpected EOF while reading next chunk of request body")
671 return false
672 end
673 local zeros, lenstr = string.match(line, "^(0*)([1-9A-Fa-f]+[0-9A-Fa-f]*)\r?\n$")
674 local chunkext
675 if lenstr then
676 chunkext = ""
677 else
678 zeros, lenstr, chunkext = string.match(line, "^(0*)([1-9A-Fa-f]+[0-9A-Fa-f]*)([ \t;].-)\r?\n$")
679 end
680 if not lenstr or #lenstr > 13 then
681 self:_error("400 Bad Request", "Encoding error while reading chunk of request body")
682 return false
683 end
684 local len = tonumber("0x" .. lenstr)
685 remaining = remaining - (#zeros + #chunkext + len)
686 if remaining < 0 then
687 self:_error("400 Bad Request", "Request body size limit exceeded")
688 return false
689 end
690 if len == 0 then break end
691 if self:_read_body_bytes(len) == false then
692 return false
693 end
694 local term, status = self:_read(2, "\n")
695 if status == "eof" then
696 self:_error("400 Bad Request", "Unexpected EOF while reading next chunk of request body")
697 return false
698 end
699 if term ~= "\r\n" and term ~= "\n" then
700 self:_error("400 Bad Request", "Encoding error while reading chunk of request body")
701 return false
702 end
703 end
704 while true do
705 local line, status = self:_read(2 + remaining, "\n")
706 if status == "eof" then
707 self:_error("400 Bad Request", "Unexpected EOF while reading chunk of request body")
708 return false
709 end
710 if line == "\r\n" or line == "\n" then break end
711 remaining = remaining - #line
712 if remaining < 0 then
713 self:_error("413 Request Entity Too Large", "Request body size limit exceeded while reading trailer section of chunked request body")
714 return false
715 end
716 end
717 elseif request_body_content_length then
718 if self._read_body_bytes(request_body_content_length) == false then
719 return false
720 end
721 end
722 -- indicate success:
723 return true
724 end
726 function request_pt:_read_body_bytes(remaining, callback)
727 while remaining > 0 do
728 local limit
729 if remaining > self._input_chunk_size then
730 limit = self._input_chunk_size
731 else
732 limit = remaining
733 end
734 local chunk, status = self:_read(limit)
735 if status == "eof" then
736 self:_error("400 Bad Request", "Unexpected EOF while reading chunk of request body")
737 return false
738 end
739 remaining = remaining - limit
740 if self._body_streamer then
741 self._body_streamer(chunk)
742 end
743 end
744 return true
745 end
747 function request_pt:_assert_not_faulty()
748 assert(not self._faulty, "Tried to use faulty request handle")
749 end
751 function request_pt:_write_yield()
752 self:_consume_some_input()
753 self._poll(self._socket_set, self._socket_set)
754 end
756 function request_pt:_write(...)
757 assert(self._socket:write_call(self._write_yield_closure, ...))
758 end
760 function request_pt:_flush(...)
761 assert(self._socket:write_call(self._write_yield_closure, ...))
762 end
764 -- function creating a HTTP handler:
765 function generate_handler(handler, options)
766 -- swap arguments if necessary (for convenience):
767 if type(handler) ~= "function" and type(options) == "function" then
768 handler, options = options, handler
769 end
770 local request = setmetatable({}, request_mt)
771 request:_init(handler, options)
772 return request._handler_closure
773 end
775 return _M