# HG changeset patch # User jbe # Date 1428239706 -7200 # Node ID 0ec070d6f5d9ba9d97c3b075c1b5c2493280281c # Parent 38e7bd13200ddb8fd050e4a4da24844e4a941152 Reverted experimental work on non-blocking I/O with file handles diff -r 38e7bd13200d -r 0ec070d6f5d9 moonbridge.c --- a/moonbridge.c Sun Apr 05 01:17:06 2015 +0200 +++ b/moonbridge.c Sun Apr 05 15:15:06 2015 +0200 @@ -936,20 +936,10 @@ return lua_gettop(L); } -/* Lua method for socket object to read from input stream using xread */ -static int moonbr_child_lua_xread_stream(lua_State *L) { +/* Lua method for socket object to read from input stream until terminator */ +static int moonbr_child_lua_readuntil_stream(lua_State *L) { lua_getfield(L, 1, "input"); - lua_getfield(L, -1, "xread"); - lua_insert(L, 1); - lua_replace(L, 2); - lua_call(L, lua_gettop(L) - 1, LUA_MULTRET); - return lua_gettop(L); -} - -/* Lua method for socket object to read from input stream using xread_nb */ -static int moonbr_child_lua_xread_nb_stream(lua_State *L) { - lua_getfield(L, 1, "input"); - lua_getfield(L, -1, "xread_nb"); + lua_getfield(L, -1, "readuntil"); lua_insert(L, 1); lua_replace(L, 2); lua_call(L, lua_gettop(L) - 1, LUA_MULTRET); @@ -976,16 +966,6 @@ return lua_gettop(L); } -/* Lua method for socket object to write to output stream using write_nb */ -static int moonbr_child_lua_write_nb_stream(lua_State *L) { - lua_getfield(L, 1, "output"); - lua_getfield(L, -1, "write_nb"); - lua_insert(L, 1); - lua_replace(L, 2); - lua_call(L, lua_gettop(L) - 1, LUA_MULTRET); - return lua_gettop(L); -} - /* Lua method for socket object to flush the output stream */ static int moonbr_child_lua_flush_stream(lua_State *L) { lua_getfield(L, 1, "output"); @@ -1062,15 +1042,13 @@ /* Methods of (bidirectional) socket object passed to handler */ static luaL_Reg moonbr_child_lua_socket_functions[] = { - {"read", moonbr_child_lua_read_stream}, - {"xread", moonbr_child_lua_xread_stream}, - {"xread_nb", moonbr_child_lua_xread_nb_stream}, - {"lines", moonbr_child_lua_lines_stream}, - {"write", moonbr_child_lua_write_stream}, - {"write_nb", moonbr_child_lua_write_nb_stream}, - {"flush", moonbr_child_lua_flush_stream}, - {"close", moonbr_child_lua_close_both_streams}, - {"cancel", moonbr_child_lua_cancel_both_streams}, + {"read", moonbr_child_lua_read_stream}, + {"readuntil", moonbr_child_lua_readuntil_stream}, + {"lines", moonbr_child_lua_lines_stream}, + {"write", moonbr_child_lua_write_stream}, + {"flush", moonbr_child_lua_flush_stream}, + {"close", moonbr_child_lua_close_both_streams}, + {"cancel", moonbr_child_lua_cancel_both_streams}, {NULL, NULL} }; @@ -2258,7 +2236,46 @@ return ptr; } -/* Helper function to convert a value at the given stack index to a natural number */ +/* New method for Lua file objects: read until terminator or length exceeded */ +static int moonbr_readuntil(lua_State *L) { + luaL_Stream *stream; + FILE *file; + const char *terminatorstr; + size_t terminatorlen; + luaL_Buffer buf; + lua_Integer maxlen; + char terminator; + int byte; + stream = luaL_checkudata(L, 1, LUA_FILEHANDLE); + terminatorstr = luaL_checklstring(L, 2, &terminatorlen); + luaL_argcheck(L, terminatorlen == 1, 2, "single byte expected"); + maxlen = luaL_optinteger(L, 3, 0); + if (!stream->closef) luaL_error(L, "attempt to use a closed file"); + file = stream->f; + luaL_buffinit(L, &buf); + if (!maxlen) maxlen = -1; + terminator = terminatorstr[0]; + while (maxlen > 0 ? maxlen-- : maxlen) { + byte = fgetc(file); + if (byte == EOF) { + if (ferror(file)) { + char errmsg[MOONBR_MAXSTRERRORLEN]; + strerror_r(errno, errmsg, MOONBR_MAXSTRERRORLEN); /* use thread-safe call in case child created threads */ + lua_pushnil(L); + lua_pushstring(L, errmsg); + return 2; + } else { + break; + } + } + luaL_addchar(&buf, byte); + if (byte == terminator) break; + } + luaL_pushresult(&buf); + if (!lua_rawlen(L, -1)) lua_pushnil(L); + return 1; +} + static int moonbr_lua_tonatural(lua_State *L, int idx) { int isnum; lua_Number n; @@ -2267,7 +2284,6 @@ else return -1; } -/* Helper function to convert a value at the given stack index to a timeval struct */ static int moonbr_lua_totimeval(lua_State *L, int idx, struct timeval *value) { int isnum; lua_Number n; @@ -2281,275 +2297,6 @@ } } -/* New function io.poll(...) */ -static int moonbr_io_poll(lua_State *L) { - int i; - luaL_Stream *stream; - int fd, isnum; - int nfds = 0; - fd_set readfds, writefds, exceptfds; - struct timeval timeout = {0, }; - int status; - FD_ZERO(&readfds); - FD_ZERO(&writefds); - FD_ZERO(&exceptfds); - if (!lua_isnoneornil(L, 1)) { - luaL_checktype(L, 1, LUA_TTABLE); - for (i=1; ; i++) { -#if LUA_VERSION_NUM >= 503 - if (lua_geti(L, 1, i) == LUA_TNIL) break; -#else - lua_pushinteger(L, i); - lua_gettable(L, 1); - if (lua_isnil(L, -1)) break; -#endif - stream = luaL_testudata(L, -1, LUA_FILEHANDLE); - if (stream) { - /* in case of file handle, check for pending data in buffer */ - FILE *file = stream->f; - int flags, chr; - flockfile(file); - fd = fileno_unlocked(file); - if (ferror(file) || feof(file)) goto moonbr_io_poll_pending_shortcut1; - flags = fcntl(fd, F_GETFL, 0); - if (flags == -1) goto moonbr_io_poll_pending_error; - if (fcntl(fd, F_SETFL, flags | O_NONBLOCK) == -1) goto moonbr_io_poll_pending_error; - chr = getc_unlocked(file); - if (chr == EOF) { - if (ferror(file) && errno == EAGAIN) clearerr_unlocked(file); - else goto moonbr_io_poll_pending_shortcut2; - } else { - if (ungetc(chr, file) == EOF) { - fcntl(fd, F_SETFL, flags); - goto moonbr_io_poll_pending_error; - } - goto moonbr_io_poll_pending_shortcut2; - } - if (fcntl(fd, F_SETFL, flags) == -1) goto moonbr_io_poll_pending_error; - funlockfile(file); - goto moonbr_io_poll_pending_cont; - moonbr_io_poll_pending_shortcut2: - if (fcntl(fd, F_SETFL, flags) == -1) goto moonbr_io_poll_pending_error; - moonbr_io_poll_pending_shortcut1: - funlockfile(file); - lua_pushboolean(L, 1); - return 1; - moonbr_io_poll_pending_error: - funlockfile(file); - { - char errmsg[MOONBR_MAXSTRERRORLEN]; - strerror_r(errno, errmsg, MOONBR_MAXSTRERRORLEN); /* use thread-safe call in case child created threads */ - return luaL_error(L, "Unexpected error while checking buffer of FILE handle: %s", errmsg); - } - } - fd = lua_tointegerx(L, -1, &isnum); - if (!isnum) luaL_error(L, "File descriptor is not an integer"); - moonbr_io_poll_pending_cont: - FD_SET(fd, &readfds); - if (fd+1 > nfds) nfds = fd+1; - lua_pop(L, 1); - } - } - if (!lua_isnoneornil(L, 2)) { - luaL_checktype(L, 2, LUA_TTABLE); - for (i=1; ; i++) { -#if LUA_VERSION_NUM >= 503 - if (lua_geti(L, 2, i) == LUA_TNIL) break; -#else - lua_pushinteger(L, i); - lua_gettable(L, 2); - if (lua_isnil(L, -1)) break; -#endif - stream = luaL_testudata(L, -1, LUA_FILEHANDLE); - if (stream) { - fd = fileno(stream->f); - } else { - fd = lua_tointegerx(L, -1, &isnum); - if (!isnum) luaL_error(L, "File descriptor is not an integer"); - } - FD_SET(fd, &writefds); - if (fd+1 > nfds) nfds = fd+1; - lua_pop(L, 1); - } - lua_pop(L, 2); - } - if (!lua_isnoneornil(L, 3)) { - luaL_argcheck(L, moonbr_lua_totimeval(L, 3, &timeout), 3, "not a valid timeout"); - } - status = select(nfds, &readfds, &writefds, &exceptfds, &timeout); - if (status == -1) { - if (errno == EINTR) { - lua_pushboolean(L, 0); - lua_pushliteral(L, "Signal received while polling file descriptors"); - return 2; - } else { - char errmsg[MOONBR_MAXSTRERRORLEN]; - strerror_r(errno, errmsg, MOONBR_MAXSTRERRORLEN); /* use thread-safe call in case child created threads */ - return luaL_error(L, "Unexpected error during \"select\" system call: %s", errmsg); - } - } else if (status == 0) { - lua_pushboolean(L, 0); - lua_pushliteral(L, "Timeout while polling file descriptors"); - return 2; - } else { - lua_pushboolean(L, 1); - return 1; - } -} - -/* New methods for Lua file objects: read until terminator or length exceeded (blocking and non-blocking) */ -static int moonbr_io_xread_impl(lua_State *L, int nonblock) { - luaL_Stream *stream; - lua_Integer maxlen; - const char *terminatorstr; - size_t terminatorlen; - FILE *file; - int chr, terminator; - luaL_Buffer buf; - int wouldblock = 0; - stream = luaL_checkudata(L, 1, LUA_FILEHANDLE); - maxlen = luaL_optinteger(L, 2, 0); - terminatorstr = luaL_optlstring(L, 3, "", &terminatorlen); - luaL_argcheck(L, terminatorlen <= 1, 3, "single byte expected"); - if (!stream->closef) luaL_error(L, "attempt to use a closed file"); - file = stream->f; - if (ferror(file)) { - lua_pushnil(L); - lua_pushliteral(L, "Previous error condition on file handle"); - return 2; - } - if (!maxlen) maxlen = -1; - terminator = terminatorlen ? terminatorstr[0] : -1; - luaL_buffinit(L, &buf); - while (maxlen > 0 ? maxlen-- : maxlen) { - chr = getc(file); - if (chr == EOF) { - if (ferror(file)) { - if (nonblock && errno == EAGAIN) { - clearerr(file); - wouldblock = 1; - } else { - char errmsg[MOONBR_MAXSTRERRORLEN]; - strerror_r(errno, errmsg, MOONBR_MAXSTRERRORLEN); /* use thread-safe call in case child created threads */ - lua_pushnil(L); - lua_pushstring(L, errmsg); - return 2; - } - } - break; - } - luaL_addchar(&buf, chr); - if (chr == terminator) break; - } - luaL_pushresult(&buf); - if (wouldblock || lua_rawlen(L, -1)) { - return 1; - } else { - lua_pushboolean(L, 0); - lua_pushliteral(L, "End of file"); - return 2; - } -} -static int moonbr_io_xread(lua_State *L) { - return moonbr_io_xread_impl(L, 0); -} -static int moonbr_io_xread_nb_impl(lua_State *L) { - return moonbr_io_xread_impl(L, 1); -} -static int moonbr_io_xread_nb(lua_State *L) { - luaL_Stream *stream; - int fd, flags; - stream = luaL_checkudata(L, 1, LUA_FILEHANDLE); - if (!stream->closef) luaL_error(L, "attempt to use a closed file"); - lua_pushcfunction(L, moonbr_io_xread_nb_impl); - lua_insert(L, 1); - lua_settop(L, 4); - fd = fileno(stream->f); - flags = fcntl(fd, F_GETFL, 0); - if (flags == -1) goto moonbr_io_xread_nb_error; - if (fcntl(fd, F_SETFL, flags | O_NONBLOCK) == -1) goto moonbr_io_xread_nb_error; - if (lua_pcall(L, 3, LUA_MULTRET, 0)) { - fcntl(fd, F_SETFL, flags); - return lua_error(L); - } - if (fcntl(fd, F_SETFL, flags) == -1) goto moonbr_io_xread_nb_error; - return lua_gettop(L); - moonbr_io_xread_nb_error: - { - char errmsg[MOONBR_MAXSTRERRORLEN]; - strerror_r(errno, errmsg, MOONBR_MAXSTRERRORLEN); /* use thread-safe call in case child created threads */ - return luaL_error(L, "Unexpected error in xread_nb method: %s", errmsg); - } -} - -/* New methods for Lua file objects: non-blocking writing */ -static int moonbr_io_write_nb_impl(lua_State *L) { - int fd, argc, i; - const char *str; - size_t strlen; - ssize_t written; - luaL_Buffer buf; - fd = lua_tointeger(L, 1); - argc = lua_gettop(L) - 1; - for (i=0; iclosef) luaL_error(L, "attempt to use a closed file"); - file = stream->f; - if (fflush(file)) goto moonbr_io_write_nb_error; - lua_pushcfunction(L, moonbr_io_write_nb_impl); - lua_insert(L, 2); - fd = fileno(file); - lua_pushinteger(L, fd); - lua_insert(L, 3); - flags = fcntl(fd, F_GETFL, 0); - if (flags == -1) goto moonbr_io_write_nb_error; - if (fcntl(fd, F_SETFL, flags | O_NONBLOCK) == -1) goto moonbr_io_write_nb_error; - if (lua_pcall(L, lua_gettop(L) - 2, LUA_MULTRET, 0)) { - fcntl(fd, F_SETFL, flags); - return lua_error(L); - } - if (fcntl(fd, F_SETFL, flags) == -1) goto moonbr_io_write_nb_error; - return lua_gettop(L) - 1; - moonbr_io_write_nb_error: - { - char errmsg[MOONBR_MAXSTRERRORLEN]; - strerror_r(errno, errmsg, MOONBR_MAXSTRERRORLEN); /* use thread-safe call in case child created threads */ - return luaL_error(L, "Unexpected error in write_nb method: %s", errmsg); - } -} - -/* New global function timeout(...) */ static int moonbr_timeout(lua_State *L) { struct itimerval oldval; if (lua_isnoneornil(L, 1) && lua_isnoneornil(L, 2)) { @@ -3017,21 +2764,13 @@ #ifdef MOONBR_LUA_CPATH moonbr_modify_path(L, "cpath", MOONBR_LUA_CPATH); #endif - lua_getglobal(L, "io"); - lua_pushcfunction(L, moonbr_io_poll); - lua_setfield(L, -2, "poll"); - lua_pop(L, 1); if (luaL_newmetatable(L, LUA_FILEHANDLE)) { moonbr_log(LOG_CRIT, "Lua metatable LUA_FILEHANDLE does not exist"); moonbr_terminate_error(); } lua_getfield(L, -1, "__index"); - lua_pushcfunction(L, moonbr_io_xread); - lua_setfield(L, -2, "xread"); - lua_pushcfunction(L, moonbr_io_xread_nb); - lua_setfield(L, -2, "xread_nb"); - lua_pushcfunction(L, moonbr_io_write_nb); - lua_setfield(L, -2, "write_nb"); + lua_pushcfunction(L, moonbr_readuntil); + lua_setfield(L, -2, "readuntil"); lua_pop(L, 2); lua_pushcfunction(L, moonbr_timeout); lua_setglobal(L, "timeout"); diff -r 38e7bd13200d -r 0ec070d6f5d9 moonbridge_http.lua --- a/moonbridge_http.lua Sun Apr 05 01:17:06 2015 +0200 +++ b/moonbridge_http.lua Sun Apr 05 15:15:06 2015 +0200 @@ -180,16 +180,13 @@ end local input_chunk_size = options.maximum_input_chunk_size or options.chunk_size or 16384 local output_chunk_size = options.minimum_output_chunk_size or options.chunk_size or 1024 - local request_idle_timeout, request_header_timeout, response_timeout - if options.request_idle_timeout ~= nil then - request_idle_timeout = options.request_idle_timeout or 0 + local request_header_timeout, response_timeout + if options.request_header_timeout ~= nil then + request_header_timeout = options.request_header_timeout + elseif options.timeout ~= nil then + request_header_timeout = options.timeout or 0 else - request_idle_timeout = 330 - end - if options.request_header_timeout ~= nil then - request_header_timeout = options.request_header_timeout or 0 - else - request_header_timeout = 30 + request_header_timeout = 360 end if options.timeout ~= nil then response_timeout = options.timeout or 0 @@ -200,6 +197,8 @@ return function(socket) local survive = true -- set to false if process shall be terminated later repeat + -- (re)set timeout: + timeout(request_header_timeout or 0) -- process named arguments "request_header_size_limit" and "request_body_size_limit": local remaining_header_size_limit = options.request_header_size_limit or 1024*1024 local remaining_body_size_limit = options.request_body_size_limit or 64*1024*1024 @@ -833,7 +832,7 @@ end if request.headers_flags["Transfer-Encoding"]["chunked"] then while true do - local line = socket:xread(32 + remaining_body_size_limit, "\n") + local line = socket:readuntil("\n", 32 + remaining_body_size_limit) if not line then request_error(true, "400 Bad Request", "Unexpected EOF while reading next chunk of request body") end @@ -854,13 +853,13 @@ end if len == 0 then break end read_body_bytes(len, callback) - local term = socket:xread(2, "\n") + local term = socket:readuntil("\n", 2) if term ~= "\r\n" and term ~= "\n" then request_error(true, "400 Bad Request", "Encoding error while reading chunk of request body") end end while true do - local line = socket:xread(2 + remaining_body_size_limit, "\n") + local line = socket:readuntil("\n", 2 + remaining_body_size_limit) if line == "\r\n" or line == "\n" then break end remaining_body_size_limit = remaining_body_size_limit - #line if remaining_body_size_limit < 0 then @@ -894,14 +893,8 @@ end end }) - -- wait for input: - if not io.poll({socket.input}, nil, request_idle_timeout) then - return request_error(false, "408 Request Timeout") - end - -- set timeout for request header processing: - timeout(request_header_timeout) -- read and parse request line: - local line = socket:xread(remaining_header_size_limit, "\n") + local line = socket:readuntil("\n", remaining_header_size_limit) if not line then return survive end remaining_header_size_limit = remaining_header_size_limit - #line if remaining_header_size_limit == 0 then @@ -917,7 +910,7 @@ end -- read and parse headers: while true do - local line = socket:xread(remaining_header_size_limit, "\n"); + local line = socket:readuntil("\n", remaining_header_size_limit); remaining_header_size_limit = remaining_header_size_limit - #line if not line then return request_error(false, "400 Bad Request") @@ -998,14 +991,12 @@ request.cookies[decode_uri(rawkey)] = decode_uri(rawvalue) end end - -- (re)set timeout for handler: + -- (re)set timeout: timeout(response_timeout or 0) -- call underlying handler and remember boolean result: if handler(request) ~= true then survive = false end -- finish request (unless already done by underlying handler): request:finish() - -- stop timeout timer: - timeout(0) until connection_close_responded return survive end diff -r 38e7bd13200d -r 0ec070d6f5d9 reference.txt --- a/reference.txt Sun Apr 05 01:17:06 2015 +0200 +++ b/reference.txt Sun Apr 05 15:15:06 2015 +0200 @@ -45,28 +45,6 @@ -Function io.poll(read_fds, write_fds, timeout) ----------------------------------------------- - -This function allows to wait for file descriptors to become ready for reading -or writing. It accepts the following arguments: - -1. Table of file descriptors to wait for reading (optional, may be nil) -2. Table of file descriptors to wait for writing (optional, may be nil) -3. Timeout in seconds (optional, may be nil or zero to disable timeout) - -Alternatively to file descriptors, the tables may contain file handles, in -which case the file descriptor is automatically extracted from the file handle -and, in case of waiting for reading, the file handle's buffer is additionally -tested for pending data. - -Please note that support for non-blocking I/O operations is limited if you use -ordinary file handles (as Moonbridge does). It is possible, however, to wait -until the first byte to read is available at a file handle. For more -information, see socket.input:pending(). - - - Socket object passed to "connect" handler ----------------------------------------- @@ -112,34 +90,6 @@ Supports the same methods as io.open()'s return values. -### socket.input:xread(maxlen, terminator) - -Reads as many bytes until either the maximum length is reached (first argument) -or a terminating character (second argument as string with a length of 1) is -encountered. In the latter case, the terminating character will be included in -the result. - -On EOF, false (as first result value) and an error message (as second result -value) are returned. In case of an I/O error, nil and an error message are -returned. - -This method is also available for any other Lua file handle. - - -### socket.input:xread_nb(maxlen, terminator) - -Same as socket.input:xread(maxlen, terminator), but non-blocking. If no more -data can be read due to blocking, this method returns a string containing the -data which has already been read. If no data could be read due to blocking, -then an empty string is returned. - -On EOF, false (as first result value) and an error message (as second result -value) are returned. In case of an I/O error, nil and an error message are -returned. - -This method is also available for any other Lua file handle. - - ### socket.interval Set to the name of an interval timer if the "connect" handler was called due to @@ -174,40 +124,23 @@ Supports the same methods as io.open()'s return values. -### socket.output:close() - -Performs a half-close (i.e. sends a TCP FIN package in case of a TCP socket). - -Note: In order to shut down a TCP connection properly, it may be necessary to -read any pending data from socket.input before closing the socket completely -(e.g. with socket:close() or by returning from the connect handler). If there -is still incoming data, a TCP RST packet might be sent which can cause loss of -transmitted data. - - -### socket.output:write_nb(...) - -Non-blocking write. This function attempts to write as many bytes as possible, -returning a string containing all data that could not be written due to -blocking (potentially concatenating some or all remaining arguments to create -that string). - -In case of an I/O error, nil (as first result value) and an error message (as -second result value) are returned. - -Note: Using this method will automatically flush any unflushed data written -through previous socket.output:write(...) calls. Thus, if any data was -previously written in a blocking fashion and has not been flushed, then the -socket.output:write_nb(...) call may block. - -This method is also available for any other Lua file handle. - - ### socket:read(...) Alias for socket.input:read() +### socket:readuntil(terminator, maxlen) + +Reads as many bytes until a byte equal to the terminator value occurs. An +optional maximum length may be specified. The terminating byte is included in +the return value (unless the maximum length would be exceeded). On EOF, nil is +returned. In case of an I/O error, nil (as first result value) plus an error +message (as second result value) is returned. + +This method is also available as :readuntil(...) for any other Lua file handle +(including socket.input). + + ### socket.remote_ip4 Remote IPv4 address used for the connection. Encoded as 4 raw bytes in form of @@ -230,21 +163,6 @@ Alias for socket.output:write(...) -### socket:write_nb(...) - -Alias for socket.output:write(...) - - -### socket:xread(maxlen, terminator) - -Alias for socket.input:xread(...) - - -### socket:xread_nb(maxlen, terminator) - -Alias for socket.input:xread_nb(...) - - HTTP module -----------