# HG changeset patch # User jbe # Date 1428117005 -7200 # Node ID 3d1f23f1dbc6711909e8edcb4b80348cc076a1f5 # Parent 8090fe97518ad9702ad98298244df803a2d9047c Work on non-blocking I/O; Improved efficiency of :readuntil(...) diff -r 8090fe97518a -r 3d1f23f1dbc6 moonbridge.c --- a/moonbridge.c Sat Apr 04 03:22:06 2015 +0200 +++ b/moonbridge.c Sat Apr 04 05:10:05 2015 +0200 @@ -2202,27 +2202,6 @@ return 1; } -static int moonbr_lua_tonatural(lua_State *L, int idx) { - int isnum; - lua_Number n; - n = lua_tonumberx(L, idx, &isnum); - if (isnum && n>=0 && n=0 && n<=100000000) { - value->tv_sec = n; - value->tv_usec = 1e6 * (n - value->tv_sec); - return 1; - } else { - return 0; - } -} - /* Memory allocator that allows limiting memory consumption */ static void *moonbr_alloc (void *ud, void *ptr, size_t osize, size_t nsize) { (void)ud; /* not used */ @@ -2257,11 +2236,33 @@ return ptr; } +/* Helper function to convert a value at the given stack index to a natural number */ +static int moonbr_lua_tonatural(lua_State *L, int idx) { + int isnum; + lua_Number n; + n = lua_tonumberx(L, idx, &isnum); + if (isnum && n>=0 && n=0 && n<=100000000) { + value->tv_sec = n; + value->tv_usec = 1e6 * (n - value->tv_sec); + return 1; + } else { + return 0; + } +} + /* New function io.poll(...) */ static int moonbr_io_poll(lua_State *L) { - int idx_tbl = 1; - int idx_timeout = 0; int i; + luaL_Stream *stream; int fd, isnum; int nfds = 0; fd_set readfds, writefds, exceptfds; @@ -2270,82 +2271,73 @@ FD_ZERO(&readfds); FD_ZERO(&writefds); FD_ZERO(&exceptfds); - if (lua_type(L, 1) == LUA_TNUMBER) { - idx_timeout = 1; - idx_tbl = 2; - } - luaL_checktype(L, idx_tbl, LUA_TTABLE); - for (i=1; ; i++) { -#if LUA_VERSION_NUM >= 503 - if (lua_geti(L, idx_tbl, i) == LUA_TNIL) break; -#else - lua_pushinteger(L, i); - lua_gettable(L, idx_tbl); - if (lua_isnil(L, -1)) break; -#endif - fd = lua_tointegerx(L, -1, &isnum); - if (!isnum) luaL_error(L, "File descriptor is not an integer"); - FD_SET(fd, &readfds); - if (fd+1 > nfds) nfds = fd+1; - lua_pop(L, 1); - } - if (!idx_timeout && lua_type(L, 2) == LUA_TNUMBER) { - idx_timeout = 2; - idx_tbl = 3; - } else { - idx_tbl = 2; - } - if (!lua_isnoneornil(L, idx_tbl)) { - luaL_checktype(L, idx_tbl, LUA_TTABLE); + if (!lua_isnoneornil(L, 1)) { + luaL_checktype(L, 1, LUA_TTABLE); for (i=1; ; i++) { #if LUA_VERSION_NUM >= 503 - if (lua_geti(L, idx_tbl, i) == LUA_TNIL) break; + if (lua_geti(L, 1, i) == LUA_TNIL) break; #else lua_pushinteger(L, i); - lua_gettable(L, idx_tbl); + lua_gettable(L, 1); if (lua_isnil(L, -1)) break; #endif - fd = lua_tointegerx(L, -1, &isnum); - if (!isnum) luaL_error(L, "File descriptor is not an integer"); + stream = luaL_testudata(L, -1, LUA_FILEHANDLE); + if (stream) { + fd = fileno(stream->f); + } else { + fd = lua_tointegerx(L, -1, &isnum); + if (!isnum) luaL_error(L, "File descriptor is not an integer"); + } + FD_SET(fd, &readfds); + if (fd+1 > nfds) nfds = fd+1; + lua_pop(L, 1); + } + } + if (!lua_isnoneornil(L, 2)) { + luaL_checktype(L, 2, LUA_TTABLE); + for (i=1; ; i++) { +#if LUA_VERSION_NUM >= 503 + if (lua_geti(L, 2, i) == LUA_TNIL) break; +#else + lua_pushinteger(L, i); + lua_gettable(L, 2); + if (lua_isnil(L, -1)) break; +#endif + stream = luaL_testudata(L, -1, LUA_FILEHANDLE); + if (stream) { + fd = fileno(stream->f); + } else { + fd = lua_tointegerx(L, -1, &isnum); + if (!isnum) luaL_error(L, "File descriptor is not an integer"); + } FD_SET(fd, &writefds); if (fd+1 > nfds) nfds = fd+1; lua_pop(L, 1); } lua_pop(L, 2); - if (!idx_timeout && !lua_isnoneornil(L, 3)) idx_timeout = 3; } - if (idx_timeout) { - luaL_argcheck( - L, - moonbr_lua_totimeval(L, idx_timeout, &timeout), - idx_timeout, - "not a valid timeout" - ); + if (!lua_isnoneornil(L, 3)) { + luaL_argcheck(L, moonbr_lua_totimeval(L, 3, &timeout), 3, "not a valid timeout"); } status = select(nfds, &readfds, &writefds, &exceptfds, &timeout); if (status == -1) { if (errno == EINTR) { lua_pushboolean(L, 0); + lua_pushliteral(L, "Signal received while polling file descriptors"); + return 2; } else { char errmsg[MOONBR_MAXSTRERRORLEN]; strerror_r(errno, errmsg, MOONBR_MAXSTRERRORLEN); /* use thread-safe call in case child created threads */ - luaL_error(L, "Unexpected error during \"select\" system call: %s", errmsg); + return luaL_error(L, "Unexpected error during \"select\" system call: %s", errmsg); } } else if (status == 0) { lua_pushboolean(L, 0); + lua_pushliteral(L, "Timeout while polling file descriptors"); + return 2; } else { lua_pushboolean(L, 1); + return 1; } - return 1; -} - -/* New method for Lua file objects: get file descriptor */ -static int moonbr_io_getfd(lua_State *L) { - luaL_Stream *stream; - stream = luaL_checkudata(L, 1, LUA_FILEHANDLE); - if (!stream->closef) luaL_error(L, "attempt to use a closed file"); - lua_pushinteger(L, fileno(stream->f)); - return 1; } /* New method for Lua file objects: check if non-blocking reading is possible */ @@ -2389,32 +2381,6 @@ return 0; } -/* New method for Lua file objects: set blocking or non-blocking I/O */ -static int moonbr_io_setblocking(lua_State *L) { - luaL_Stream *stream; - int blocking; - int fd, flags; - stream = luaL_checkudata(L, 1, LUA_FILEHANDLE); - luaL_checktype(L, 2, LUA_TBOOLEAN); - blocking = lua_toboolean(L, 2); - if (!stream->closef) luaL_error(L, "attempt to use a closed file"); - fd = fileno_unlocked(stream->f); - flags = fcntl(fd, F_GETFL, 0); - if (flags == -1) goto moonbr_io_setblocking_error; - if (blocking) flags &= ~ O_NONBLOCK; - else flags |= O_NONBLOCK; - if (fcntl(fd, F_SETFL, flags) == -1) goto moonbr_io_setblocking_error; - lua_pushboolean(L, 1); - return 1; - moonbr_io_setblocking_error: - { - char errmsg[MOONBR_MAXSTRERRORLEN]; - strerror_r(errno, errmsg, MOONBR_MAXSTRERRORLEN); /* use thread-safe call in case child created threads */ - luaL_error(L, "Unexpected error in method \"setblocking\" of FILE handle: %s", errmsg); - } - return 0; -} - /* New method for Lua file objects: read until terminator or length exceeded */ static int moonbr_io_readuntil(lua_State *L) { luaL_Stream *stream; @@ -2434,12 +2400,14 @@ luaL_buffinit(L, &buf); if (!maxlen) maxlen = -1; terminator = terminatorstr[0]; + flockfile(file); while (maxlen > 0 ? maxlen-- : maxlen) { - byte = fgetc(file); + byte = getc_unlocked(file); if (byte == EOF) { if (ferror(file)) { char errmsg[MOONBR_MAXSTRERRORLEN]; strerror_r(errno, errmsg, MOONBR_MAXSTRERRORLEN); /* use thread-safe call in case child created threads */ + funlockfile(file); lua_pushnil(L); lua_pushstring(L, errmsg); return 2; @@ -2450,6 +2418,7 @@ luaL_addchar(&buf, byte); if (byte == terminator) break; } + funlockfile(file); luaL_pushresult(&buf); if (!lua_rawlen(L, -1)) lua_pushnil(L); return 1; @@ -2931,12 +2900,8 @@ moonbr_terminate_error(); } lua_getfield(L, -1, "__index"); - lua_pushcfunction(L, moonbr_io_getfd); - lua_setfield(L, -2, "getfd"); - lua_pushcfunction(L, moonbr_io_pending); + eua_pushcfunction(L, moonbr_io_pending); lua_setfield(L, -2, "pending"); - lua_pushcfunction(L, moonbr_io_setblocking); - lua_setfield(L, -2, "setblocking"); lua_pushcfunction(L, moonbr_io_readuntil); lua_setfield(L, -2, "readuntil"); lua_pop(L, 2); diff -r 8090fe97518a -r 3d1f23f1dbc6 moonbridge_http.lua --- a/moonbridge_http.lua Sat Apr 04 03:22:06 2015 +0200 +++ b/moonbridge_http.lua Sat Apr 04 05:10:05 2015 +0200 @@ -180,13 +180,16 @@ end local input_chunk_size = options.maximum_input_chunk_size or options.chunk_size or 16384 local output_chunk_size = options.minimum_output_chunk_size or options.chunk_size or 1024 - local request_header_timeout, response_timeout + local request_idle_timeout, request_header_timeout, response_timeout + if options.request_idle_timeout ~= nil then + request_idle_timeout = options.request_idle_timeout or 0 + else + request_idle_timeout = 330 + end if options.request_header_timeout ~= nil then - request_header_timeout = options.request_header_timeout - elseif options.timeout ~= nil then - request_header_timeout = options.timeout or 0 + request_header_timeout = options.request_header_timeout or 0 else - request_header_timeout = 360 + request_header_timeout = 30 end if options.timeout ~= nil then response_timeout = options.timeout or 0 @@ -197,8 +200,6 @@ return function(socket) local survive = true -- set to false if process shall be terminated later repeat - -- (re)set timeout: - timeout(request_header_timeout or 0) -- process named arguments "request_header_size_limit" and "request_body_size_limit": local remaining_header_size_limit = options.request_header_size_limit or 1024*1024 local remaining_body_size_limit = options.request_body_size_limit or 64*1024*1024 @@ -893,6 +894,14 @@ end end }) + -- wait for input: + if not socket.input:pending() then + if not io.poll({socket.input}, nil, request_idle_timeout) then + return request_error(false, "408 Request Timeout") + end + end + -- set timeout for request header processing: + timeout(request_header_timeout) -- read and parse request line: local line = socket:readuntil("\n", remaining_header_size_limit) if not line then return survive end @@ -991,12 +1000,14 @@ request.cookies[decode_uri(rawkey)] = decode_uri(rawvalue) end end - -- (re)set timeout: + -- (re)set timeout for handler: timeout(response_timeout or 0) -- call underlying handler and remember boolean result: if handler(request) ~= true then survive = false end -- finish request (unless already done by underlying handler): request:finish() + -- stop timeout timer: + timeout(0) until connection_close_responded return survive end diff -r 8090fe97518a -r 3d1f23f1dbc6 reference.txt --- a/reference.txt Sat Apr 04 03:22:06 2015 +0200 +++ b/reference.txt Sat Apr 04 05:10:05 2015 +0200 @@ -45,6 +45,26 @@ +Function io.poll(read_fds, write_fds, timeout) +---------------------------------------------- + +This function allows to wait for file descriptors to become ready for reading +or writing. It accepts the following arguments: + +1. Table of file descriptors to wait for reading (optional, may be nil) +2. Table of file descriptors to wait for writing (optional, may be nil) +3. Timeout in seconds (optional, may be nil or zero to disable timeout) + +Alternatively to file descriptors, the tables may contain file handles (in +which case the file descriptor is automatically extracted). + +Please note that support for non-blocking I/O operations is limited if you use +ordinary file handles (as Moonbridge does). It is possible, however, to wait +until the first byte to read is available at a file handle. For more +information, see socket.input:pending(). + + + Socket object passed to "connect" handler ----------------------------------------- @@ -90,6 +110,38 @@ Supports the same methods as io.open()'s return values. +### socket.input:pending() + +Returns true if there is at least one byte to read. In case of I/O errors, true +is returned as well (to avoid lockup and cause error on subsequent read). + +Note: Subsequent calls of socket.input:read(...) or socket.input:readuntil(...) +may still block when attempting to read more than one byte. To avoid hanging +processes, the timeout(...) function may be used as a watchdog that will +terminate the process in case of unexpected delay. Because file handles are +buffered, data may be still pending even if the underlaying file descriptor +does not have any more data to read. Thus, all file handles passed to +io.poll(...) to wait for reading should be tested for pending data first. + +This method is also available as :pending() for any other Lua file handle. + + +### socket.input:readuntil(terminator, maxlen) + +Reads as many bytes until a byte equal to the terminator value occurs. An +optional maximum length may be specified. The terminating byte is included in +the return value (unless the maximum length would be exceeded). On EOF, nil is +returned. In case of an I/O error, nil (as first result value) plus an error +message (as second result value) is returned. + +Note: This function may provide a significant speedup compared to byte-wise +reading using socket.input:read(1) in a loop. However, this function will block +when no data is available. The timeout(...) function may be used as a watchdog +that will terminate the process in case of unexpected delay. + +This method is also available as :readuntil(...) for any other Lua file handle. + + ### socket.interval Set to the name of an interval timer if the "connect" handler was called due to @@ -124,6 +176,17 @@ Supports the same methods as io.open()'s return values. +### socket.output:close() + +Performs a half-close (i.e. sends a TCP FIN package in case of a TCP socket). + +Note: In order to shut down a TCP connection properly, it may be necessary to +read any pending data from socket.input before closing the socket completely +(e.g. with socket:close() or by returning from the connect handler). If there +is still incoming data, a TCP RST packet might be sent which can cause loss of +transmitted data. + + ### socket:read(...) Alias for socket.input:read() @@ -131,14 +194,7 @@ ### socket:readuntil(terminator, maxlen) -Reads as many bytes until a byte equal to the terminator value occurs. An -optional maximum length may be specified. The terminating byte is included in -the return value (unless the maximum length would be exceeded). On EOF, nil is -returned. In case of an I/O error, nil (as first result value) plus an error -message (as second result value) is returned. - -This method is also available as :readuntil(...) for any other Lua file handle -(including socket.input). +Alias for socket.input:readuntil(terminator, maxlen) ### socket.remote_ip4