Anubis / Xlib

Commit d17b1121d7da446b0d15cd91bd303693cdc010bf

Authored by Cédric RICARD
1 parent aa11d0af

Several great optimizations for HTTP server: 

- headers are now read using a buffer and no more bytes per bytes
- adding headers 'Date' and 'Last-Modified' allowing to improve efficiency of browser cache
- adding header 'Server' just for decoration.
Showing 1 changed file with 215 additions and 48 deletions   Show diff stats
Inline Side-by-side
calexium_lib/web/CXM_multihost_http_server.anubis
  Show/Hide comments   View file @d17b112
... ... @@ -592,7 +592,10 @@ type EncodingType:
592 592 www_url,
593 593 multipart_form_data.
594 594  
595   -
  595 +type BufferedConnection:
  596 + buffered_connection(Connection conn,
  597 + Var(ByteArray) buffer,
  598 + Var(Int32) read_pos).
596 599  
597 600  
598 601  
... ... @@ -687,12 +690,55 @@ define Result(Error,Word8)
687 690 " as dubious after "+(dead_line-*sttm)+" seconds. Total: "+
688 691 length(*list_of_dubious(dos))+"\n");
689 692 error(timeout(dead_line)).
690   -
  693 +
  694 +define String
  695 + pid
  696 + =
  697 + "[" + virtual_machine_id + "] ".
  698 +
  699 +define ReadResult
  700 + read
  701 + (
  702 + BufferedConnection connection,
  703 + Int32 size,
  704 + Int32 time_out
  705 + ) =
  706 + //println(pid + "read(" + size + ")");
  707 +
  708 + if *connection.read_pos < length(*connection.buffer) then
  709 + //println(pid + " reading from buffer (size = " + length(*connection.buffer) + ", pos = " + *connection.read_pos);
  710 + with result = extract(*connection.buffer, *connection.read_pos, *connection.read_pos + size),
  711 + size_read = length(result),
  712 + connection.read_pos <- *connection.read_pos + size_read;
  713 + if size > size_read
  714 + then
  715 + //println("Wanted " + size + ", read only " + size_read);
  716 + if read(connection, size - size_read, time_out) is
  717 + {
  718 + error then error,
  719 + timeout then ok(result),
  720 + ok(ba) then ok(result + ba)
  721 + }
  722 + else ok(result)
  723 + else
  724 + //if now > dead_line then record_dubious_connection(connection,dead_line,dos) else
  725 + if read(connection.conn, 16384, time_out) is // the connection is closed after 10 minutes of inactivity
  726 + {
  727 + error then println(pid + "read failed)"); error,
  728 + timeout then timeout,
  729 + ok(ba) then
  730 +// println(pid + "ba = " + length(ba));
  731 + connection.buffer <- ba;
  732 + connection.read_pos <- 0;
  733 + //println(pid + "rb = " + length(*read_buffer));
  734 + read(connection, size, time_out)
  735 + }.
  736 +
691 737  
692 738 define Result(Error,Word8)
693 739 read_one_byte
694 740 (
695   - Connection connection,
  741 + BufferedConnection connection,
696 742 Int32 dead_line,
697 743 DenialOfService dos
698 744 ) =
... ... @@ -705,16 +751,54 @@ define Result(Error,Word8)
705 751 ok(ba) then if nth(0,ba) is
706 752 {
707 753 failure then error(cannot_read_from_connection),
708   - success(c) then ok(c)
  754 + success(c) then
  755 +// println("-" + pid + "read [" + implode([c]) + "]\t");
  756 + ok(c)
709 757 }
710 758 }.
  759 +
  760 +
  761 +variable ByteArray read_buffer = constant_byte_array(0, 0).
  762 +variable Int32 read_offset = 0.
  763 +
  764 + define Result(Error,Word8)
  765 + read_one_byte
  766 + (
  767 + BufferedConnection connection,
  768 + Int32 dead_line,
  769 + DenialOfService dos
  770 + ) =
  771 + if *read_offset < length(*read_buffer) then
  772 + if nth(*read_offset, *read_buffer) is
  773 + {
  774 + failure then println(pid + "nth failed)"); error(cannot_read_from_connection),
  775 + success(c) then
  776 + println("-" + pid + "read [" + implode([c]) + "]\tat " + *read_offset);
  777 + read_offset <- *read_offset + 1;
  778 + ok(c)
  779 + }
  780 + else
  781 + //if now > dead_line then record_dubious_connection(connection,dead_line,dos) else
  782 + println(pid + "len = " + length(*read_buffer));
  783 + if read(connection, 16384, 600) is // the connection is closed after 10 minutes of inactivity
  784 + {
  785 + error then println(pid + "read failed)"); error(cannot_read_from_connection),
  786 + timeout then error(timeout(600)),
  787 + //record_dubious_connection(connection,dead_line,dos),
  788 + ok(ba) then
  789 + println(pid + "ba = " + length(ba));
  790 + read_buffer <- ba;
  791 + read_offset <- 0;
  792 + println(pid + "rb = " + length(*read_buffer));
  793 + read_one_byte(connection, dead_line, dos)
  794 + }.
711 795  
712 796  
713 797 define Result(Error,Word8)
714 798 next_char // reading a character (check the list first, and read on the connection
715 799 // only when the list is empty).
716 800 (
717   - Connection connection,
  801 + BufferedConnection connection,
718 802 Int32 dead_line,
719 803 DenialOfService dos
720 804 ) =
... ... @@ -742,13 +826,13 @@ define Result(Error,Word8)
742 826 define Result(Error,One)
743 827 read_and_ignore
744 828 (
745   - Connection connection, // to client
  829 + BufferedConnection connection, // to client
746 830 Int32 dead_line,
747 831 Int32 number_of_characters, // number of characters to read and ignore
748 832 DenialOfService dos
749 833 ) =
750 834 if number_of_characters =< 0 then ok(unique) else
751   - if next_char(connection,dead_line,dos) is
  835 + if next_char(connection, dead_line, dos) is
752 836 {
753 837 error(msg) then error(msg),
754 838 ok(c) then read_and_ignore(connection,dead_line,number_of_characters-1,dos)
... ... @@ -770,12 +854,12 @@ define Result(Error,One)
770 854 define Result(Error,String)
771 855 read_string
772 856 (
773   - Connection connection, // connection with the client
  857 + BufferedConnection connection, // connection with the client
774 858 Int32 dead_line,
775 859 List(Word8) so_far, // characters read so far (in reverse order)
776 860 DenialOfService dos
777 861 ) =
778   - if next_char(connection,dead_line,dos) is
  862 + if next_char(connection, dead_line,dos) is
779 863 {
780 864 error(msg) then error(msg),
781 865 ok(c) then
... ... @@ -1092,7 +1176,7 @@ define Bool
1092 1176 define Result(Error,One)
1093 1177 skip_http_blanks
1094 1178 (
1095   - Connection connection,
  1179 + BufferedConnection connection,
1096 1180 Int32 dead_line,
1097 1181 DenialOfService dos
1098 1182 ) =
... ... @@ -1148,7 +1232,7 @@ define Result(Error,One)
1148 1232 define Result(Error,One)
1149 1233 read_new_line
1150 1234 (
1151   - Connection connection,
  1235 + BufferedConnection connection,
1152 1236 Int32 dead_line,
1153 1237 DenialOfService dos
1154 1238 ) =
... ... @@ -1197,7 +1281,7 @@ define Result(Error,One)
1197 1281 define Result(Error,String)
1198 1282 read_word_aux
1199 1283 (
1200   - Connection connection,
  1284 + BufferedConnection connection,
1201 1285 Int32 dead_line,
1202 1286 List(Word8) so_far,
1203 1287 DenialOfService dos
... ... @@ -1215,7 +1299,7 @@ define Result(Error,String)
1215 1299 define Result(Error,String)
1216 1300 read_word
1217 1301 (
1218   - Connection connection,
  1302 + BufferedConnection connection,
1219 1303 Int32 dead_line,
1220 1304 DenialOfService dos
1221 1305 ) =
... ... @@ -1388,7 +1472,7 @@ define Result(Error,HTTP_RequestType)
1388 1472 define Result(Error,HTTP_RequestLine)
1389 1473 read_request_line
1390 1474 (
1391   - Connection connection,
  1475 + BufferedConnection connection,
1392 1476 Int32 dead_line,
1393 1477 DenialOfService dos
1394 1478 ) =
... ... @@ -1445,7 +1529,7 @@ define Bool
1445 1529 define Result(Error,String)
1446 1530 read_header_name
1447 1531 (
1448   - Connection connection,
  1532 + BufferedConnection connection,
1449 1533 Int32 dead_line,
1450 1534 List(Word8) so_far,
1451 1535 DenialOfService dos
... ... @@ -1462,7 +1546,7 @@ define Result(Error,String)
1462 1546 define Result(Error,One)
1463 1547 skip_colon
1464 1548 (
1465   - Connection connection,
  1549 + BufferedConnection connection,
1466 1550 Int32 dead_line,
1467 1551 DenialOfService dos
1468 1552 ) =
... ... @@ -1483,7 +1567,7 @@ define Result(Error,One)
1483 1567 define Result(Error,String)
1484 1568 read_header_value
1485 1569 (
1486   - Connection connection,
  1570 + BufferedConnection connection,
1487 1571 Int32 dead_line,
1488 1572 List(Word8) so_far,
1489 1573 DenialOfService dos
... ... @@ -1517,7 +1601,7 @@ define Result(Error,String)
1517 1601 define Result(Error,Maybe(HTTP_header))
1518 1602 read_header
1519 1603 (
1520   - Connection connection,
  1604 + BufferedConnection connection,
1521 1605 Int32 dead_line,
1522 1606 DenialOfService dos
1523 1607 ) =
... ... @@ -1555,7 +1639,7 @@ define Result(Error,Maybe(HTTP_header))
1555 1639 define Result(Error,List(HTTP_header))
1556 1640 read_http_headers
1557 1641 (
1558   - Connection connection,
  1642 + BufferedConnection connection,
1559 1643 Int32 dead_line,
1560 1644 DenialOfService dos
1561 1645 ) =
... ... @@ -1626,7 +1710,7 @@ define Result(Error,Int32)
1626 1710 define Result(Error,ByteArray)
1627 1711 read_http_body
1628 1712 (
1629   - Connection connection,
  1713 + BufferedConnection connection,
1630 1714 Int32 body_size,
1631 1715 ByteArray so_far, // when calling this function, 'so_far' is the empty byte array
1632 1716 Int32 retries // this function is called with retries = 10
... ... @@ -1850,10 +1934,68 @@ define Printable_tree
1850 1934 }.
1851 1935  
1852 1936  
  1937 +
  1938 +define String
  1939 + month_abrv
  1940 + (
  1941 + Date_and_Time d
  1942 + ) =
  1943 + if d.month = 1 then "Jan"
  1944 + else if d.month = 2 then "Feb"
  1945 + else if d.month = 3 then "Mar"
  1946 + else if d.month = 4 then "Apr"
  1947 + else if d.month = 5 then "May"
  1948 + else if d.month = 6 then "Jun"
  1949 + else if d.month = 7 then "Jul"
  1950 + else if d.month = 8 then "Aug"
  1951 + else if d.month = 9 then "Sep"
  1952 + else if d.month = 10 then "Oct"
  1953 + else if d.month = 11 then "Nov"
  1954 + else if d.month = 12 then "Dec"
  1955 + else
  1956 + println("Bad month value [" + d.month + "] on Date_and_Time");
  1957 + "XXX".
  1958 +
  1959 +define String
  1960 + weekday_abrv
  1961 + (
  1962 + Date_and_Time d
  1963 + ) =
  1964 + if d.week_day = 0 then "Sun"
  1965 + else if d.week_day = 1 then "Mon"
  1966 + else if d.week_day = 2 then "Tue"
  1967 + else if d.week_day = 3 then "Wed"
  1968 + else if d.week_day = 4 then "Thu"
  1969 + else if d.week_day = 5 then "Fri"
  1970 + else if d.week_day = 6 then "Sat"
  1971 + else
  1972 + println("Bad weekday value [" + d.week_day + "] on Date_and_Time");
  1973 + "XXX".
  1974 +
  1975 +/**
  1976 + * Format a date with the followin format : "Mon, 23 Jul 2007 11:33:43 GMT"
  1977 + * Currently, this function can't output a GMT time, but only local time.
  1978 + * So the final GMT is totally fake, but needed by protocol.
  1979 + */
  1980 +public define String
  1981 + format_http_date
  1982 + (
  1983 + Date_and_Time d
  1984 + ) =
  1985 + weekday_abrv(d) + ", " + zero_pad_n(2,day(d)) + " " + month_abrv(d) + " " + year(d)
  1986 + + " " + zero_pad_n(2,hour(d)) + ":" + zero_pad_n(2,minute(d)) + ":" + zero_pad_n(2,second(d)) + " GMT".
  1987 +
  1988 +/**
  1989 + * Same as previous format_http_date() function, but with seconds count from the UNIX epoch as input.
  1990 + */
  1991 +public define String
  1992 + format_http_date
  1993 + (
  1994 + Int32 date
  1995 + ) =
  1996 + format_http_date(convert_time(date)).
1853 1997  
1854   -
1855   -
1856   -
  1998 +
1857 1999 *** [5.5] Sending a file.
1858 2000  
1859 2001 We send 2 headers 'Content-Type' and 'Content-Length'.
... ... @@ -1864,12 +2006,20 @@ define List(HTTP_header)
1864 2006 String mime_type,
1865 2007 Int32 size,
1866 2008 String etag,
1867   - ) =
1868   - [
1869   - http_header("Content-Type",mime_type),
1870   - http_header("Etag", etag),
1871   - http_header("Content-Length",integer_to_string(size)),
1872   - ].
  2009 + Maybe(FileTimes) mb_ftimes,
  2010 + ) =
  2011 + with headers = (List(HTTP_header))
  2012 + [
  2013 + http_header("Content-Type",mime_type),
  2014 + http_header("Etag", etag),
  2015 + http_header("Content-Length",integer_to_string(size)),
  2016 + ],
  2017 + if mb_ftimes is
  2018 + {
  2019 + failure then headers,
  2020 + success(ftimes) then [http_header("Last-Modified", format_http_date(ftimes.last_modified)) . headers]
  2021 + }
  2022 + .
1873 2023  
1874 2024  
1875 2025  
... ... @@ -1905,11 +2055,12 @@ define String
1905 2055 compute_etag
1906 2056 (
1907 2057 String filename,
  2058 + Maybe(FileTimes) mb_ftimes,
1908 2059 Int32 size,
1909 2060 ) =
1910   - if get_file_times(filename) is
  2061 + if mb_ftimes is
1911 2062 {
1912   - failure then println("FAILED to get get_file_times() for file '" + filename + "'"); to_ascii(sha1((filename, size))),
  2063 + failure then println("Warning: no file times for '" + filename + "', etag won't be very accurate."); to_ascii(sha1((filename, size))),
1913 2064 success(ftimes) then to_ascii(sha1((filename, ftimes, size)))
1914 2065 }.
1915 2066  
... ... @@ -1943,12 +2094,13 @@ define One
1943 2094 ) =
1944 2095 action_before_send_file(unique);
1945 2096 with input_etag = http_header_value(input_headers, "If-None-Match"),
1946   - current_etag = compute_etag(full_path, size),
  2097 + mb_ftimes = get_file_times(full_path),
  2098 + current_etag = compute_etag(full_path, mb_ftimes, size),
1947 2099 if are_same_etag(input_etag, current_etag) is
1948 2100 {
1949 2101 false then
1950 2102 forget(reliable_write(connection,to_byte_array("HTTP/1.1 200 OK"+crlf)));
1951   - forget(reliable_write(connection,[format_headers(headers + headers_for_send_file(mime_type, size, current_etag)) , crlf]));
  2103 + forget(reliable_write(connection,[format_headers(headers + headers_for_send_file(mime_type, size, current_etag, mb_ftimes)) , crlf]));
1952 2104 send_file_body(desc,connection,file,size,0,filename),
1953 2105 true then
1954 2106 forget(reliable_write(connection,to_byte_array("HTTP/1.1 304 Not Modified"+crlf)));
... ... @@ -1986,6 +2138,7 @@ define One
1986 2138 Connection connection,
1987 2139 String uri,
1988 2140 List(HTTP_header) input_headers,
  2141 + List(HTTP_header) output_headers,
1989 2142 Maybe(String) mbauthorization,
1990 2143 One -> One action_before_send_file
1991 2144 ) =
... ... @@ -2004,7 +2157,7 @@ define One
2004 2157 send_file(desc,
2005 2158 connection,
2006 2159 input_headers,
2007   - [],
  2160 + output_headers,
2008 2161 size,
2009 2162 file(f),
2010 2163 uri,
... ... @@ -2034,7 +2187,7 @@ define One
2034 2187 send_file(desc,
2035 2188 connection,
2036 2189 input_headers,
2037   - [],
  2190 + output_headers,
2038 2191 size,
2039 2192 file(f),
2040 2193 uri,
... ... @@ -2059,6 +2212,14 @@ define One
2059 2212  
2060 2213 define List(HTTP_header)
2061 2214 standard_headers
  2215 + =
  2216 + [
  2217 + http_header("Date", format_http_date(now)),
  2218 + http_header("Server", "Anubis Embedded Server v" + major_version_number + "." + minor_version_number)
  2219 + ].
  2220 +
  2221 +define List(HTTP_header)
  2222 + standard_headers_for_html
2062 2223 (
2063 2224 Int32 answer_body_size,
2064 2225 String charset
... ... @@ -2100,7 +2261,8 @@ define One
2100 2261 if answer_headers_body is (additional_headers,answer_body) then
2101 2262 forget(reliable_write(connection,
2102 2263 [ "HTTP/1.1 200 OK", crlf,
2103   - format_headers(standard_headers(length(answer_body),charset(desc))),
  2264 + format_headers(standard_headers),
  2265 + format_headers(standard_headers_for_html(length(answer_body),charset(desc))),
2104 2266 format_headers(additional_headers),
2105 2267 crlf .
2106 2268 answer_body])))
... ... @@ -2108,6 +2270,7 @@ define One
2108 2270 connection,
2109 2271 uri,
2110 2272 headers,
  2273 + standard_headers,
2111 2274 if web_arg_value(all_web_args,"zauth") is
2112 2275 {
2113 2276 not_found then failure,
... ... @@ -2551,7 +2714,7 @@ define One
2551 2714 if answer_headers_body is (additional_headers,answer_body) then
2552 2715 forget(reliable_write(connection,
2553 2716 [ "HTTP/1.1 200 OK",crlf,
2554   - format_headers(standard_headers(length(answer_body),charset(desc))),
  2717 + format_headers(standard_headers_for_html(length(answer_body),charset(desc))),
2555 2718 format_headers(additional_headers),
2556 2719 crlf .
2557 2720 answer_body])))
... ... @@ -2790,14 +2953,14 @@ define One
2790 2953 http_https_handler
2791 2954 (
2792 2955 List(Web_Site_Description) sites,
2793   - Connection connection,
  2956 + BufferedConnection connection,
2794 2957 Bool is_https,
2795 2958 DenialOfService dos
2796 2959 ) =
2797 2960 with start_time = (Int32)now,
2798 2961 sttm <- start_time;
2799 2962 if dos is denial_of_service(mc_v,rld_v,hd_v,ad_v,ld_v,ra_v) then
2800   - if remote_IP_address_and_port(connection) is (ip_addr,port) then
  2963 + if remote_IP_address_and_port(connection.conn) is (ip_addr,port) then
2801 2964 if read_request_line(connection,start_time+*rld_v,dos) is
2802 2965 {
2803 2966 error(msg) then print(format(msg)),
... ... @@ -2817,8 +2980,8 @@ define One
2817 2980 {
2818 2981 error(msg) then log_journal_msg(desc,format(msg)),
2819 2982 ok(body) then
2820   - send_answer(host_name,desc,connection,request_line,headers,body,
2821   - make_generate_trust_ticket(dos))
  2983 + send_answer(host_name, desc,connection.conn, request_line, headers, body,
  2984 + make_generate_trust_ticket(dos))
2822 2985 }
2823 2986 }
2824 2987 }
... ... @@ -2837,11 +3000,13 @@ define Server -&gt; ((RWStream) -&gt; One)
2837 3000 List(Web_Site_Description) sites,
2838 3001 DenialOfService dos
2839 3002 ) =
2840   - (Server server) |-> (RWStream connection) |->
2841   - if remote_IP_address_and_port(connection) is (addr,_) then
  3003 + (Server server) |-> (RWStream conn) |->
  3004 + if remote_IP_address_and_port(conn) is (addr,_) then
2842 3005 if is_dubious_IP(addr,dos)
2843 3006 then print("Rejecting dubious IP address "+ip_addr_to_string(addr)+"\n")
2844   - else http_https_handler(sites,tcp(connection),false,dos).
  3007 + else
  3008 + with connection = buffered_connection(tcp(conn), var(constant_byte_array(0, 0)), var(0)),
  3009 + http_https_handler(sites, connection, false, dos).
2845 3010  
2846 3011 define Server -> (SSL_Connection -> One)
2847 3012 make_https_handler
... ... @@ -2849,8 +3014,9 @@ define Server -&gt; (SSL_Connection -&gt; One)
2849 3014 List(Web_Site_Description) sites,
2850 3015 DenialOfService dos
2851 3016 ) =
2852   - (Server server) |-> (SSL_Connection connection) |->
2853   - http_https_handler(sites,ssl(connection),true,dos).
  3017 + (Server server) |-> (SSL_Connection conn) |->
  3018 + with connection = buffered_connection(ssl(conn), var(constant_byte_array(0, 0)), var(0)),
  3019 + http_https_handler(sites, connection, true, dos).
2854 3020  
2855 3021  
2856 3022  
... ... @@ -3240,11 +3406,12 @@ define Server -&gt; ((RWStream) -&gt; One)
3240 3406 ) =
3241 3407 (Server server) |-> (RWStream conn) |->
3242 3408 with start_time = (Int32)now,
3243   - if read_request_line(tcp(conn),start_time+*request_line_delay(dos),dos) is
  3409 + connection = buffered_connection(tcp(conn), var(constant_byte_array(0, 0)), var(0)),
  3410 + if read_request_line(connection, start_time+*request_line_delay(dos), dos) is
3244 3411 {
3245 3412 error(msg) then print(format(msg)),
3246 3413 ok(request_line) then
3247   - if read_http_headers(tcp(conn),start_time+*headers_delay(dos),dos) is
  3414 + if read_http_headers(connection, start_time+*headers_delay(dos), dos) is
3248 3415 {
3249 3416 error(msg) then print(format(msg)),
3250 3417 ok(headers) then if get_host_header_value(headers) is
... ...