Several great optimizations for HTTP server:

- headers are now read using a buffer and no more bytes per bytes - adding headers 'Date' and 'Last-Modified' allowing to improve efficiency of browser cache - adding header 'Server' just for decoration.

Several great optimizations for HTTP server:
- headers are now read using a buffer and no more bytes per bytes - adding headers 'Date' and 'Last-Modified' allowing to improve efficiency of browser cache - adding header 'Server' just for decoration.
Cédric RICARD
1 parent aa11d0af
Showing 1 changed file with 215 additions and 48 deletions Show diff stats
calexium_lib/web/CXM_multihost_http_server.anubis
@@ -592,7 +592,10 @@ type EncodingType:
    www_url,
    multipart_form_data. 
-
+type BufferedConnection:
+	buffered_connection(Connection 			conn,
+											Var(ByteArray)	buffer,
+											Var(Int32)			read_pos).
@@ -687,12 +690,55 @@ define Result(Error,Word8)
             " as dubious after "+(dead_line-*sttm)+" seconds. Total: "+
                  length(*list_of_dubious(dos))+"\n");
    error(timeout(dead_line)).
-   
+
+define String
+	pid
+	=
+	"[" + virtual_machine_id + "] ".
+
+define ReadResult
+   read
+     (
+       BufferedConnection   connection,
+       Int32                size,
+       Int32         				time_out
+     ) =
+  //println(pid + "read(" + size + ")"); 
+
+	if *connection.read_pos < length(*connection.buffer) then
+		//println(pid + "  reading from buffer (size = " + length(*connection.buffer) + ", pos = " + *connection.read_pos);
+		with result = extract(*connection.buffer, *connection.read_pos, *connection.read_pos + size),
+				 size_read	= length(result),
+		connection.read_pos <- *connection.read_pos + size_read;
+		if size > size_read
+			then 
+				//println("Wanted " + size + ", read only " + size_read);
+				if read(connection, size - size_read, time_out) is
+				{
+					error 	then error,
+					timeout then ok(result),
+					ok(ba) 	then ok(result + ba)
+				}
+			else ok(result)
+	else
+		//if now > dead_line then record_dubious_connection(connection,dead_line,dos) else
+		if read(connection.conn, 16384, time_out) is // the connection is closed after 10 minutes of inactivity
+		{
+		  error   then println(pid + "read failed)"); error,
+		  timeout then timeout,
+		  ok(ba)  then 
+//				println(pid + "ba = " + length(ba)); 
+		  	connection.buffer <- ba;
+		  	connection.read_pos <- 0;
+				//println(pid + "rb = " + length(*read_buffer)); 
+		  	read(connection, size, time_out)
+		}.
+
 define Result(Error,Word8)
    read_one_byte
      (
-       Connection              connection,
+       BufferedConnection      connection,
        Int32                   dead_line,
        DenialOfService         dos
      ) =
@@ -705,16 +751,54 @@ define Result(Error,Word8)
        ok(ba)  then if nth(0,ba) is 
          {
            failure then error(cannot_read_from_connection),
-           success(c) then ok(c)
+           success(c) then 
+//	           println("-" + pid + "read [" + implode([c]) + "]\t");
+             ok(c)
          }
      }.
+
+
+variable ByteArray read_buffer = constant_byte_array(0, 0).
+variable Int32 read_offset = 0.
+
+ define Result(Error,Word8)
+   read_one_byte
+     (
+       BufferedConnection      connection,
+       Int32                   dead_line,
+       DenialOfService         dos
+     ) =
+	if *read_offset < length(*read_buffer) then
+		if nth(*read_offset, *read_buffer) is
+		{
+			failure	then println(pid + "nth failed)"); error(cannot_read_from_connection),
+			success(c) then 
+				println("-" + pid + "read [" + implode([c]) + "]\tat " + *read_offset);
+				read_offset <- *read_offset + 1;
+				ok(c)
+		}
+	else
+		//if now > dead_line then record_dubious_connection(connection,dead_line,dos) else
+		println(pid + "len = " + length(*read_buffer)); 
+		if read(connection, 16384, 600) is // the connection is closed after 10 minutes of inactivity
+		{
+		  error   then println(pid + "read failed)"); error(cannot_read_from_connection),
+		  timeout then error(timeout(600)),
+		                     //record_dubious_connection(connection,dead_line,dos), 
+		  ok(ba)  then 
+				println(pid + "ba = " + length(ba)); 
+		  	read_buffer <- ba;
+		  	read_offset <- 0;
+				println(pid + "rb = " + length(*read_buffer)); 
+		  	read_one_byte(connection, dead_line, dos)
+		}.
 define Result(Error,Word8)
    next_char        // reading a character (check the list first, and read on the connection
                     //   only when the list is empty). 
      (
-       Connection              connection,
+       BufferedConnection      connection,
        Int32                   dead_line,
        DenialOfService         dos
      ) =
@@ -742,13 +826,13 @@ define Result(Error,Word8)
 define Result(Error,One)
    read_and_ignore
      (
-       Connection            connection,             // to client
+       BufferedConnection    connection,             // to client
        Int32                 dead_line, 
        Int32                 number_of_characters,   // number of characters to read and ignore
        DenialOfService       dos
      ) =
    if number_of_characters =< 0 then ok(unique) else
-   if next_char(connection,dead_line,dos) is 
+   if next_char(connection, dead_line, dos) is 
      {
        error(msg) then error(msg),
        ok(c) then read_and_ignore(connection,dead_line,number_of_characters-1,dos)
@@ -770,12 +854,12 @@ define Result(Error,One)
 define Result(Error,String)   
    read_string
      (
-       Connection            connection,         // connection with the client
+       BufferedConnection    connection,         // connection with the client
        Int32                 dead_line, 
        List(Word8)            so_far,             // characters read so far (in reverse order)
        DenialOfService       dos
      ) =
-   if next_char(connection,dead_line,dos) is 
+   if next_char(connection, dead_line,dos) is 
      {
        error(msg) then error(msg),
        ok(c) then 
@@ -1092,7 +1176,7 @@ define Bool
 define Result(Error,One)
    skip_http_blanks
      (
-       Connection            connection,
+       BufferedConnection    connection,
        Int32                 dead_line,
        DenialOfService       dos
      ) = 
@@ -1148,7 +1232,7 @@ define Result(Error,One)
 define Result(Error,One)
    read_new_line
      (
-       Connection            connection,
+       BufferedConnection    connection,
        Int32                 dead_line,
        DenialOfService       dos
      ) =
@@ -1197,7 +1281,7 @@ define Result(Error,One)
 define Result(Error,String)
    read_word_aux
      (
-       Connection           connection, 
+       BufferedConnection   connection, 
        Int32                dead_line, 
        List(Word8)           so_far,
        DenialOfService      dos
@@ -1215,7 +1299,7 @@ define Result(Error,String)
 define Result(Error,String)
    read_word
      (
-       Connection            connection,
+       BufferedConnection    connection,
        Int32                 dead_line,
        DenialOfService       dos
      ) =
@@ -1388,7 +1472,7 @@ define Result(Error,HTTP_RequestType)
 define Result(Error,HTTP_RequestLine)   
    read_request_line
      (
-       Connection            connection,
+       BufferedConnection    connection,
        Int32                 dead_line,
        DenialOfService       dos
      ) =
@@ -1445,7 +1529,7 @@ define Bool
 define Result(Error,String)
    read_header_name
      (
-       Connection            connection,
+       BufferedConnection    connection,
        Int32                 dead_line, 
        List(Word8)            so_far,
        DenialOfService       dos
@@ -1462,7 +1546,7 @@ define Result(Error,String)
 define Result(Error,One)
    skip_colon
      (
-       Connection            connection,
+       BufferedConnection    connection,
        Int32                 dead_line,
        DenialOfService       dos   
      ) =
@@ -1483,7 +1567,7 @@ define Result(Error,One)
 define Result(Error,String)
    read_header_value
      (
-       Connection           connection,
+       BufferedConnection   connection,
        Int32                dead_line, 
        List(Word8)           so_far,
        DenialOfService      dos
@@ -1517,7 +1601,7 @@ define Result(Error,String)
 define Result(Error,Maybe(HTTP_header))
    read_header
      (
-       Connection            connection,
+       BufferedConnection    connection,
        Int32                 dead_line,
        DenialOfService       dos
      ) =
@@ -1555,7 +1639,7 @@ define Result(Error,Maybe(HTTP_header))
 define Result(Error,List(HTTP_header))
    read_http_headers
      (
-       Connection             connection, 
+       BufferedConnection     connection, 
        Int32                  dead_line,
        DenialOfService        dos
      ) =
@@ -1626,7 +1710,7 @@ define Result(Error,Int32)
 define Result(Error,ByteArray)
    read_http_body
      (
-       Connection connection, 
+       BufferedConnection connection, 
        Int32 body_size,
        ByteArray so_far,          // when calling this function, 'so_far' is the empty byte array
        Int32 retries              // this function is called with retries = 10
@@ -1850,10 +1934,68 @@ define Printable_tree
      }.
+
+define String
+	month_abrv
+	(
+		Date_and_Time d
+	) =
+	if d.month = 1 				then "Jan"
+	else if d.month = 2 	then "Feb"
+	else if d.month = 3 	then "Mar"
+	else if d.month = 4 	then "Apr"
+	else if d.month = 5 	then "May"
+	else if d.month = 6 	then "Jun"
+	else if d.month = 7 	then "Jul"
+	else if d.month = 8 	then "Aug"
+	else if d.month = 9 	then "Sep"
+	else if d.month = 10 	then "Oct"
+	else if d.month = 11 	then "Nov"
+	else if d.month = 12 	then "Dec"
+	else 
+		println("Bad month value [" + d.month + "] on Date_and_Time");
+		"XXX".
+
+define String
+	weekday_abrv
+	(
+		Date_and_Time d
+	) =
+	if d.week_day = 0 			then "Sun"
+	else if d.week_day = 1	then "Mon"
+	else if d.week_day = 2 	then "Tue"
+	else if d.week_day = 3 	then "Wed"
+	else if d.week_day = 4 	then "Thu"
+	else if d.week_day = 5 	then "Fri"
+	else if d.week_day = 6 	then "Sat"
+	else 
+		println("Bad weekday value [" + d.week_day + "] on Date_and_Time");
+		"XXX".
+	
+/**
+ * Format a date with the followin format : "Mon, 23 Jul 2007 11:33:43 GMT"
+ * Currently, this function can't output a GMT time, but only local time.
+ * So the final GMT is totally fake, but needed by protocol.
+ */
+public define String
+	format_http_date
+	(
+		Date_and_Time d
+	) =
+   weekday_abrv(d) + ", " + zero_pad_n(2,day(d)) + " " + month_abrv(d) + " " + year(d) 
+   + " " + zero_pad_n(2,hour(d)) + ":" + zero_pad_n(2,minute(d)) + ":" + zero_pad_n(2,second(d)) + " GMT".
+
+/**
+ * Same as previous format_http_date() function, but with seconds count from the UNIX epoch as input.
+ */
+public define String
+	format_http_date
+	(
+		Int32 date
+	) =
+	format_http_date(convert_time(date)).
-   
- 
-   
+
       *** [5.5] Sending a file. 
    We send 2 headers 'Content-Type' and 'Content-Length'. 
@@ -1864,12 +2006,20 @@ define List(HTTP_header)
        String mime_type,
        Int32 size,
        String etag,
-     ) =
-   [
-     http_header("Content-Type",mime_type),
-     http_header("Etag", etag),
-     http_header("Content-Length",integer_to_string(size)),
-   ].
+       Maybe(FileTimes) mb_ftimes,
+     ) =
+   with headers = (List(HTTP_header))
+							   [
+							     http_header("Content-Type",mime_type),
+							     http_header("Etag", etag),
+							     http_header("Content-Length",integer_to_string(size)),
+							   ],
+   if mb_ftimes is
+   {
+   		failure					then headers,
+   		success(ftimes)	then [http_header("Last-Modified", format_http_date(ftimes.last_modified)) . headers]
+   }
+   .
@@ -1905,11 +2055,12 @@ define String
 	compute_etag
 	(
 		String                      filename,
+		Maybe(FileTimes)						mb_ftimes,
     Int32                       size,
 	) =
-    if get_file_times(filename) is 
+    if mb_ftimes is 
     {
-      failure 				then println("FAILED to get get_file_times() for file '" + filename + "'"); to_ascii(sha1((filename, size))),
+      failure 				then println("Warning: no file times for '" + filename + "', etag won't be very accurate."); to_ascii(sha1((filename, size))),
       success(ftimes) then to_ascii(sha1((filename, ftimes, size)))
 		}.
@@ -1943,12 +2094,13 @@ define One
      ) =
    action_before_send_file(unique); 
    with input_etag = http_header_value(input_headers, "If-None-Match"),
-   			current_etag = compute_etag(full_path, size),
+   			mb_ftimes = get_file_times(full_path),
+   			current_etag =	compute_etag(full_path, mb_ftimes, size),
    if are_same_etag(input_etag, current_etag) is
    {
    		false then 
 			   forget(reliable_write(connection,to_byte_array("HTTP/1.1 200 OK"+crlf))); 
-			   forget(reliable_write(connection,[format_headers(headers + headers_for_send_file(mime_type, size, current_etag)) , crlf])); 
+			   forget(reliable_write(connection,[format_headers(headers + headers_for_send_file(mime_type, size, current_etag, mb_ftimes)) , crlf])); 
 			   send_file_body(desc,connection,file,size,0,filename),
 			true then
 			   forget(reliable_write(connection,to_byte_array("HTTP/1.1 304 Not Modified"+crlf))); 
@@ -1986,6 +2138,7 @@ define One
        Connection                  connection, 
        String                      uri, 
        List(HTTP_header) 					 input_headers,
+       List(HTTP_header) 					 output_headers,
        Maybe(String)               mbauthorization,
        One -> One                  action_before_send_file
      ) =
@@ -2004,7 +2157,7 @@ define One
                    send_file(desc,
                              connection,
                              input_headers,
-                             [],
+                             output_headers,
                              size, 
                              file(f),
                              uri,
@@ -2034,7 +2187,7 @@ define One
                      send_file(desc,
                                connection, 
                                input_headers,
-                               [],
+                               output_headers,
                                size, 
                                file(f), 
                                uri,
@@ -2059,6 +2212,14 @@ define One
 define List(HTTP_header)
    standard_headers
+   =
+   [
+     http_header("Date", format_http_date(now)), 
+     http_header("Server", "Anubis Embedded Server v" + major_version_number + "." + minor_version_number)
+   ].
+
+define List(HTTP_header)
+   standard_headers_for_html
      (
        Int32     answer_body_size,
        String    charset
@@ -2100,7 +2261,8 @@ define One
                if answer_headers_body is (additional_headers,answer_body) then 
                forget(reliable_write(connection,               
                         [ "HTTP/1.1 200 OK", crlf,
-                          format_headers(standard_headers(length(answer_body),charset(desc))),
+                        	format_headers(standard_headers),
+                          format_headers(standard_headers_for_html(length(answer_body),charset(desc))),
                           format_headers(additional_headers),
                           crlf .
                           answer_body])))
@@ -2108,6 +2270,7 @@ define One
                          connection,
                          uri,
                          headers,
+                         standard_headers,
                          if web_arg_value(all_web_args,"zauth") is 
                            {
                              not_found then failure,
@@ -2551,7 +2714,7 @@ define One
                if answer_headers_body is (additional_headers,answer_body) then 
                forget(reliable_write(connection,
                         [ "HTTP/1.1 200 OK",crlf,
-                          format_headers(standard_headers(length(answer_body),charset(desc))),
+                          format_headers(standard_headers_for_html(length(answer_body),charset(desc))),
                           format_headers(additional_headers),
                           crlf .
                           answer_body])))
@@ -2790,14 +2953,14 @@ define One
    http_https_handler
      (
        List(Web_Site_Description)   sites,
-       Connection                   connection, 
+       BufferedConnection           connection, 
        Bool                         is_https,
        DenialOfService              dos
      ) =
    with start_time = (Int32)now, 
    sttm <- start_time; 
    if dos is denial_of_service(mc_v,rld_v,hd_v,ad_v,ld_v,ra_v) then 
-   if remote_IP_address_and_port(connection) is (ip_addr,port) then 
+   if remote_IP_address_and_port(connection.conn) is (ip_addr,port) then 
    if read_request_line(connection,start_time+*rld_v,dos) is 
      {
      error(msg) then print(format(msg)),
@@ -2817,8 +2980,8 @@ define One
                  {
                  error(msg) then log_journal_msg(desc,format(msg)), 
                  ok(body) then 
-                   send_answer(host_name,desc,connection,request_line,headers,body,
-                     make_generate_trust_ticket(dos)) 
+                   send_answer(host_name, desc,connection.conn, request_line, headers, body,
+                     					 make_generate_trust_ticket(dos)) 
                  }
              }
          }
@@ -2837,11 +3000,13 @@ define Server -&gt; ((RWStream) -&gt; One)
        List(Web_Site_Description)    sites, 
        DenialOfService               dos
      ) =
-   (Server server) |-> (RWStream connection) |-> 
-     if remote_IP_address_and_port(connection) is (addr,_) then 
+   (Server server) |-> (RWStream conn) |-> 
+     if remote_IP_address_and_port(conn) is (addr,_) then 
      if is_dubious_IP(addr,dos)
      then print("Rejecting dubious IP address "+ip_addr_to_string(addr)+"\n")
-     else http_https_handler(sites,tcp(connection),false,dos). 
+     else 
+   			with connection = buffered_connection(tcp(conn), var(constant_byte_array(0, 0)), var(0)),
+     		http_https_handler(sites, connection, false, dos).
 define Server -> (SSL_Connection -> One)
    make_https_handler
@@ -2849,8 +3014,9 @@ define Server -&gt; (SSL_Connection -&gt; One)
        List(Web_Site_Description)    sites, 
        DenialOfService               dos
      ) =
-   (Server server) |-> (SSL_Connection connection) |-> 
-     http_https_handler(sites,ssl(connection),true,dos). 
+   (Server server) |-> (SSL_Connection conn) |-> 
+   	 with connection = buffered_connection(ssl(conn), var(constant_byte_array(0, 0)), var(0)),
+     http_https_handler(sites, connection, true, dos). 
@@ -3240,11 +3406,12 @@ define Server -&gt; ((RWStream) -&gt; One)
      ) =
    (Server server) |-> (RWStream conn) |-> 
    with start_time = (Int32)now,
-   if read_request_line(tcp(conn),start_time+*request_line_delay(dos),dos) is 
+   			connection = buffered_connection(tcp(conn), var(constant_byte_array(0, 0)), var(0)),
+   if read_request_line(connection, start_time+*request_line_delay(dos), dos) is 
      {
        error(msg) then print(format(msg)), 
        ok(request_line) then 
-         if read_http_headers(tcp(conn),start_time+*headers_delay(dos),dos) is 
+         if read_http_headers(connection, start_time+*headers_delay(dos), dos) is 
            {
              error(msg) then print(format(msg)),
              ok(headers) then if get_host_header_value(headers) is