Anubis / Xlib

Commit cde771f1f9fbb88fc725550f76edd7b6d527314a

Authored by totoro
1 parent 973a3fae

[!] improve speed on receiving files under HTTP server

Showing 1 changed file with 437 additions and 317 deletions   Show diff stats
Inline Side-by-side
web/CXM_multihost_http_server.anubis
  Show/Hide comments   View file @cde771f
@@ -11,8 +11,9 @@ @@ -11,8 +11,9 @@
11 Cédric Ricard 11 Cédric Ricard
12 12
13 13
14 - *Revised* July 2007.  
15 - 14 + *Revised*
  15 + June 2015 : Optimization of the file upload
  16 + July 2007 : Initial version in calexium_lib
16 17
17 18
18 *Overviews* 19 *Overviews*
@@ -169,6 +170,7 @@ read tools/printable_tree.anubis @@ -169,6 +170,7 @@ read tools/printable_tree.anubis
169 read system/string.anubis 170 read system/string.anubis
170 read system/files.anubis 171 read system/files.anubis
171 read system/lists.anubis 172 read system/lists.anubis
  173 +read system/data_io.anubis
172 read web/mime.anubis 174 read web/mime.anubis
173 175
174 176
@@ -843,7 +845,6 @@ define ReadResult @@ -843,7 +845,6 @@ define ReadResult
843 connection.read_pos <- *connection.read_pos + size_read; 845 connection.read_pos <- *connection.read_pos + size_read;
844 //accumulate_t1(t1_tmp); 846 //accumulate_t1(t1_tmp);
845 if size > size_read then 847 if size > size_read then
846 - //println("Wanted " + size + ", read only " + size_read);  
847 848
848 terminal read_from_connexion(connection, size - size_read, time_out, result_buffer, position+size_read) 849 terminal read_from_connexion(connection, size - size_read, time_out, result_buffer, position+size_read)
849 // { 850 // {
@@ -862,7 +863,6 @@ define ReadResult @@ -862,7 +863,6 @@ define ReadResult
862 timeout then timeout, 863 timeout then timeout,
863 ok(ba) then 864 ok(ba) then
864 with size_read = length(ba), 865 with size_read = length(ba),
865 - //println("Wanted 32768, read only " + size_read);  
866 //the needed read is higher than the 866 //the needed read is higher than the
867 if size > size_read then 867 if size > size_read then
868 put(ba, result_buffer, position, 0); 868 put(ba, result_buffer, position, 0);
@@ -934,7 +934,31 @@ define Result(Error,Word8) @@ -934,7 +934,31 @@ define Result(Error,Word8)
934 ok(h) 934 ok(h)
935 }. 935 }.
936 936
937 - 937 +define ByteArray
  938 + get_and_erase_buffer
  939 + (
  940 + BufferedConnection connection,
  941 + SState s
  942 + )=
  943 + with head = to_byte_array(implode(*s.unput_chars)),
  944 + tail = extract(*connection.buffer, *connection.read_pos, length(*connection.buffer)),
  945 +// println("--- get_and_erase_buffer ----");
  946 +// println("unput char length : "+length(to_string(head)));
  947 +// println("connection.read_pos : "+*connection.read_pos);
  948 +// println("connection.buffer length : "+length(*connection.buffer));
  949 +// println(" tail length : "+length(tail));
  950 +// println(" -- unputchar content ");
  951 +// println("["+to_string(head)+"]");
  952 +// println(" -- buffer content ");
  953 +// println("["+to_string(*connection.buffer)+"]");
  954 +// println(" -- tail content ");
  955 +// println("["+to_string(tail)+"]");
  956 + s.unput_chars <- [];
  957 + connection.buffer <- constant_byte_array(0,0);
  958 + connection.read_pos <- 0;
  959 + head + tail
  960 + .
  961 +
938 962
939 963
940 964
@@ -1632,43 +1656,44 @@ define Bool @@ -1632,43 +1656,44 @@ define Bool
1632 if ('a' +=< c & c +=< 'z') then true else 1656 if ('a' +=< c & c +=< 'z') then true else
1633 if ('A' +=< c & c +=< 'Z') then true else 1657 if ('A' +=< c & c +=< 'Z') then true else
1634 if ('0' +=< c & c +=< '9') then true else 1658 if ('0' +=< c & c +=< '9') then true else
1635 - if c = '-' then true else 1659 + if c = '-' then true else
1636 c = '_'. 1660 c = '_'.
1637 1661
1638 define Result(Error,String) 1662 define Result(Error,String)
1639 - read_header_name  
1640 - (  
1641 - BufferedConnection connection,  
1642 - Int dead_line,  
1643 - List(Word8) so_far,  
1644 - DenialOfService dos,  
1645 - SState s  
1646 - ) = 1663 + read_header_name
  1664 + (
  1665 + BufferedConnection connection,
  1666 + Int dead_line,
  1667 + List(Word8) so_far,
  1668 + DenialOfService dos,
  1669 + SState s
  1670 + ) =
1647 if next_char(connection, dead_line, dos, s) is 1671 if next_char(connection, dead_line, dos, s) is
1648 { 1672 {
1649 error(msg) then error(msg), 1673 error(msg) then error(msg),
1650 - ok(c) then 1674 + ok(c) then
1651 if is_header_name_char(c) 1675 if is_header_name_char(c)
1652 - then read_header_name(connection,dead_line,[to_lower(c) . so_far],dos, s) 1676 + then read_header_name(connection, dead_line, [to_lower(c) . so_far], dos, s)
1653 else unput(c, s); ok(implode(reverse(so_far))) 1677 else unput(c, s); ok(implode(reverse(so_far)))
1654 }. 1678 }.
1655 1679
1656 define Result(Error,One) 1680 define Result(Error,One)
1657 - skip_colon  
1658 - (  
1659 - BufferedConnection connection,  
1660 - Int dead_line,  
1661 - DenialOfService dos,  
1662 - SState s  
1663 - ) =  
1664 - if skip_http_blanks(connection, dead_line, dos, s) is  
1665 - {  
1666 - error(msg) then error(msg),  
1667 - ok(_) then  
1668 - if next_char(connection, dead_line, dos, s) is  
1669 - {  
1670 - error(msg) then error(msg),  
1671 - ok(c) then 1681 + skip_colon
  1682 + (
  1683 + BufferedConnection connection,
  1684 + Int dead_line,
  1685 + DenialOfService dos,
  1686 + SState s
  1687 + ) =
  1688 + //Skip the blank char until ':'
  1689 + if skip_http_blanks(connection, dead_line, dos, s) is
  1690 + {
  1691 + error(msg) then error(msg),
  1692 + ok(_) then
  1693 + if next_char(connection, dead_line, dos, s) is
  1694 + {
  1695 + error(msg) then error(msg),
  1696 + ok(c) then
1672 if c = ':' 1697 if c = ':'
1673 then ok(unique) 1698 then ok(unique)
1674 else error(colon_expected) 1699 else error(colon_expected)
@@ -1718,49 +1743,54 @@ define Result(Error,Maybe(HTTP_header)) @@ -1718,49 +1743,54 @@ define Result(Error,Maybe(HTTP_header))
1718 DenialOfService dos, 1743 DenialOfService dos,
1719 SState s 1744 SState s
1720 ) = 1745 ) =
1721 - if read_header_name(connection, dead_line, [], dos, s) is  
1722 - {  
1723 - error(msg) then error(msg),  
1724 - ok(name) then  
1725 - if name = "" then  
1726 - if read_and_ignore(connection, dead_line, 2, dos, s) /* 13 and 10 */ is  
1727 - {  
1728 - error(msg) then error(msg),  
1729 - ok(_) then // this is the blank line  
1730 - ok(failure) // end of headers 1746 + //Find the name
  1747 + if read_header_name(connection, dead_line, [], dos, s) is
  1748 + {
  1749 + error(msg) then error(msg),
  1750 + ok(name) then
  1751 + if name = "" then
  1752 + if read_and_ignore(connection, dead_line, 2, dos, s) /* 13 and 10 */ is
  1753 + {
  1754 + error(msg) then error(msg),
  1755 + ok(_) then // this is the blank line
  1756 + ok(failure) // end of headers
  1757 + }
  1758 + //skip the ':' and blank before and after it
  1759 + else if skip_colon(connection, dead_line, dos, s) is
  1760 + {
  1761 + error(msg) then error(msg),
  1762 + ok(_) then
  1763 + //skip the blank char after the ':'
  1764 + if skip_http_blanks(connection, dead_line, dos, s) is
  1765 + {
  1766 + error(msg) then error(msg),
  1767 + ok(_) then
  1768 + //Now read the value
  1769 + if read_header_value(connection, dead_line, [], dos, s) is
  1770 + {
  1771 + error(msg) then error(msg),
  1772 + ok(value) then ok(success(http_header(name,value)))
1731 } 1773 }
1732 - else if skip_colon(connection, dead_line, dos, s) is  
1733 - {  
1734 - error(msg) then error(msg),  
1735 - ok(_) then if skip_http_blanks(connection, dead_line, dos, s) is  
1736 - {  
1737 - error(msg) then error(msg),  
1738 - ok(_) then if read_header_value(connection, dead_line, [], dos, s) is  
1739 - {  
1740 - error(msg) then error(msg),  
1741 - ok(value) then  
1742 - ok(success(http_header(name,value)))  
1743 - }  
1744 - }  
1745 - }  
1746 - }. 1774 + }
  1775 + }
  1776 + }.
1747 1777
1748 1778
1749 1779
1750 Reading all the headers. 1780 Reading all the headers.
1751 1781
1752 define Result(Error,List(HTTP_header)) 1782 define Result(Error,List(HTTP_header))
1753 - read_http_headers  
1754 - (  
1755 - BufferedConnection connection,  
1756 - Int dead_line,  
1757 - DenialOfService dos,  
1758 - SState s  
1759 - ) =  
1760 - if read_header(connection, dead_line, dos, s) is  
1761 - {  
1762 - error(msg) then error(msg),  
1763 - ok(mbh) then if mbh is 1783 + read_http_headers
  1784 + (
  1785 + BufferedConnection connection,
  1786 + Int dead_line,
  1787 + DenialOfService dos,
  1788 + SState s
  1789 + ) =
  1790 + if read_header(connection, dead_line, dos, s) is
  1791 + {
  1792 + error(msg) then error(msg),
  1793 + ok(mbh) then if mbh is
1764 { 1794 {
1765 failure then ok([ ]), 1795 failure then ok([ ]),
1766 success(header) then 1796 success(header) then
@@ -1784,21 +1814,21 @@ define Result(Error,List(HTTP_header)) @@ -1784,21 +1814,21 @@ define Result(Error,List(HTTP_header))
1784 header is not present, the size is assumed to be zero. 1814 header is not present, the size is assumed to be zero.
1785 1815
1786 define Result(Error,Int) 1816 define Result(Error,Int)
1787 - get_body_size  
1788 - (  
1789 - List(HTTP_header) headers  
1790 - ) =  
1791 - if headers is  
1792 - {  
1793 - [ ] then ok(0),  
1794 - [h . t] then if h is http_header(name,value) then  
1795 - if name = "content-length"  
1796 - then if decimal_scan(value) is  
1797 - {  
1798 - failure then error(incorrect_content_length_value),  
1799 - success(n) then ok(n)  
1800 - }  
1801 - else get_body_size(t) 1817 + get_body_size
  1818 + (
  1819 + List(HTTP_header) headers
  1820 + ) =
  1821 + if headers is
  1822 + {
  1823 + [ ] then ok(0),
  1824 + [h . t] then if h is http_header(name,value) then
  1825 + if name = "content-length"
  1826 + then if decimal_scan(value) is
  1827 + {
  1828 + failure then error(incorrect_content_length_value),
  1829 + success(n) then ok(n)
  1830 + }
  1831 + else get_body_size(t)
1802 }. 1832 }.
1803 1833
1804 1834
@@ -1820,17 +1850,18 @@ define Result(Error,Int) @@ -1820,17 +1850,18 @@ define Result(Error,Int)
1820 required number of bytes. However, if the number of bytes read is zero, the connection 1850 required number of bytes. However, if the number of bytes read is zero, the connection
1821 may be broken. In that case, we must not try to read indefinitely. On the contrary, we 1851 may be broken. In that case, we must not try to read indefinitely. On the contrary, we
1822 make at most 10 retries, with a small sleeping time between any two of them. 1852 make at most 10 retries, with a small sleeping time between any two of them.
1823 - 1853 +
1824 define Result(Error, ByteArray) 1854 define Result(Error, ByteArray)
1825 read_http_body 1855 read_http_body
1826 ( 1856 (
1827 BufferedConnection connection, 1857 BufferedConnection connection,
1828 - Int body_size,  
1829 - ByteArray so_far, // when calling this function, 'so_far' is the empty byte array  
1830 - Int retries // this function is called with retries = 10 1858 + Int body_size,
  1859 + ByteArray so_far, // when calling this function, 'so_far' is the empty byte array
  1860 + Int retries // this function is called with retries = 10
1831 ) = 1861 ) =
1832 if body_size = 0 then ok(constant_byte_array(0,0)) else 1862 if body_size = 0 then ok(constant_byte_array(0,0)) else
1833 if retries =< 0 then error(cannot_read_from_connection) else 1863 if retries =< 0 then error(cannot_read_from_connection) else
  1864 +
1834 if read_from_connexion(connection,body_size,60,constant_byte_array(body_size,0),0) is 1865 if read_from_connexion(connection,body_size,60,constant_byte_array(body_size,0),0) is
1835 { 1866 {
1836 error then error(cannot_read_from_connection), 1867 error then error(cannot_read_from_connection),
@@ -2016,19 +2047,19 @@ public define Bool @@ -2016,19 +2047,19 @@ public define Bool
2016 2047
2017 2048
2018 define Maybe(MIME) 2049 define Maybe(MIME)
2019 - recognize_mime_type_from_ext  
2020 - (  
2021 - String ext,  
2022 - List(MIME) l  
2023 - ) =  
2024 - if l is  
2025 - {  
2026 - [ ] then success(mime("application", "octet-stream", [])), // failure,  
2027 - [h . t] then if h is mime(type, subtype, extensions) then  
2028 - if contains_no_case(extensions, ext)  
2029 - then success(h)  
2030 - else recognize_mime_type_from_ext(ext,t)  
2031 - }. 2050 + recognize_mime_type_from_ext
  2051 + (
  2052 + String ext,
  2053 + List(MIME) l
  2054 + ) =
  2055 + if l is
  2056 + {
  2057 + [ ] then success(mime("application", "octet-stream", [])), // failure,
  2058 + [h . t] then if h is mime(type, subtype, extensions) then
  2059 + if contains_no_case(extensions, ext)
  2060 + then success(h)
  2061 + else recognize_mime_type_from_ext(ext,t)
  2062 + }.
2032 2063
2033 define Maybe(MIME) 2064 define Maybe(MIME)
2034 recognize_mime_type_from_uri 2065 recognize_mime_type_from_uri
@@ -2038,13 +2069,8 @@ define Maybe(MIME) @@ -2038,13 +2069,8 @@ define Maybe(MIME)
2038 ) = 2069 ) =
2039 recognize_mime_type_from_ext(get_uri_extension(uri),known_mime_types(desc)). 2070 recognize_mime_type_from_ext(get_uri_extension(uri),known_mime_types(desc)).
2040 2071
2041 -  
2042 2072
2043 2073
2044 -  
2045 -  
2046 -  
2047 -  
2048 *** [5.4] Formating HTTP headers. 2074 *** [5.4] Formating HTTP headers.
2049 2075
2050 This is the formating for sending to the client (hence, it has nothing to do with the 2076 This is the formating for sending to the client (hence, it has nothing to do with the
@@ -2187,11 +2213,11 @@ define String @@ -2187,11 +2213,11 @@ define String
2187 Maybe(FileTimes) mb_ftimes, 2213 Maybe(FileTimes) mb_ftimes,
2188 Int size, 2214 Int size,
2189 ) = 2215 ) =
2190 - if mb_ftimes is  
2191 - {  
2192 - failure then println("Warning: no file times for '" + filename + "', etag won't be very accurate."); to_ascii(sha1((filename, size))),  
2193 - success(ftimes) then to_ascii(md5((filename, ftimes, size)))  
2194 - }. 2216 + if mb_ftimes is
  2217 + {
  2218 + failure then println("Warning: no file times for '" + filename + "', etag won't be very accurate."); to_ascii(sha1((filename, size))),
  2219 + success(ftimes) then to_ascii(md5((filename, ftimes, size)))
  2220 + }.
2195 2221
2196 define Bool 2222 define Bool
2197 are_same_etag 2223 are_same_etag
@@ -2243,18 +2269,18 @@ define One @@ -2243,18 +2269,18 @@ define One
2243 Checking if a connection is under SSL. 2269 Checking if a connection is under SSL.
2244 2270
2245 define Bool 2271 define Bool
2246 - is_SSL  
2247 - (  
2248 - Connection c  
2249 - ) =  
2250 - if c is  
2251 - {  
2252 - file_r(_) then false,  
2253 - file_w(_) then false,  
2254 - file_rw(_) then false,  
2255 - tcp(_) then false,  
2256 - ssl(_) then true  
2257 - }. 2272 + is_SSL
  2273 + (
  2274 + Connection c
  2275 + ) =
  2276 + if c is
  2277 + {
  2278 + file_r(_) then false,
  2279 + file_w(_) then false,
  2280 + file_rw(_) then false,
  2281 + tcp(_) then false,
  2282 + ssl(_) then true
  2283 + }.
2258 2284
2259 2285
2260 2286
@@ -2527,10 +2553,10 @@ define Maybe(String) @@ -2527,10 +2553,10 @@ define Maybe(String)
2527 }. 2553 }.
2528 2554
2529 define Maybe(String) 2555 define Maybe(String)
2530 - get_boundary  
2531 - (  
2532 - List(HTTP_header) headers  
2533 - ) = 2556 + get_boundary
  2557 + (
  2558 + List(HTTP_header) headers
  2559 + )=
2534 if headers is 2560 if headers is
2535 { 2561 {
2536 [ ] then failure, 2562 [ ] then failure,
@@ -2586,13 +2612,13 @@ define Maybe(String) @@ -2586,13 +2612,13 @@ define Maybe(String)
2586 argument. 2612 argument.
2587 2613
2588 define Maybe(Int) 2614 define Maybe(Int)
2589 - find  
2590 - (  
2591 - String what,  
2592 - ByteArray where,  
2593 - Int start,  
2594 - Int end  
2595 - ) = 2615 + find
  2616 + (
  2617 + String what,
  2618 + ByteArray where,
  2619 + Int start,
  2620 + Int end
  2621 + ) =
2596 if find(to_byte_array(what),where,start) is 2622 if find(to_byte_array(what),where,start) is
2597 { 2623 {
2598 failure then failure, 2624 failure then failure,
@@ -2601,7 +2627,23 @@ define Maybe(Int) @@ -2601,7 +2627,23 @@ define Maybe(Int)
2601 then failure 2627 then failure
2602 else success(n) 2628 else success(n)
2603 }. 2629 }.
2604 - 2630 +
  2631 +define Maybe(Int)
  2632 + find
  2633 + (
  2634 + String what,
  2635 + String where,
  2636 + Int start,
  2637 + Int end
  2638 + ) =
  2639 + if find_string(where,what,start) is
  2640 + {
  2641 + failure then failure,
  2642 + success(n) then
  2643 + if n+length(what) >= end
  2644 + then failure
  2645 + else success(n)
  2646 + }.
2605 2647
2606 define String 2648 define String
2607 read_attribute_value 2649 read_attribute_value
@@ -2642,26 +2684,26 @@ define Maybe(String) @@ -2642,26 +2684,26 @@ define Maybe(String)
2642 2684
2643 2685
2644 define Maybe((String,Maybe(String))) 2686 define Maybe((String,Maybe(String)))
2645 - find_name_and_filename  
2646 - (  
2647 - ByteArray body,  
2648 - Int start,  
2649 - Int end  
2650 - ) =  
2651 - if find(to_byte_array("Content-Disposition"),body,start) is  
2652 - {  
2653 - failure then failure,  
2654 - success(n) then  
2655 - if find_attribute("name",body,n+19,end) is  
2656 - {  
2657 - failure then failure,  
2658 - success(name_value) then if find_attribute("filename",body,n+19,end) is  
2659 - {  
2660 - failure then success((name_value,failure)),  
2661 - success(filename_value) then success((name_value,success(filename_value)))  
2662 - }  
2663 - }  
2664 - }. 2687 + find_name_and_filename
  2688 + (
  2689 + ByteArray body,
  2690 + Int start,
  2691 + Int end
  2692 + ) =
  2693 + if find(to_byte_array("Content-Disposition"),body,start) is
  2694 + {
  2695 + failure then failure,
  2696 + success(n) then
  2697 + if find_attribute("name",body,n+19,end) is
  2698 + {
  2699 + failure then failure,
  2700 + success(name_value) then if find_attribute("filename",body,n+19,end) is
  2701 + {
  2702 + failure then success((name_value,failure)),
  2703 + success(filename_value) then success((name_value,success(filename_value)))
  2704 + }
  2705 + }
  2706 + }.
2665 2707
2666 2708
2667 2709
@@ -2697,27 +2739,38 @@ define Maybe((String,Maybe(String))) @@ -2697,27 +2739,38 @@ define Maybe((String,Maybe(String)))
2697 define Maybe(String) // returns the temporary file name 2739 define Maybe(String) // returns the temporary file name
2698 save_uploaded_file 2740 save_uploaded_file
2699 ( 2741 (
2700 - Web_Site_Description desc,  
2701 - ByteArray body,  
2702 - Int start,  
2703 - Int end,  
2704 - SState s 2742 + Web_Site_Description desc,
  2743 + RStream body_fd,
  2744 +// ByteArray body,
  2745 + Int start,
  2746 + Int end,
  2747 + SState s
2705 ) = 2748 ) =
2706 s.uploaded_file_count <- 1 + *(s.uploaded_file_count); 2749 s.uploaded_file_count <- 1 + *(s.uploaded_file_count);
2707 with tfn = "_"+to_decimal(virtual_machine_id)+"_"+to_decimal(*(s.uploaded_file_count)), 2750 with tfn = "_"+to_decimal(virtual_machine_id)+"_"+to_decimal(*(s.uploaded_file_count)),
2708 - if (Maybe(RWStream))file(site_directory(desc)+"/upload_temporary/"+tfn, new) is  
2709 - {  
2710 - failure then failure,  
2711 - success(f) then  
2712 - if reliable_write(file(f),extract(body,start,end)) is  
2713 - {  
2714 - failure then failure,  
2715 - success(nw) then  
2716 - if nw = end - start  
2717 - then success(tfn)  
2718 - else failure  
2719 - }  
2720 - }. 2751 +// println("save_uploaded_file to :"+site_directory(desc)+"/upload_temporary/"+tfn);
  2752 +// println("start offset = "+start+" end offset = "+end);
  2753 + //make data_io which is the size of the file to extract
  2754 + if copy_Data_IO_to_file(make_data_io(body_fd, start, end - start), site_directory(desc)+"/upload_temporary/"+tfn) is copy_ok(_)
  2755 + then success(tfn)
  2756 + else failure
  2757 + .
  2758 +
  2759 +
  2760 +
  2761 +// if (Maybe(RWStream))file(site_directory(desc)+"/upload_temporary/"+tfn, new) is
  2762 +// {
  2763 +// failure then failure,
  2764 +// success(f) then
  2765 +// if reliable_write(file(f),extract(body,start,end)) is
  2766 +// {
  2767 +// failure then failure,
  2768 +// success(nw) then
  2769 +// if nw = end - start
  2770 +// then success(tfn)
  2771 +// else failure
  2772 +// }
  2773 +// }.
2721 2774
2722 2775
2723 2776
@@ -2763,94 +2816,114 @@ define String @@ -2763,94 +2816,114 @@ define String
2763 *** [5.7.6] Reading a multipart entity. 2816 *** [5.7.6] Reading a multipart entity.
2764 2817
2765 define Maybe(Web_arg) 2818 define Maybe(Web_arg)
2766 - get_multipart_entity  
2767 - (  
2768 - Web_Site_Description desc,  
2769 - ByteArray body,  
2770 - Int start,  
2771 - Int end,  
2772 - SState s  
2773 - ) =  
2774 - if find(to_byte_array(crlf+crlf),body,start) is  
2775 - {  
2776 - failure then failure,  
2777 - success(k) then  
2778 - if k >= end // must be within this entity, not the next one  
2779 - then failure  
2780 - else if find_name_and_filename(body,start,k) is  
2781 - {  
2782 - failure then failure,  
2783 - success(n_mbfn) then if n_mbfn is (name,mbfn) then 2819 + get_multipart_entity
  2820 + (
  2821 + Web_Site_Description desc,
  2822 + String body_temp_file,
  2823 + Int start_offset, //real offset in source file of the part
  2824 + Int end_offset, //real offset in source file of the part
  2825 + SState s
  2826 + )=
  2827 + //Get header of the part
  2828 + if find_the_first(body_temp_file, crlf+crlf, start_offset, end_offset) is
  2829 + {
  2830 + failure then failure,
  2831 + success(k) then
  2832 + if (Maybe(RStream))file(body_temp_file, read) is
  2833 + {
  2834 + failure then failure,
  2835 + success(body_fd) then
  2836 + if read_bytes(make_data_io(body_fd, start_offset, k), k) is success(header_part) then
  2837 +
  2838 + if find_name_and_filename(header_part, 0, k) is
  2839 + {
  2840 + failure then failure,
  2841 + success(n_mbfn) then if n_mbfn is (name, mbfn) then
2784 if mbfn is 2842 if mbfn is
2785 - { 2843 + {
2786 failure then 2844 failure then
2787 - success(web_arg(name,to_string(extract(body,k+4,end-2)))), 2845 + if read_bytes(make_data_io(body_fd, start_offset+k+4, (end_offset-2)-(start_offset+k+4)), (end_offset-2)-(start_offset+k+4)) is success(attachement) then
  2846 + success(web_arg(name,to_string(attachement)))
2788 // we must substract 2 to end because of crlf just before the boundary 2847 // we must substract 2 to end because of crlf just before the boundary
  2848 + else
  2849 + failure,
2789 2850
2790 success(fn) then 2851 success(fn) then
2791 - if save_uploaded_file(desc, body, k+4, end-2, s) is  
2792 - {  
2793 - failure then failure,  
2794 - success(tfn) then  
2795 - success(upload(name,remove_path(fn),  
2796 - site_directory(desc)+"/upload_temporary/"+tfn)) 2852 + if save_uploaded_file(desc, body_fd, start_offset+k+4, end_offset-2, s) is
  2853 + {
  2854 + failure then failure,
  2855 + success(tfn) then success(upload(name, remove_path(fn), site_directory(desc)+"/upload_temporary/"+tfn))
2797 2856
2798 } 2857 }
2799 } 2858 }
2800 } 2859 }
2801 - }. 2860 + else
  2861 + failure
  2862 + }
  2863 + }.
2802 2864
2803 2865
2804 2866
2805 define List(Web_arg) 2867 define List(Web_arg)
2806 - read_multipart_form_data_encoded_web_args  
2807 - (  
2808 - Web_Site_Description desc,  
2809 - ByteArray body,  
2810 - ByteArray __boundary,  
2811 - Int i,  
2812 - SState s  
2813 - ) =  
2814 - if find(__boundary,body,i) is  
2815 - {  
2816 - failure then [ ],  
2817 - success(n) then  
2818 - if find(__boundary,body,n+length(__boundary)) is  
2819 - {  
2820 - failure then [ ],  
2821 - success(m) then  
2822 - if get_multipart_entity(desc, body, n+length(__boundary), m, s) is  
2823 - {  
2824 - failure then [ ],  
2825 - success(wa) then  
2826 - [wa . read_multipart_form_data_encoded_web_args(desc, body, __boundary, m, s)]  
2827 - }  
2828 - }  
2829 - }. 2868 + read_multipart_form_data_encoded_web_args
  2869 + (
  2870 + Web_Site_Description desc,
  2871 + String body_temp_file,
  2872 + String __boundary,
  2873 + Int file_offset,
  2874 + SState s
  2875 + ) =
  2876 + with boundary_length = length(__boundary),
  2877 +// println(
  2878 +//"read_multipart_form_data_encoded_web_args
  2879 +// boundary["+__boundary+"]
  2880 +// body_temp_file : "+body_temp_file+"
  2881 +// file_offset : "+file_offset);
  2882 + //get the first boundary position
  2883 + if find_the_first(body_temp_file, __boundary, file_offset) is
  2884 + {
  2885 + failure then println("first boundary not found");[ ],
  2886 + success(first) then
  2887 + with first = first + file_offset, //adjust the offset to real offset in file
  2888 + //get the second boundary position
  2889 + if find_the_first(body_temp_file, __boundary, first + boundary_length) is
  2890 + {
  2891 + failure then println("last boundary found");[ ],
  2892 + success(last) then
  2893 + with last = last + first + boundary_length, //adjust the offset to real offset in file
  2894 + //Extract the file content here
  2895 + if get_multipart_entity(desc, body_temp_file, first+boundary_length, last, s) is
  2896 + {
  2897 + failure then [ ],
  2898 + success(wa) then
  2899 + [wa . read_multipart_form_data_encoded_web_args(desc, body_temp_file, __boundary, last, s)]
  2900 + }
  2901 + }
  2902 + }.
2830 2903
2831 2904
2832 2905
2833 define One 2906 define One
2834 multipart_form_data_answer 2907 multipart_form_data_answer
2835 ( 2908 (
2836 - String host_name,  
2837 - Web_Site_Description desc,  
2838 - Connection connection,  
2839 - Word32 ip_addr,  
2840 - HTTP_RequestLine request_line,  
2841 - List(HTTP_header) headers,  
2842 - ByteArray body,  
2843 - One -> String generate_tt,  
2844 - SState s 2909 + String host_name,
  2910 + Web_Site_Description desc,
  2911 + Connection connection,
  2912 + Word32 ip_addr,
  2913 + HTTP_RequestLine request_line,
  2914 + List(HTTP_header) headers,
  2915 + String body_temp_file,
  2916 + One -> String generate_tt,
  2917 + SState s
2845 ) = 2918 ) =
2846 - if get_boundary(headers) is  
2847 - {  
2848 - failure then unique,  
2849 - success(boundary) then  
2850 - with all_web_args = query_string(request_line) + 2919 + if get_boundary(headers) is
  2920 + {
  2921 + failure then unique,
  2922 + success(boundary) then
  2923 + with all_web_args = query_string(request_line) +
2851 read_multipart_form_data_encoded_web_args(desc, 2924 read_multipart_form_data_encoded_web_args(desc,
2852 - body,  
2853 - to_byte_array("--"+boundary), 2925 + body_temp_file,
  2926 + "--"+boundary,
2854 0, 2927 0,
2855 s), 2928 s),
2856 uri = uri(request_line), 2929 uri = uri(request_line),
@@ -2967,47 +3040,46 @@ define String @@ -2967,47 +3040,46 @@ define String
2967 is achieved through the header 'Content-Type'. 3040 is achieved through the header 'Content-Type'.
2968 3041
2969 define EncodingType 3042 define EncodingType
2970 - get_encoding_type  
2971 - (  
2972 - List(HTTP_header) headers  
2973 - ) =  
2974 - if headers is  
2975 - {  
2976 - [ ] then www_url, // this is the default  
2977 - [h . t] then if h is http_header(name,value) then  
2978 - if name = "content-type"  
2979 - then if find("multipart/form-data",value,0) is  
2980 - {  
2981 - failure then www_url,  
2982 - success(_) then multipart_form_data  
2983 - }  
2984 - else get_encoding_type(t) 3043 + get_encoding_type
  3044 + (
  3045 + List(HTTP_header) headers
  3046 + ) =
  3047 + if headers is
  3048 + {
  3049 + [ ] then www_url, // this is the default
  3050 + [h . t] then if h is http_header(name, value) then
  3051 + if name = "content-type"
  3052 + then if find("multipart/form-data",value,0) is
  3053 + {
  3054 + failure then www_url,
  3055 + success(_) then multipart_form_data
  3056 + }
  3057 + else
  3058 + get_encoding_type(t) //content-type not found, check the next header line
2985 }. 3059 }.
2986 3060
2987 3061
2988 3062
2989 -define One  
2990 - send_answer  
2991 - (  
2992 - String host_name,  
2993 - Web_Site_Description desc,  
2994 - Connection connection,  
2995 - HTTP_RequestLine rqline,  
2996 - List(HTTP_header) headers,  
2997 - ByteArray body,  
2998 - One -> String generate_tt,  
2999 - SState s  
3000 - ) =  
3001 - if rqline is request_line(type,uri,qstring) then  
3002 - with rqline2 = request_line(type,handle_redirection(redirections(desc),uri,headers),qstring),  
3003 - if remote_IP_address_and_port(connection) is (ip_addr,_) then  
3004 - if get_encoding_type(headers) is  
3005 - {  
3006 - www_url then  
3007 - www_url_answer(host_name,desc,connection,ip_addr,rqline2,headers,body,generate_tt),  
3008 - multipart_form_data then  
3009 - multipart_form_data_answer(host_name,desc,connection,ip_addr,rqline2,headers,body,generate_tt, s)  
3010 - }. 3063 + define One
  3064 + send_answer
  3065 + (
  3066 + String host_name,
  3067 + Web_Site_Description desc,
  3068 + Connection connection,
  3069 + HTTP_RequestLine rqline,
  3070 + List(HTTP_header) headers,
  3071 + ByteArray body,
  3072 + One -> String generate_tt,
  3073 + SState s
  3074 + )=
  3075 + if rqline is request_line(type, uri, qstring) then
  3076 + with rqline2 = request_line(type, handle_redirection(redirections(desc), uri, headers), qstring),
  3077 + if remote_IP_address_and_port(connection) is (ip_addr,_) then
  3078 + if get_encoding_type(headers) is
  3079 + {
  3080 + www_url then www_url_answer(host_name, desc, connection, ip_addr, rqline2, headers, body, generate_tt),
  3081 + multipart_form_data then multipart_form_data_answer(host_name, desc, connection, ip_addr, rqline2, headers, body, generate_tt, s)
  3082 + }.
3011 3083
3012 3084
3013 3085
@@ -3110,54 +3182,102 @@ define One @@ -3110,54 +3182,102 @@ define One
3110 SState s 3182 SState s
3111 ) = 3183 ) =
3112 //t0 <- (UTime)unow; 3184 //t0 <- (UTime)unow;
3113 - with start_time = (Int)now,  
3114 - s.sttm <- start_time;  
3115 - //println("Request time: " + format_http_date(start_time));  
3116 - if dos is denial_of_service(mc_v,rld_v,hd_v,ad_v,ld_v,ra_v) then  
3117 - if remote_IP_address_and_port(connection.conn) is (ip_addr,port) then  
3118 - if read_request_line(connection, start_time+*rld_v, dos, s) is  
3119 - {  
3120 - error(msg) then print(format(msg)),  
3121 - ok(request_line) then  
3122 - //print_delta("read_request_line");  
3123 - if read_http_headers(connection, start_time+*hd_v, dos, s) is  
3124 - {  
3125 - error(msg) then print(format(msg)),  
3126 - ok(headers) then //print_delta("read_http_headers");  
3127 - if get_site(headers,sites) is  
3128 - {  
3129 - failure then unique,  
3130 - success(p) then if p is (host_name,desc) then  
3131 - //print_delta("get_site");  
3132 - if get_body_size(headers) is  
3133 - {  
3134 - error(msg) then log_journal_msg(desc,format(msg)),  
3135 - ok(body_size) then 3185 + with start_time = (Int)now,
  3186 + s.sttm <- start_time;
  3187 + //println("Request time: " + format_http_date(start_time));
  3188 + if dos is denial_of_service(mc_v,rld_v,hd_v,ad_v,ld_v,ra_v) then
  3189 + if remote_IP_address_and_port(connection.conn) is (ip_addr,port) then
  3190 + if read_request_line(connection, start_time+*rld_v, dos, s) is
  3191 + {
  3192 + error(msg) then print(format(msg)),
  3193 + ok(rqline) then //request line
  3194 + //print_delta("read_request_line");
  3195 + if read_http_headers(connection, start_time+*hd_v, dos, s) is
  3196 + {
  3197 + error(msg) then print(format(msg)),
  3198 + ok(headers) then //print_delta("read_http_headers");
  3199 + if get_site(headers,sites) is
  3200 + {
  3201 + failure then unique,
  3202 + success(p) then if p is (host_name, desc) then
  3203 + //Body Size
  3204 + if get_body_size(headers) is
  3205 + {
  3206 + error(msg) then log_journal_msg(desc,format(msg)),
  3207 + ok(body_size) then
  3208 + if rqline is request_line(type, uri, qstring) then
  3209 + with rqline2 = request_line(type, handle_redirection(redirections(desc), uri, headers), qstring),
  3210 +
  3211 + //Get the type of encoding which decide if we read the content in ByteArray for www_url or in
  3212 + //temporary file for multipart.
  3213 + with generate_tt = make_generate_trust_ticket(dos),
  3214 + if get_encoding_type(headers) is
  3215 + {
  3216 + //WWW_URL
  3217 + www_url then
  3218 + if read_http_body(connection,body_size,constant_byte_array(0,0),1000) is
  3219 + {
  3220 + error(msg) then log_journal_msg(desc,format(msg)),
  3221 + ok(body) then
  3222 + www_url_answer(host_name, desc, connection.conn, ip_addr, rqline2, headers, body, generate_tt);
  3223 + //it's HTTP 1.1 keep-alive is default
  3224 + http_https_handler(sites, connection, is_https, dos, s)
  3225 + }
  3226 + //MULTIPART_FORM_DATA
  3227 + multipart_form_data then
  3228 +
  3229 + if body_size > 0 then
  3230 + with t0 = (UTime)unow,
  3231 + println("read body, size "+body_size);
  3232 +
  3233 + if get_socket_from_connection(conn(connection)) is
  3234 + {
  3235 + failure then println("can't get socket"),
  3236 + success(source) then
3136 3237
3137 - with t0 = (UTime)unow,  
3138 - (if body_size > 0 then  
3139 - println("read body, size "+body_size)  
3140 - else  
3141 - unique);  
3142 - if read_http_body(connection,body_size,constant_byte_array(0,0),1000) is  
3143 - {  
3144 - error(msg) then log_journal_msg(desc,format(msg)),  
3145 - ok(body) then  
3146 - with duration = (UTime) unow - t0,  
3147 - (if body_size > 0 then  
3148 - println("Read body "+body_size+" duration: " + __utime_to_string(duration))  
3149 - else  
3150 - unique); 3238 + with tmp_body_file = "/temp/" + virtual_machine_id + "-" + now,
  3239 + if (Maybe(RWStream))file(desc.site_directory + tmp_body_file, new) is
  3240 + {
  3241 + failure then println("can't create target file "+desc.site_directory + tmp_body_file),//nothing to write
  3242 + success(target) then
  3243 +
  3244 + //get the content of the current buffer and unput char list
  3245 + with buffer = get_and_erase_buffer(connection, s),
  3246 + buffer_size = length(buffer),
  3247 + //println("buffer Size = "+buffer_size);
  3248 + //println("Old body size "+body_size+" New body size request = "+body_size - buffer_size);
  3249 + if flush(buffer, weaken(target)) is
  3250 + {
  3251 + failure then println("Can't flush the buffer"),
  3252 + success(_) then
  3253 + if copy_file(weaken(source), weaken(target), body_size - buffer_size) is copy_ok(read_size) then
  3254 +
  3255 + with duration = (UTime) unow - t0,
  3256 + println("Read body "+read_size+" duration: " + __utime_to_string(duration));
  3257 +
  3258 + multipart_form_data_answer(host_name, desc, connection.conn, ip_addr, rqline2, headers, desc.site_directory + tmp_body_file, generate_tt, s);
  3259 +
  3260 + //it's HTTP 1.1 keep-alive is default
  3261 + http_https_handler(sites, connection, is_https, dos, s)
  3262 +
  3263 + else
  3264 + println("Can't copy data from stream to temporary file ")
  3265 + }}}
  3266 + else
  3267 + println("bbody_size = 0 !")
  3268 + }
  3269 +
3151 //print_delta("before send_answer"); 3270 //print_delta("before send_answer");
3152 - send_answer(host_name, desc,connection.conn, request_line, headers, body,  
3153 - make_generate_trust_ticket(dos), s);  
3154 - //it's HTTP 1.1 keep-alive is default  
3155 - http_https_handler(sites, connection, is_https, dos, s) 3271 +// with body = constant_byte_array(0,0),
  3272 +// send_answer(host_name, desc,connection.conn, request_line, headers, body,
  3273 +// make_generate_trust_ticket(dos), s);
  3274 +// //it's HTTP 1.1 keep-alive is default
  3275 +// http_https_handler(sites, connection, is_https, dos, s)
3156 //with duration = (UTime) unow - *t0, 3276 //with duration = (UTime) unow - *t0,
3157 //println("Request duration: " + __utime_to_string(duration)) 3277 //println("Request duration: " + __utime_to_string(duration))
3158 //println("BufferRead duration: " + __utime_to_string(*t1)); 3278 //println("BufferRead duration: " + __utime_to_string(*t1));
3159 //println("next_char duration: " + __utime_to_string(*t2)) 3279 //println("next_char duration: " + __utime_to_string(*t2))
3160 - } 3280 +
3161 } 3281 }
3162 } 3282 }
3163 } 3283 }