Anubis / Xlib

Commit cde771f1f9fbb88fc725550f76edd7b6d527314a

Authored by totoro
1 parent 973a3fae

[!] improve speed on receiving files under HTTP server

Showing 1 changed file with 437 additions and 317 deletions   Show diff stats
Inline Side-by-side
web/CXM_multihost_http_server.anubis
  Show/Hide comments   View file @cde771f
... ... @@ -11,8 +11,9 @@
11 11 Cédric Ricard
12 12  
13 13  
14   - *Revised* July 2007.
15   -
  14 + *Revised*
  15 + June 2015 : Optimization of the file upload
  16 + July 2007 : Initial version in calexium_lib
16 17  
17 18  
18 19 *Overviews*
... ... @@ -169,6 +170,7 @@ read tools/printable_tree.anubis
169 170 read system/string.anubis
170 171 read system/files.anubis
171 172 read system/lists.anubis
  173 +read system/data_io.anubis
172 174 read web/mime.anubis
173 175  
174 176  
... ... @@ -843,7 +845,6 @@ define ReadResult
843 845 connection.read_pos <- *connection.read_pos + size_read;
844 846 //accumulate_t1(t1_tmp);
845 847 if size > size_read then
846   - //println("Wanted " + size + ", read only " + size_read);
847 848  
848 849 terminal read_from_connexion(connection, size - size_read, time_out, result_buffer, position+size_read)
849 850 // {
... ... @@ -862,7 +863,6 @@ define ReadResult
862 863 timeout then timeout,
863 864 ok(ba) then
864 865 with size_read = length(ba),
865   - //println("Wanted 32768, read only " + size_read);
866 866 //the needed read is higher than the
867 867 if size > size_read then
868 868 put(ba, result_buffer, position, 0);
... ... @@ -934,7 +934,31 @@ define Result(Error,Word8)
934 934 ok(h)
935 935 }.
936 936  
937   -
  937 +define ByteArray
  938 + get_and_erase_buffer
  939 + (
  940 + BufferedConnection connection,
  941 + SState s
  942 + )=
  943 + with head = to_byte_array(implode(*s.unput_chars)),
  944 + tail = extract(*connection.buffer, *connection.read_pos, length(*connection.buffer)),
  945 +// println("--- get_and_erase_buffer ----");
  946 +// println("unput char length : "+length(to_string(head)));
  947 +// println("connection.read_pos : "+*connection.read_pos);
  948 +// println("connection.buffer length : "+length(*connection.buffer));
  949 +// println(" tail length : "+length(tail));
  950 +// println(" -- unputchar content ");
  951 +// println("["+to_string(head)+"]");
  952 +// println(" -- buffer content ");
  953 +// println("["+to_string(*connection.buffer)+"]");
  954 +// println(" -- tail content ");
  955 +// println("["+to_string(tail)+"]");
  956 + s.unput_chars <- [];
  957 + connection.buffer <- constant_byte_array(0,0);
  958 + connection.read_pos <- 0;
  959 + head + tail
  960 + .
  961 +
938 962  
939 963  
940 964  
... ... @@ -1632,43 +1656,44 @@ define Bool
1632 1656 if ('a' +=< c & c +=< 'z') then true else
1633 1657 if ('A' +=< c & c +=< 'Z') then true else
1634 1658 if ('0' +=< c & c +=< '9') then true else
1635   - if c = '-' then true else
  1659 + if c = '-' then true else
1636 1660 c = '_'.
1637 1661  
1638 1662 define Result(Error,String)
1639   - read_header_name
1640   - (
1641   - BufferedConnection connection,
1642   - Int dead_line,
1643   - List(Word8) so_far,
1644   - DenialOfService dos,
1645   - SState s
1646   - ) =
  1663 + read_header_name
  1664 + (
  1665 + BufferedConnection connection,
  1666 + Int dead_line,
  1667 + List(Word8) so_far,
  1668 + DenialOfService dos,
  1669 + SState s
  1670 + ) =
1647 1671 if next_char(connection, dead_line, dos, s) is
1648 1672 {
1649 1673 error(msg) then error(msg),
1650   - ok(c) then
  1674 + ok(c) then
1651 1675 if is_header_name_char(c)
1652   - then read_header_name(connection,dead_line,[to_lower(c) . so_far],dos, s)
  1676 + then read_header_name(connection, dead_line, [to_lower(c) . so_far], dos, s)
1653 1677 else unput(c, s); ok(implode(reverse(so_far)))
1654 1678 }.
1655 1679  
1656 1680 define Result(Error,One)
1657   - skip_colon
1658   - (
1659   - BufferedConnection connection,
1660   - Int dead_line,
1661   - DenialOfService dos,
1662   - SState s
1663   - ) =
1664   - if skip_http_blanks(connection, dead_line, dos, s) is
1665   - {
1666   - error(msg) then error(msg),
1667   - ok(_) then
1668   - if next_char(connection, dead_line, dos, s) is
1669   - {
1670   - error(msg) then error(msg),
1671   - ok(c) then
  1681 + skip_colon
  1682 + (
  1683 + BufferedConnection connection,
  1684 + Int dead_line,
  1685 + DenialOfService dos,
  1686 + SState s
  1687 + ) =
  1688 + //Skip the blank char until ':'
  1689 + if skip_http_blanks(connection, dead_line, dos, s) is
  1690 + {
  1691 + error(msg) then error(msg),
  1692 + ok(_) then
  1693 + if next_char(connection, dead_line, dos, s) is
  1694 + {
  1695 + error(msg) then error(msg),
  1696 + ok(c) then
1672 1697 if c = ':'
1673 1698 then ok(unique)
1674 1699 else error(colon_expected)
... ... @@ -1718,49 +1743,54 @@ define Result(Error,Maybe(HTTP_header))
1718 1743 DenialOfService dos,
1719 1744 SState s
1720 1745 ) =
1721   - if read_header_name(connection, dead_line, [], dos, s) is
1722   - {
1723   - error(msg) then error(msg),
1724   - ok(name) then
1725   - if name = "" then
1726   - if read_and_ignore(connection, dead_line, 2, dos, s) /* 13 and 10 */ is
1727   - {
1728   - error(msg) then error(msg),
1729   - ok(_) then // this is the blank line
1730   - ok(failure) // end of headers
  1746 + //Find the name
  1747 + if read_header_name(connection, dead_line, [], dos, s) is
  1748 + {
  1749 + error(msg) then error(msg),
  1750 + ok(name) then
  1751 + if name = "" then
  1752 + if read_and_ignore(connection, dead_line, 2, dos, s) /* 13 and 10 */ is
  1753 + {
  1754 + error(msg) then error(msg),
  1755 + ok(_) then // this is the blank line
  1756 + ok(failure) // end of headers
  1757 + }
  1758 + //skip the ':' and blank before and after it
  1759 + else if skip_colon(connection, dead_line, dos, s) is
  1760 + {
  1761 + error(msg) then error(msg),
  1762 + ok(_) then
  1763 + //skip the blank char after the ':'
  1764 + if skip_http_blanks(connection, dead_line, dos, s) is
  1765 + {
  1766 + error(msg) then error(msg),
  1767 + ok(_) then
  1768 + //Now read the value
  1769 + if read_header_value(connection, dead_line, [], dos, s) is
  1770 + {
  1771 + error(msg) then error(msg),
  1772 + ok(value) then ok(success(http_header(name,value)))
1731 1773 }
1732   - else if skip_colon(connection, dead_line, dos, s) is
1733   - {
1734   - error(msg) then error(msg),
1735   - ok(_) then if skip_http_blanks(connection, dead_line, dos, s) is
1736   - {
1737   - error(msg) then error(msg),
1738   - ok(_) then if read_header_value(connection, dead_line, [], dos, s) is
1739   - {
1740   - error(msg) then error(msg),
1741   - ok(value) then
1742   - ok(success(http_header(name,value)))
1743   - }
1744   - }
1745   - }
1746   - }.
  1774 + }
  1775 + }
  1776 + }.
1747 1777  
1748 1778  
1749 1779  
1750 1780 Reading all the headers.
1751 1781  
1752 1782 define Result(Error,List(HTTP_header))
1753   - read_http_headers
1754   - (
1755   - BufferedConnection connection,
1756   - Int dead_line,
1757   - DenialOfService dos,
1758   - SState s
1759   - ) =
1760   - if read_header(connection, dead_line, dos, s) is
1761   - {
1762   - error(msg) then error(msg),
1763   - ok(mbh) then if mbh is
  1783 + read_http_headers
  1784 + (
  1785 + BufferedConnection connection,
  1786 + Int dead_line,
  1787 + DenialOfService dos,
  1788 + SState s
  1789 + ) =
  1790 + if read_header(connection, dead_line, dos, s) is
  1791 + {
  1792 + error(msg) then error(msg),
  1793 + ok(mbh) then if mbh is
1764 1794 {
1765 1795 failure then ok([ ]),
1766 1796 success(header) then
... ... @@ -1784,21 +1814,21 @@ define Result(Error,List(HTTP_header))
1784 1814 header is not present, the size is assumed to be zero.
1785 1815  
1786 1816 define Result(Error,Int)
1787   - get_body_size
1788   - (
1789   - List(HTTP_header) headers
1790   - ) =
1791   - if headers is
1792   - {
1793   - [ ] then ok(0),
1794   - [h . t] then if h is http_header(name,value) then
1795   - if name = "content-length"
1796   - then if decimal_scan(value) is
1797   - {
1798   - failure then error(incorrect_content_length_value),
1799   - success(n) then ok(n)
1800   - }
1801   - else get_body_size(t)
  1817 + get_body_size
  1818 + (
  1819 + List(HTTP_header) headers
  1820 + ) =
  1821 + if headers is
  1822 + {
  1823 + [ ] then ok(0),
  1824 + [h . t] then if h is http_header(name,value) then
  1825 + if name = "content-length"
  1826 + then if decimal_scan(value) is
  1827 + {
  1828 + failure then error(incorrect_content_length_value),
  1829 + success(n) then ok(n)
  1830 + }
  1831 + else get_body_size(t)
1802 1832 }.
1803 1833  
1804 1834  
... ... @@ -1820,17 +1850,18 @@ define Result(Error,Int)
1820 1850 required number of bytes. However, if the number of bytes read is zero, the connection
1821 1851 may be broken. In that case, we must not try to read indefinitely. On the contrary, we
1822 1852 make at most 10 retries, with a small sleeping time between any two of them.
1823   -
  1853 +
1824 1854 define Result(Error, ByteArray)
1825 1855 read_http_body
1826 1856 (
1827 1857 BufferedConnection connection,
1828   - Int body_size,
1829   - ByteArray so_far, // when calling this function, 'so_far' is the empty byte array
1830   - Int retries // this function is called with retries = 10
  1858 + Int body_size,
  1859 + ByteArray so_far, // when calling this function, 'so_far' is the empty byte array
  1860 + Int retries // this function is called with retries = 10
1831 1861 ) =
1832 1862 if body_size = 0 then ok(constant_byte_array(0,0)) else
1833 1863 if retries =< 0 then error(cannot_read_from_connection) else
  1864 +
1834 1865 if read_from_connexion(connection,body_size,60,constant_byte_array(body_size,0),0) is
1835 1866 {
1836 1867 error then error(cannot_read_from_connection),
... ... @@ -2016,19 +2047,19 @@ public define Bool
2016 2047  
2017 2048  
2018 2049 define Maybe(MIME)
2019   - recognize_mime_type_from_ext
2020   - (
2021   - String ext,
2022   - List(MIME) l
2023   - ) =
2024   - if l is
2025   - {
2026   - [ ] then success(mime("application", "octet-stream", [])), // failure,
2027   - [h . t] then if h is mime(type, subtype, extensions) then
2028   - if contains_no_case(extensions, ext)
2029   - then success(h)
2030   - else recognize_mime_type_from_ext(ext,t)
2031   - }.
  2050 + recognize_mime_type_from_ext
  2051 + (
  2052 + String ext,
  2053 + List(MIME) l
  2054 + ) =
  2055 + if l is
  2056 + {
  2057 + [ ] then success(mime("application", "octet-stream", [])), // failure,
  2058 + [h . t] then if h is mime(type, subtype, extensions) then
  2059 + if contains_no_case(extensions, ext)
  2060 + then success(h)
  2061 + else recognize_mime_type_from_ext(ext,t)
  2062 + }.
2032 2063  
2033 2064 define Maybe(MIME)
2034 2065 recognize_mime_type_from_uri
... ... @@ -2038,13 +2069,8 @@ define Maybe(MIME)
2038 2069 ) =
2039 2070 recognize_mime_type_from_ext(get_uri_extension(uri),known_mime_types(desc)).
2040 2071  
2041   -
2042 2072  
2043 2073  
2044   -
2045   -
2046   -
2047   -
2048 2074 *** [5.4] Formating HTTP headers.
2049 2075  
2050 2076 This is the formating for sending to the client (hence, it has nothing to do with the
... ... @@ -2187,11 +2213,11 @@ define String
2187 2213 Maybe(FileTimes) mb_ftimes,
2188 2214 Int size,
2189 2215 ) =
2190   - if mb_ftimes is
2191   - {
2192   - failure then println("Warning: no file times for '" + filename + "', etag won't be very accurate."); to_ascii(sha1((filename, size))),
2193   - success(ftimes) then to_ascii(md5((filename, ftimes, size)))
2194   - }.
  2216 + if mb_ftimes is
  2217 + {
  2218 + failure then println("Warning: no file times for '" + filename + "', etag won't be very accurate."); to_ascii(sha1((filename, size))),
  2219 + success(ftimes) then to_ascii(md5((filename, ftimes, size)))
  2220 + }.
2195 2221  
2196 2222 define Bool
2197 2223 are_same_etag
... ... @@ -2243,18 +2269,18 @@ define One
2243 2269 Checking if a connection is under SSL.
2244 2270  
2245 2271 define Bool
2246   - is_SSL
2247   - (
2248   - Connection c
2249   - ) =
2250   - if c is
2251   - {
2252   - file_r(_) then false,
2253   - file_w(_) then false,
2254   - file_rw(_) then false,
2255   - tcp(_) then false,
2256   - ssl(_) then true
2257   - }.
  2272 + is_SSL
  2273 + (
  2274 + Connection c
  2275 + ) =
  2276 + if c is
  2277 + {
  2278 + file_r(_) then false,
  2279 + file_w(_) then false,
  2280 + file_rw(_) then false,
  2281 + tcp(_) then false,
  2282 + ssl(_) then true
  2283 + }.
2258 2284  
2259 2285  
2260 2286  
... ... @@ -2527,10 +2553,10 @@ define Maybe(String)
2527 2553 }.
2528 2554  
2529 2555 define Maybe(String)
2530   - get_boundary
2531   - (
2532   - List(HTTP_header) headers
2533   - ) =
  2556 + get_boundary
  2557 + (
  2558 + List(HTTP_header) headers
  2559 + )=
2534 2560 if headers is
2535 2561 {
2536 2562 [ ] then failure,
... ... @@ -2586,13 +2612,13 @@ define Maybe(String)
2586 2612 argument.
2587 2613  
2588 2614 define Maybe(Int)
2589   - find
2590   - (
2591   - String what,
2592   - ByteArray where,
2593   - Int start,
2594   - Int end
2595   - ) =
  2615 + find
  2616 + (
  2617 + String what,
  2618 + ByteArray where,
  2619 + Int start,
  2620 + Int end
  2621 + ) =
2596 2622 if find(to_byte_array(what),where,start) is
2597 2623 {
2598 2624 failure then failure,
... ... @@ -2601,7 +2627,23 @@ define Maybe(Int)
2601 2627 then failure
2602 2628 else success(n)
2603 2629 }.
2604   -
  2630 +
  2631 +define Maybe(Int)
  2632 + find
  2633 + (
  2634 + String what,
  2635 + String where,
  2636 + Int start,
  2637 + Int end
  2638 + ) =
  2639 + if find_string(where,what,start) is
  2640 + {
  2641 + failure then failure,
  2642 + success(n) then
  2643 + if n+length(what) >= end
  2644 + then failure
  2645 + else success(n)
  2646 + }.
2605 2647  
2606 2648 define String
2607 2649 read_attribute_value
... ... @@ -2642,26 +2684,26 @@ define Maybe(String)
2642 2684  
2643 2685  
2644 2686 define Maybe((String,Maybe(String)))
2645   - find_name_and_filename
2646   - (
2647   - ByteArray body,
2648   - Int start,
2649   - Int end
2650   - ) =
2651   - if find(to_byte_array("Content-Disposition"),body,start) is
2652   - {
2653   - failure then failure,
2654   - success(n) then
2655   - if find_attribute("name",body,n+19,end) is
2656   - {
2657   - failure then failure,
2658   - success(name_value) then if find_attribute("filename",body,n+19,end) is
2659   - {
2660   - failure then success((name_value,failure)),
2661   - success(filename_value) then success((name_value,success(filename_value)))
2662   - }
2663   - }
2664   - }.
  2687 + find_name_and_filename
  2688 + (
  2689 + ByteArray body,
  2690 + Int start,
  2691 + Int end
  2692 + ) =
  2693 + if find(to_byte_array("Content-Disposition"),body,start) is
  2694 + {
  2695 + failure then failure,
  2696 + success(n) then
  2697 + if find_attribute("name",body,n+19,end) is
  2698 + {
  2699 + failure then failure,
  2700 + success(name_value) then if find_attribute("filename",body,n+19,end) is
  2701 + {
  2702 + failure then success((name_value,failure)),
  2703 + success(filename_value) then success((name_value,success(filename_value)))
  2704 + }
  2705 + }
  2706 + }.
2665 2707  
2666 2708  
2667 2709  
... ... @@ -2697,27 +2739,38 @@ define Maybe((String,Maybe(String)))
2697 2739 define Maybe(String) // returns the temporary file name
2698 2740 save_uploaded_file
2699 2741 (
2700   - Web_Site_Description desc,
2701   - ByteArray body,
2702   - Int start,
2703   - Int end,
2704   - SState s
  2742 + Web_Site_Description desc,
  2743 + RStream body_fd,
  2744 +// ByteArray body,
  2745 + Int start,
  2746 + Int end,
  2747 + SState s
2705 2748 ) =
2706 2749 s.uploaded_file_count <- 1 + *(s.uploaded_file_count);
2707 2750 with tfn = "_"+to_decimal(virtual_machine_id)+"_"+to_decimal(*(s.uploaded_file_count)),
2708   - if (Maybe(RWStream))file(site_directory(desc)+"/upload_temporary/"+tfn, new) is
2709   - {
2710   - failure then failure,
2711   - success(f) then
2712   - if reliable_write(file(f),extract(body,start,end)) is
2713   - {
2714   - failure then failure,
2715   - success(nw) then
2716   - if nw = end - start
2717   - then success(tfn)
2718   - else failure
2719   - }
2720   - }.
  2751 +// println("save_uploaded_file to :"+site_directory(desc)+"/upload_temporary/"+tfn);
  2752 +// println("start offset = "+start+" end offset = "+end);
  2753 + //make data_io which is the size of the file to extract
  2754 + if copy_Data_IO_to_file(make_data_io(body_fd, start, end - start), site_directory(desc)+"/upload_temporary/"+tfn) is copy_ok(_)
  2755 + then success(tfn)
  2756 + else failure
  2757 + .
  2758 +
  2759 +
  2760 +
  2761 +// if (Maybe(RWStream))file(site_directory(desc)+"/upload_temporary/"+tfn, new) is
  2762 +// {
  2763 +// failure then failure,
  2764 +// success(f) then
  2765 +// if reliable_write(file(f),extract(body,start,end)) is
  2766 +// {
  2767 +// failure then failure,
  2768 +// success(nw) then
  2769 +// if nw = end - start
  2770 +// then success(tfn)
  2771 +// else failure
  2772 +// }
  2773 +// }.
2721 2774  
2722 2775  
2723 2776  
... ... @@ -2763,94 +2816,114 @@ define String
2763 2816 *** [5.7.6] Reading a multipart entity.
2764 2817  
2765 2818 define Maybe(Web_arg)
2766   - get_multipart_entity
2767   - (
2768   - Web_Site_Description desc,
2769   - ByteArray body,
2770   - Int start,
2771   - Int end,
2772   - SState s
2773   - ) =
2774   - if find(to_byte_array(crlf+crlf),body,start) is
2775   - {
2776   - failure then failure,
2777   - success(k) then
2778   - if k >= end // must be within this entity, not the next one
2779   - then failure
2780   - else if find_name_and_filename(body,start,k) is
2781   - {
2782   - failure then failure,
2783   - success(n_mbfn) then if n_mbfn is (name,mbfn) then
  2819 + get_multipart_entity
  2820 + (
  2821 + Web_Site_Description desc,
  2822 + String body_temp_file,
  2823 + Int start_offset, //real offset in source file of the part
  2824 + Int end_offset, //real offset in source file of the part
  2825 + SState s
  2826 + )=
  2827 + //Get header of the part
  2828 + if find_the_first(body_temp_file, crlf+crlf, start_offset, end_offset) is
  2829 + {
  2830 + failure then failure,
  2831 + success(k) then
  2832 + if (Maybe(RStream))file(body_temp_file, read) is
  2833 + {
  2834 + failure then failure,
  2835 + success(body_fd) then
  2836 + if read_bytes(make_data_io(body_fd, start_offset, k), k) is success(header_part) then
  2837 +
  2838 + if find_name_and_filename(header_part, 0, k) is
  2839 + {
  2840 + failure then failure,
  2841 + success(n_mbfn) then if n_mbfn is (name, mbfn) then
2784 2842 if mbfn is
2785   - {
  2843 + {
2786 2844 failure then
2787   - success(web_arg(name,to_string(extract(body,k+4,end-2)))),
  2845 + if read_bytes(make_data_io(body_fd, start_offset+k+4, (end_offset-2)-(start_offset+k+4)), (end_offset-2)-(start_offset+k+4)) is success(attachement) then
  2846 + success(web_arg(name,to_string(attachement)))
2788 2847 // we must substract 2 to end because of crlf just before the boundary
  2848 + else
  2849 + failure,
2789 2850  
2790 2851 success(fn) then
2791   - if save_uploaded_file(desc, body, k+4, end-2, s) is
2792   - {
2793   - failure then failure,
2794   - success(tfn) then
2795   - success(upload(name,remove_path(fn),
2796   - site_directory(desc)+"/upload_temporary/"+tfn))
  2852 + if save_uploaded_file(desc, body_fd, start_offset+k+4, end_offset-2, s) is
  2853 + {
  2854 + failure then failure,
  2855 + success(tfn) then success(upload(name, remove_path(fn), site_directory(desc)+"/upload_temporary/"+tfn))
2797 2856  
2798 2857 }
2799 2858 }
2800 2859 }
2801   - }.
  2860 + else
  2861 + failure
  2862 + }
  2863 + }.
2802 2864  
2803 2865  
2804 2866  
2805 2867 define List(Web_arg)
2806   - read_multipart_form_data_encoded_web_args
2807   - (
2808   - Web_Site_Description desc,
2809   - ByteArray body,
2810   - ByteArray __boundary,
2811   - Int i,
2812   - SState s
2813   - ) =
2814   - if find(__boundary,body,i) is
2815   - {
2816   - failure then [ ],
2817   - success(n) then
2818   - if find(__boundary,body,n+length(__boundary)) is
2819   - {
2820   - failure then [ ],
2821   - success(m) then
2822   - if get_multipart_entity(desc, body, n+length(__boundary), m, s) is
2823   - {
2824   - failure then [ ],
2825   - success(wa) then
2826   - [wa . read_multipart_form_data_encoded_web_args(desc, body, __boundary, m, s)]
2827   - }
2828   - }
2829   - }.
  2868 + read_multipart_form_data_encoded_web_args
  2869 + (
  2870 + Web_Site_Description desc,
  2871 + String body_temp_file,
  2872 + String __boundary,
  2873 + Int file_offset,
  2874 + SState s
  2875 + ) =
  2876 + with boundary_length = length(__boundary),
  2877 +// println(
  2878 +//"read_multipart_form_data_encoded_web_args
  2879 +// boundary["+__boundary+"]
  2880 +// body_temp_file : "+body_temp_file+"
  2881 +// file_offset : "+file_offset);
  2882 + //get the first boundary position
  2883 + if find_the_first(body_temp_file, __boundary, file_offset) is
  2884 + {
  2885 + failure then println("first boundary not found");[ ],
  2886 + success(first) then
  2887 + with first = first + file_offset, //adjust the offset to real offset in file
  2888 + //get the second boundary position
  2889 + if find_the_first(body_temp_file, __boundary, first + boundary_length) is
  2890 + {
  2891 + failure then println("last boundary found");[ ],
  2892 + success(last) then
  2893 + with last = last + first + boundary_length, //adjust the offset to real offset in file
  2894 + //Extract the file content here
  2895 + if get_multipart_entity(desc, body_temp_file, first+boundary_length, last, s) is
  2896 + {
  2897 + failure then [ ],
  2898 + success(wa) then
  2899 + [wa . read_multipart_form_data_encoded_web_args(desc, body_temp_file, __boundary, last, s)]
  2900 + }
  2901 + }
  2902 + }.
2830 2903  
2831 2904  
2832 2905  
2833 2906 define One
2834 2907 multipart_form_data_answer
2835 2908 (
2836   - String host_name,
2837   - Web_Site_Description desc,
2838   - Connection connection,
2839   - Word32 ip_addr,
2840   - HTTP_RequestLine request_line,
2841   - List(HTTP_header) headers,
2842   - ByteArray body,
2843   - One -> String generate_tt,
2844   - SState s
  2909 + String host_name,
  2910 + Web_Site_Description desc,
  2911 + Connection connection,
  2912 + Word32 ip_addr,
  2913 + HTTP_RequestLine request_line,
  2914 + List(HTTP_header) headers,
  2915 + String body_temp_file,
  2916 + One -> String generate_tt,
  2917 + SState s
2845 2918 ) =
2846   - if get_boundary(headers) is
2847   - {
2848   - failure then unique,
2849   - success(boundary) then
2850   - with all_web_args = query_string(request_line) +
  2919 + if get_boundary(headers) is
  2920 + {
  2921 + failure then unique,
  2922 + success(boundary) then
  2923 + with all_web_args = query_string(request_line) +
2851 2924 read_multipart_form_data_encoded_web_args(desc,
2852   - body,
2853   - to_byte_array("--"+boundary),
  2925 + body_temp_file,
  2926 + "--"+boundary,
2854 2927 0,
2855 2928 s),
2856 2929 uri = uri(request_line),
... ... @@ -2967,47 +3040,46 @@ define String
2967 3040 is achieved through the header 'Content-Type'.
2968 3041  
2969 3042 define EncodingType
2970   - get_encoding_type
2971   - (
2972   - List(HTTP_header) headers
2973   - ) =
2974   - if headers is
2975   - {
2976   - [ ] then www_url, // this is the default
2977   - [h . t] then if h is http_header(name,value) then
2978   - if name = "content-type"
2979   - then if find("multipart/form-data",value,0) is
2980   - {
2981   - failure then www_url,
2982   - success(_) then multipart_form_data
2983   - }
2984   - else get_encoding_type(t)
  3043 + get_encoding_type
  3044 + (
  3045 + List(HTTP_header) headers
  3046 + ) =
  3047 + if headers is
  3048 + {
  3049 + [ ] then www_url, // this is the default
  3050 + [h . t] then if h is http_header(name, value) then
  3051 + if name = "content-type"
  3052 + then if find("multipart/form-data",value,0) is
  3053 + {
  3054 + failure then www_url,
  3055 + success(_) then multipart_form_data
  3056 + }
  3057 + else
  3058 + get_encoding_type(t) //content-type not found, check the next header line
2985 3059 }.
2986 3060  
2987 3061  
2988 3062  
2989   -define One
2990   - send_answer
2991   - (
2992   - String host_name,
2993   - Web_Site_Description desc,
2994   - Connection connection,
2995   - HTTP_RequestLine rqline,
2996   - List(HTTP_header) headers,
2997   - ByteArray body,
2998   - One -> String generate_tt,
2999   - SState s
3000   - ) =
3001   - if rqline is request_line(type,uri,qstring) then
3002   - with rqline2 = request_line(type,handle_redirection(redirections(desc),uri,headers),qstring),
3003   - if remote_IP_address_and_port(connection) is (ip_addr,_) then
3004   - if get_encoding_type(headers) is
3005   - {
3006   - www_url then
3007   - www_url_answer(host_name,desc,connection,ip_addr,rqline2,headers,body,generate_tt),
3008   - multipart_form_data then
3009   - multipart_form_data_answer(host_name,desc,connection,ip_addr,rqline2,headers,body,generate_tt, s)
3010   - }.
  3063 + define One
  3064 + send_answer
  3065 + (
  3066 + String host_name,
  3067 + Web_Site_Description desc,
  3068 + Connection connection,
  3069 + HTTP_RequestLine rqline,
  3070 + List(HTTP_header) headers,
  3071 + ByteArray body,
  3072 + One -> String generate_tt,
  3073 + SState s
  3074 + )=
  3075 + if rqline is request_line(type, uri, qstring) then
  3076 + with rqline2 = request_line(type, handle_redirection(redirections(desc), uri, headers), qstring),
  3077 + if remote_IP_address_and_port(connection) is (ip_addr,_) then
  3078 + if get_encoding_type(headers) is
  3079 + {
  3080 + www_url then www_url_answer(host_name, desc, connection, ip_addr, rqline2, headers, body, generate_tt),
  3081 + multipart_form_data then multipart_form_data_answer(host_name, desc, connection, ip_addr, rqline2, headers, body, generate_tt, s)
  3082 + }.
3011 3083  
3012 3084  
3013 3085  
... ... @@ -3110,54 +3182,102 @@ define One
3110 3182 SState s
3111 3183 ) =
3112 3184 //t0 <- (UTime)unow;
3113   - with start_time = (Int)now,
3114   - s.sttm <- start_time;
3115   - //println("Request time: " + format_http_date(start_time));
3116   - if dos is denial_of_service(mc_v,rld_v,hd_v,ad_v,ld_v,ra_v) then
3117   - if remote_IP_address_and_port(connection.conn) is (ip_addr,port) then
3118   - if read_request_line(connection, start_time+*rld_v, dos, s) is
3119   - {
3120   - error(msg) then print(format(msg)),
3121   - ok(request_line) then
3122   - //print_delta("read_request_line");
3123   - if read_http_headers(connection, start_time+*hd_v, dos, s) is
3124   - {
3125   - error(msg) then print(format(msg)),
3126   - ok(headers) then //print_delta("read_http_headers");
3127   - if get_site(headers,sites) is
3128   - {
3129   - failure then unique,
3130   - success(p) then if p is (host_name,desc) then
3131   - //print_delta("get_site");
3132   - if get_body_size(headers) is
3133   - {
3134   - error(msg) then log_journal_msg(desc,format(msg)),
3135   - ok(body_size) then
  3185 + with start_time = (Int)now,
  3186 + s.sttm <- start_time;
  3187 + //println("Request time: " + format_http_date(start_time));
  3188 + if dos is denial_of_service(mc_v,rld_v,hd_v,ad_v,ld_v,ra_v) then
  3189 + if remote_IP_address_and_port(connection.conn) is (ip_addr,port) then
  3190 + if read_request_line(connection, start_time+*rld_v, dos, s) is
  3191 + {
  3192 + error(msg) then print(format(msg)),
  3193 + ok(rqline) then //request line
  3194 + //print_delta("read_request_line");
  3195 + if read_http_headers(connection, start_time+*hd_v, dos, s) is
  3196 + {
  3197 + error(msg) then print(format(msg)),
  3198 + ok(headers) then //print_delta("read_http_headers");
  3199 + if get_site(headers,sites) is
  3200 + {
  3201 + failure then unique,
  3202 + success(p) then if p is (host_name, desc) then
  3203 + //Body Size
  3204 + if get_body_size(headers) is
  3205 + {
  3206 + error(msg) then log_journal_msg(desc,format(msg)),
  3207 + ok(body_size) then
  3208 + if rqline is request_line(type, uri, qstring) then
  3209 + with rqline2 = request_line(type, handle_redirection(redirections(desc), uri, headers), qstring),
  3210 +
  3211 + //Get the type of encoding which decide if we read the content in ByteArray for www_url or in
  3212 + //temporary file for multipart.
  3213 + with generate_tt = make_generate_trust_ticket(dos),
  3214 + if get_encoding_type(headers) is
  3215 + {
  3216 + //WWW_URL
  3217 + www_url then
  3218 + if read_http_body(connection,body_size,constant_byte_array(0,0),1000) is
  3219 + {
  3220 + error(msg) then log_journal_msg(desc,format(msg)),
  3221 + ok(body) then
  3222 + www_url_answer(host_name, desc, connection.conn, ip_addr, rqline2, headers, body, generate_tt);
  3223 + //it's HTTP 1.1 keep-alive is default
  3224 + http_https_handler(sites, connection, is_https, dos, s)
  3225 + }
  3226 + //MULTIPART_FORM_DATA
  3227 + multipart_form_data then
  3228 +
  3229 + if body_size > 0 then
  3230 + with t0 = (UTime)unow,
  3231 + println("read body, size "+body_size);
  3232 +
  3233 + if get_socket_from_connection(conn(connection)) is
  3234 + {
  3235 + failure then println("can't get socket"),
  3236 + success(source) then
3136 3237  
3137   - with t0 = (UTime)unow,
3138   - (if body_size > 0 then
3139   - println("read body, size "+body_size)
3140   - else
3141   - unique);
3142   - if read_http_body(connection,body_size,constant_byte_array(0,0),1000) is
3143   - {
3144   - error(msg) then log_journal_msg(desc,format(msg)),
3145   - ok(body) then
3146   - with duration = (UTime) unow - t0,
3147   - (if body_size > 0 then
3148   - println("Read body "+body_size+" duration: " + __utime_to_string(duration))
3149   - else
3150   - unique);
  3238 + with tmp_body_file = "/temp/" + virtual_machine_id + "-" + now,
  3239 + if (Maybe(RWStream))file(desc.site_directory + tmp_body_file, new) is
  3240 + {
  3241 + failure then println("can't create target file "+desc.site_directory + tmp_body_file),//nothing to write
  3242 + success(target) then
  3243 +
  3244 + //get the content of the current buffer and unput char list
  3245 + with buffer = get_and_erase_buffer(connection, s),
  3246 + buffer_size = length(buffer),
  3247 + //println("buffer Size = "+buffer_size);
  3248 + //println("Old body size "+body_size+" New body size request = "+body_size - buffer_size);
  3249 + if flush(buffer, weaken(target)) is
  3250 + {
  3251 + failure then println("Can't flush the buffer"),
  3252 + success(_) then
  3253 + if copy_file(weaken(source), weaken(target), body_size - buffer_size) is copy_ok(read_size) then
  3254 +
  3255 + with duration = (UTime) unow - t0,
  3256 + println("Read body "+read_size+" duration: " + __utime_to_string(duration));
  3257 +
  3258 + multipart_form_data_answer(host_name, desc, connection.conn, ip_addr, rqline2, headers, desc.site_directory + tmp_body_file, generate_tt, s);
  3259 +
  3260 + //it's HTTP 1.1 keep-alive is default
  3261 + http_https_handler(sites, connection, is_https, dos, s)
  3262 +
  3263 + else
  3264 + println("Can't copy data from stream to temporary file ")
  3265 + }}}
  3266 + else
  3267 + println("bbody_size = 0 !")
  3268 + }
  3269 +
3151 3270 //print_delta("before send_answer");
3152   - send_answer(host_name, desc,connection.conn, request_line, headers, body,
3153   - make_generate_trust_ticket(dos), s);
3154   - //it's HTTP 1.1 keep-alive is default
3155   - http_https_handler(sites, connection, is_https, dos, s)
  3271 +// with body = constant_byte_array(0,0),
  3272 +// send_answer(host_name, desc,connection.conn, request_line, headers, body,
  3273 +// make_generate_trust_ticket(dos), s);
  3274 +// //it's HTTP 1.1 keep-alive is default
  3275 +// http_https_handler(sites, connection, is_https, dos, s)
3156 3276 //with duration = (UTime) unow - *t0,
3157 3277 //println("Request duration: " + __utime_to_string(duration))
3158 3278 //println("BufferRead duration: " + __utime_to_string(*t1));
3159 3279 //println("next_char duration: " + __utime_to_string(*t2))
3160   - }
  3280 +
3161 3281 }
3162 3282 }
3163 3283 }
... ...