-

Alain Prouté
1 parent e70f9a3c
Showing 13 changed files with 83 additions and 54 deletions Show diff stats
anubis_dev/compiler/src/predef.c
anubis_dev/library/lexical_analysis/fast_lexer.anubis
anubis_dev/library/test/predefined/fast_lexer.unit_test.anubis
anubis_distrib/library/examples/OOCounter.anubis.cpp
anubis_distrib/library/examples/logic/peano.anubis
anubis_distrib/library/examples/mathematics/runge_kutta.anubis
anubis_distrib/library/lexical_analysis/fast_lexer.anubis
anubis_distrib/library/lexical_analysis/fast_lexer_example_1.anubis
anubis_distrib/library/predefined.anubis
anubis_distrib/library/test/predefined/fast_lexer.unit_test.anubis
anubis_distrib/library/web/making_a_web_site.anubis
anubis_distrib/library/web/mime.anubis
anubis_distrib/library/web/multihost_http_server.anubis
@@ -185,7 +185,7 @@ void do_predefinitions1(void)
  
 void do_predefinitions2(void)
 {
-  /* include the following file produced by: 'anubis -predef' */ 
+  /* include the following file produced by: 'myanubis -predef' */ 
 #ifdef _no_predef_dat_
 #include "predef_npd.aux" 
 #endif   
@@ -1397,7 +1397,7 @@ define LexerOutput($Token)
  
        /* almost the same thing for accepted */
        accepted(s,a,start,end) then 
-   print("low level accepted start = "+to_decimal(start)+" end = "+to_decimal(end)+"\n");
+   //print("low level accepted start = "+to_decimal(start)+" end = "+to_decimal(end)+"\n");
          last_accept_v <- last(s,end); 
          current_v <- end; 
          if a is 
@@ -106,6 +106,9 @@ define One
    assertIsSame(ut,
                 test_fast_lexer(lexer_1,to_byte_array("  gggabuzobof"),none,0,0,0),
                 accepted(6,not_at_end_of_input,2,10),    "6ter"); 
+   assertIsSame(ut,
+                test_fast_lexer(lexer_1,to_byte_array("   "),none,0,0,0),
+                ignored(7,3),    "7"); 
  
    unique. 
  
@@ -61,7 +61,7 @@ class OOCounter
     int v;
 };
  
-void main(void)
+int main(void)
 {
     OOCounter toto(4);
  
@@ -3,11 +3,11 @@
  
  *Title*                      Playing with Peano numbers. 
  
- *Copyright*                Copyright (c) Alain Prouté 2005.
+ *Copyright*                Copyright (c) Alain Proutï¿½ 2005.
  
  *Released*  
  
- *Author* Alain Prouté
+ *Author* Alain Proutï¿½
  
  
    This  file  contains  several  classical  exercices  on  Peano  numbers  and  recursive
@@ -325,7 +325,7 @@ global define One
      (
        List(String) args
      ) =
-   with primes = first_primes(_100), 
+   with primes = first_primes(_500), 
    forget(map((Peano p) |-> print(to_string(p)+" "),reverse(primes)));
    print("\n"). 
  
@@ -45,8 +45,8 @@ define Maybe(Float)
        Maybe(Float)  x, 
        Maybe(Float)  y
      ) =   
-   sin(x*y). 
    x - y*y.
+   sin(x*y). 
    _3 * ((y*y)^(_1/_3)).
    tan(y). 
    x*x - y*y.  
@@ -1404,7 +1404,7 @@ define LexerOutput($Token)
          {
            not_at_end_of_input then 
              /* the lexeme just read must be accepted: the action is applied */
-               update_start_line_col(*buffer_v,start,start_v,line_v,col_v); 
+               update_start_line_col(*buffer_v,end,start_v,line_v,col_v); 
                last_accept_v <- none; 
                (*actions(word32(s,0)))(extract(*buffer_v,start,end),*line_v,*col_v),
  
@@ -125,7 +125,7 @@ define Maybe(LexingStream)
        failure     then failure, 
        success(f)  then make_lexing_stream("",      /* no preambule */ 
                                            f,       /* the opened file */
-                                           200,      /* size of buffer for the lexing stream */
+                                           1000,      /* size of buffer for the lexing stream */
                                            100)     /* timeout (seconds) */
      }. 
  
@@ -4402,16 +4402,15 @@ public type FastLexerState:
                 transition('z',1)]),
  
      /* state 1 */
-     accepting([transition(' ',2),transition('\t',2),transition('\n',2),
-                transition('a',1),
+     accepting([transition('a',1),
                 ...
                 transition('z',1)]),   
      /* state 2 */
      ignoring([])
    ]
  
-   Notice that a state is identified by its rank in the list. The starting state is always
-   state 0. Also notice that ignoring states may have transitions. Indeed, if we want to
+   Notice that a state is identified by its rank in the list. The starting state is always state 0. 
+   Also notice that ignoring states may have transitions. Indeed, if we want to
    detect sequences of at least two 'a' and ignore anything else, we may have an automaton
    like this one:
  
@@ -4446,9 +4445,8 @@ public type AtEndOfInput:
    not_at_end_of_input,
    at_end_of_input. 
  
-   Each (low level) fast lexer is actually a function taking a byte array as a source of
-   bytes, a position (of type 'Int') within this byte array, and the name of a starting
-   state, and returning a result of the following type:
+   Each (low level) fast lexer is actually a function taking several arguments (see below)
+   and returning a result of the following type:
  
 public type FastLexerOutput:
    rejected(Word16 where, AtEndOfInput, Int start, Int end),
@@ -4466,7 +4464,7 @@ public type FastLexerOutput:
                the last accepted token within the input byte array.
  
        (2) there is no last accepted position: it returns: rejected(w,a,s,e) where 
-               's' and 'e' are the starting and end positions of the recognized prefix. 
+               's' and 'e' are the starting and end positions of what has been read. 
  
    If 'a' is 'not_at_end_of_input', the above must be interpreted textually, i.e.
    'accepted' means accepted, and 'rejected' means rejected.  However, if 'a' is
@@ -21,41 +21,54 @@ define FastLexerOutput
         List(FastLexerState)        lexer,
         ByteArray                   input, 
         FastLexerLastAccepted       last_accepted,
-        Int                         position,
+        Int                         position,         
+        Int                         start, 
         Word16                      state
      ) =
    if make_fast_lexer(lexer) is 
      {
-       unknown_state(n) then print("fast lexer unknown state: "+to_decimal(n)+"\n"); alert, 
-       too_many_states  then print("too many states.\n"); alert,
-       ok(fl) then with result = fl(input,last_accepted,position,state),
-         
+       unknown_state(n) then print("fast lexer unknown state: "+to_decimal(n)+"\n"); should_not_happen(ignored(0,0)), 
+       too_many_states  then print("too many states.\n"); should_not_happen(ignored(0,0)),
+       ok(fl) then with result = fl(input,last_accepted,position,start,state),
+         print("Result = "); 
          if result is 
            {
-             rejected(w,e,a) then 
-               print("\nrejected("+to_decimal(w)+","+abs_to_decimal(e)+","+format(a)+")\n"),
-             accepted(w,e,a,_) then 
-               print("\naccepted("+to_decimal(w)+","+abs_to_decimal(e)+","+format(a)+")\n")
+             rejected(w,eoi,s,e) then 
+               print("rejected("+to_decimal(w)+","
+                                  +format(eoi)+","
+                                  +abs_to_decimal(s)+","
+                                  +abs_to_decimal(e)
+                                  +")\n"),
+             accepted(w,eoi,s,e) then 
+               print("accepted("+to_decimal(w)+","
+                                  +format(eoi)+","
+                                  +abs_to_decimal(s)+","
+                                  +abs_to_decimal(e)
+                                  +")\n"),
+             ignored(w,s) then 
+               print("ignored("+to_decimal(w)+","
+                                  +abs_to_decimal(s)
+                                  +")\n")       
            };
-         
          result
      }. 
  
  
  
-   A lexer accepting "g+abu", "g+abuzo",
+   A lexer accepting "g+abu", "g+abuzo", and ignoring blanks
  
 define List(FastLexerState)   
    lexer_1 
      =
    [
-     /* state 0  */ rejecting([transition('g',1)]),
+     /* state 0  */ rejecting([transition('g',1),transition(' ',7)]),
      /* state 1  */ rejecting([transition('a',2),transition('g',1)]),
      /* state 2  */ rejecting([transition('b',3)]),
      /* state 3  */ rejecting([transition('u',4)]),
      /* state 4  */ accepting([transition('z',5)]),
      /* state 5  */ rejecting([transition('o',6)]),
-     /* state 6  */ accepting([ ])
+     /* state 6  */ accepting([ ]),
+     /* state 7  */ ignoring([ ])
    ]. 
  
  
@@ -67,32 +80,39 @@ define One
  
    /* starting in state 0, position 0 */
    assertIsSame(ut,
-                test_fast_lexer(lexer_1,to_byte_array("gab"),none,0,0),
-                rejected(3,3,at_end_of_input),    "1");
-   unique. 
-   
-   
+                test_fast_lexer(lexer_1,to_byte_array("gab"),none,0,0,0),
+                rejected(3,at_end_of_input,0,3),    "1");
    assertIsSame(ut,
-                test_fast_lexer(lexer_1,to_byte_array("gabu"),none,bol,neol,0,0),
-                accepted(4,4,at_end_of_input),    "2");
+                test_fast_lexer(lexer_1,to_byte_array("gabu"),none,0,0,0),
+                accepted(4,at_end_of_input,0,4),    "2");
    assertIsSame(ut,
-                test_fast_lexer(lexer_1,to_byte_array("ggggabu"),none,bol,neol,0,0),
-                accepted(4,7,at_end_of_input),    "2bis");
+                test_fast_lexer(lexer_1,to_byte_array("ggggabu"),none,0,0,0),
+                accepted(4,at_end_of_input,0,7),    "2bis");
    assertIsSame(ut,
-                test_fast_lexer(lexer_1,to_byte_array("gabuz"),none,bol,neol,0,0),
-                accepted(4,4,at_end_of_input),    "3");
+                test_fast_lexer(lexer_1,to_byte_array("gabuz"),none,0,0,0),
+                accepted(5,at_end_of_input,0,4),    "3");
    assertIsSame(ut,
-                test_fast_lexer(lexer_1,to_byte_array("gmbuz"),none,bol,neol,0,0),
-                rejected(1,1,not_at_end_of_input),"4");
+                test_fast_lexer(lexer_1,to_byte_array("gmbuz"),none,0,0,0),
+                rejected(1,not_at_end_of_input,0,2),"4");
    assertIsSame(ut,
-                test_fast_lexer(lexer_1,to_byte_array("gabuzo"),none,bol,neol,0,0),
-                accepted(6,6,at_end_of_input),    "5"); 
+                test_fast_lexer(lexer_1,to_byte_array("gabuzo"),none,0,0,0),
+                accepted(6,at_end_of_input,0,6),    "5"); 
    assertIsSame(ut,
-                test_fast_lexer(lexer_1,to_byte_array("gabuzobof"),none,bol,neol,0,0),
-                accepted(6,6,not_at_end_of_input),    "6"); 
+                test_fast_lexer(lexer_1,to_byte_array("gabuzobof"),none,0,0,0),
+                accepted(6,not_at_end_of_input,0,6),    "6"); 
    assertIsSame(ut,
-                test_fast_lexer(lexer_1,to_byte_array("gggggabuzobof"),none,bol,neol,0,0),
-                accepted(6,10,not_at_end_of_input),    "6bis"); 
+                test_fast_lexer(lexer_1,to_byte_array("gggggabuzobof"),none,0,0,0),
+                accepted(6,not_at_end_of_input,0,10),    "6bis"); 
+   assertIsSame(ut,
+                test_fast_lexer(lexer_1,to_byte_array("  gggabuzobof"),none,0,0,0),
+                accepted(6,not_at_end_of_input,2,10),    "6ter"); 
+   assertIsSame(ut,
+                test_fast_lexer(lexer_1,to_byte_array("   "),none,0,0,0),
+                ignored(7,3),    "7"); 
+
+   unique. 
+   
+
  
    /* restarting from some other state (with or without an already accepted position) */ 
  
@@ -128,3 +148,9 @@ public define UnitTestSuite
      ]).
  
  
+global define One
+   fast_lexer_tests
+      (List(String) args) = 
+   execute_tests([make_fast_lexer_test_suite],args). 
+
+
-
+
  
  *Project*                                Anubis
  
-
+
  *Project*                             The Anubis Project
  
  *Title*                                 MIME Types definition.
@@ -68,5 +68,6 @@ public define String
     String  charset,
     String  text
   )=
-  with text1 = "=?"+charset+"?B?"+to_string(base64_encode(to_byte_array(text)))+"?=",
+//  with text1 = "=?"+charset+"?B?"+to_string(base64_encode(to_byte_array(text)))+"?=",
+  with text1 = text, 
   find_and_replace(text1, implode([13,10]), implode([13,10,32])).
-
+
  *Project*                             The Anubis Project
  
  *Title*                        A Multi Host HTTP/HTTPS Server
@@ -851,11 +851,12 @@ define String
        Web_Site_Description    desc, 
        List(HTTP_header)       headers,
      ) =
+   with all = member(journal_headers(desc),"*"), 
    if headers is 
      {
        [ ] then "",
        [h . t] then if h is http_header(name,value) then 
-         if member(journal_headers(desc),name)
+         if (all|member(journal_headers(desc),name))
          then "  | "+name+": "+value+"\n"+show_format(desc,t)
          else show_format(desc,t)
      }.