correction + iterator

HerrmannM
2 parents df8d5af6 bcb6ccd9
Showing 23 changed files with 747 additions and 96 deletions Show diff stats
anubis_dev/compiler/src/compil.h
anubis_dev/compiler/src/compile.c
anubis_dev/compiler/src/expr.cpp
anubis_dev/compiler/src/grammar.y
anubis_dev/compiler/src/interp.c
anubis_dev/compiler/src/lexer.l
anubis_dev/compiler/src/msgtexts.c
anubis_dev/compiler/src/replace.c
anubis_dev/compiler/src/show.c
anubis_dev/compiler/src/templates.c
anubis_dev/compiler/src/typetools.c
anubis_dev/library/data_base/import_csv.anubis
anubis_dev/library/data_base/read_csv.anubis
anubis_dev/library/data_base/read_csv_table.anubis
anubis_dev/library/lexical_analysis/fast_lexer_4.anubis
anubis_dev/library/system/files.anubis
anubis_dev/library/tools/bytearray.anubis
anubis_dev/library/tools/int.anubis
anubis_dev/library/tools/line_reader.anubis
anubis_dev/library/tools/list.anubis
@@ -200,7 +200,10 @@ extern void              NormalizeFileName(char *pathName);
    of the form   mcons3(file,line,col) where file, line and col Lisp integers.
 */
+extern char *path_prefix(char *name); 
+
 #define file_in(x)      (assert(((int)(integer_value(car(x)))) < max_already_included), already_included[integer_value(car(x))])
+#define dir_in(x)       path_prefix(file_in(x))
 #define line_in(x)      integer_value(second(x))
 #define col_in(x)       integer_value(cdr2(x))
@@ -631,6 +634,7 @@ extern Expr              linecol(void);
   item(unlock_var)\
   item(__line__)\
   item(__file__)\
+  item(__dir__)\
   item(lazy)\
 /* true 'dynamic' modules and 'load_adm' (added in version 1.13) */ 
@@ -3439,6 +3439,7 @@ Expr compile_term(Expr head,
      }
      break; 
+   case __dir__: 
    case __file__:  /* (__file__ . <lisp string>) */
      {
         code = cons(cons(string,
@@ -591,6 +591,7 @@ Expr _symbols_in_interp(Expr head)
     case operation:
     case __line__:
     case __file__:
+    case __dir__:
       return nil; 
     case macro: 
@@ -755,6 +756,7 @@ Expr _symbols_in_term(Expr term)
       case todo: /* (todo <lc> <filename> . <text>) */ 
       case __line__:
       case __file__:
+      case __dir__:
       case byte_array:
         return nil; 
@@ -81,7 +81,7 @@ int is_global = 0;
 %token<expr>     yy__config_file yy__verbose yy__stop_after yy__mapsto yy__rec_mapsto yy__language
 %token<expr>     yy__mapstoo yy__rec_mapstoo yy__arroww   
 %token<expr>     yy__conf_int yy__conf_string yy__conf_symbol
-%token<expr>     yy__enddot yy__eof yy__LINE__ yy__FILE__ yy__colon_equals
+%token<expr>     yy__enddot yy__eof yy__LINE__ yy__FILE__ yy__DIR__ yy__colon_equals
 %token<expr>     yy__integer yy__macro_integer yy__dummy yy__end_LBA
 %token<expr>     yy__defaults_as yy__lazy
@@ -666,6 +666,7 @@ Term: yy__alert                                 { alert_obsolete($1);
 |     yy__symbol yy__colon_equals Term                { $$ = $3; }
 |     yy__LINE__                %prec prec_symbol     { $$ = $1; }
 |     yy__FILE__                %prec prec_symbol     { $$ = $1; }
+|     yy__DIR__                 %prec prec_symbol     { $$ = $1; }
 //|     yy__Symbol                %prec prec_symbol     { $$ = mcons3(symbol,linecol(),$1); }
 |     yy__lpar yy__rpar           %prec prec_symbol     { $$ = mcons3(symbol,linecol(),pdstr_voidpars); }
 |     yy__integer               %prec prec_symbol     { $$ = mcons3(car($1),linecol(), cdr($1)); }
@@ -2316,6 +2316,7 @@ term_interpretations(Expr ttype,      /* required type for that term (may contai
     case __line__:   /* (__line__ . <lisp integer>) */
     case __file__:   /* (__file__ . <lisp string>) */
+    case __dir__:
     result = list1(cons(term,env));
     break; 
@@ -326,7 +326,7 @@ extern   void NormalizeFileName(char *pathName);
 /* open an Anubis source file. */ 
-FILE *fopensrc(const char *name, int n)    
+FILE *fopensrc(const char *name, int n)    // n is for debugging purpose (identifies from where the call comes)
                               /* The  given name may  be either  absolute or  relative. It
                                  must  be  immediately converted  into  an absolute  path,
                                  because  the  compiler  uses  ONLY  absolute  file  paths
@@ -544,7 +544,7 @@ int is_already_included(char *name)
           /* same up to case */ 
           if (!strcmp(already_included[i],fopensrc_abs_path))
           {
-            /* exactely same: file already compiled */ 
+            /* exactly same: file already compiled */ 
             fclose(fp); 
             return 1; 
           }
@@ -683,6 +683,8 @@ void come_back(void)
      if (verbose) printf("Returning to directory: %s\n",
                          include_dir[include_stack_ptr]); 
      current_file_abs_path = abs_file_paths_stack[include_stack_ptr]; 
+     fopensrc_abs_path = current_file_abs_path;    // these two variables seem to be redondant !
+     fopensrc_cur_dir = include_dir[include_stack_ptr]; 
      current_file_id = get_file_id(current_file_abs_path); 
      saturate_visi_table(); 
      if (show_reads) show_come_back(current_file_abs_path,current_file_id); 
@@ -986,6 +988,8 @@ W   [\ \t\r\n]
                                     return yy__LINE__; }
 <PAR>__FILE__                     { yylval.expr = cons(__file__,new_string(current_file_abs_path)); 
                                     return yy__FILE__; }
+<PAR>__DIR__                     { yylval.expr = cons(__dir__,new_string(path_prefix(current_file_abs_path))); 
+                                    return yy__DIR__; }
 <PAR>__TIME__                     { sprintf(str_buf,"%d",(int)time(NULL)); 
                                     yylval.expr = mcons4(of_type,linecol(),pdstr_Word32,mcons3(integer_10,
                                        linecol(),
@@ -1057,7 +1057,7 @@ const char *msgtext_undefined_macro[] =
 const char *msgtext_too_big[] =
   {
      "   Your program is too big for this version of the compiler.\n"
-     "   Consider spilling it using secondary modules.\n\n"
+     "   Consider spliting it using secondary modules.\n\n"
   }; 
 const char *msgtext_similar_paths[] =
@@ -505,6 +505,7 @@ static Expr rename_bound(Expr   head,      /* the expression within which bound 
   case __line__:
   case __file__:
+  case __dir__:
   {
     result = head; 
   }
@@ -955,6 +956,13 @@ static Expr replace_aux(Expr head,      /* where bound variables are already ren
   }
   break;  
+  case __dir__:
+  /* (__dir__ . <lisp string>) */
+  {
+    return cons(__dir__,new_string(dir_in(second(op)))); /* change the value of __DIR__ */ 
+  }
+  break;  
+  
   default: assert(0); 
    }
 } 
@@ -538,6 +538,12 @@ void show_interpretation(FILE *fp,
       }
       break;
+    case __dir__:
+      {
+        xpos += fprintf(fp,"__DIR__");
+      }
+      break;
+
 #if 0   
     case integer: /* (integer <lc> . <Cint>) */  Obsolete: replaced by 'anb_int_10' and 'anb_int_16'
       {
@@ -255,6 +255,11 @@
   {
   }
+  case __dir__:
+  /* (__dir__ . <lisp string>) */
+  {
+  }
+  
   default: assert(0); 
   }
@@ -413,6 +413,7 @@ Expr _type_from_interpretation(Expr head, Expr env)
       break; 
     case __file__:
+    case __dir__:
       result = type_String;   
       break; 
-                       
+   This file is obsoleted by data_base/read_csv.anubis                   
+   
+   
                                       The Anubis Project 
                                       Reading CSV tables. 
@@ -54,9 +56,9 @@
-read tools/basis.anubis
+ read tools/basis.anubis
-public define Maybe(List(List(String)))
+ public define Maybe(List(List(String)))
    read_table
      (
        String                                 filename,
@@ -97,7 +99,7 @@ public define Maybe(List(List(String)))
    Checking if a list of Word8 'candidate' is a prefix in a line separator. 
-define Bool
+ define Bool
    begins_line_separator
      (
        List(Word8) candidate,
@@ -121,7 +123,7 @@ define Bool
    Here is the test. 
-define Bool
+ define Bool
    begins_line_separator
      (
        List(Word8) candidate,
@@ -141,7 +143,7 @@ define Bool
    We have two cross recursive functions 'read_table' and 'read_more_lines'. 
-define List(List(String))
+ define List(List(String))
    read_table
      (
        RStream file, 
@@ -162,7 +164,7 @@ define List(List(String))
    file.  If  end of file  is read, the  last line of the  table is empty.   Otherwise, we
    return to 'read_table', with the correct 'current_line' and 'current_cell'.
-define List(List(String))
+ define List(List(String))
    read_more_lines
      (
        RStream file, 
@@ -185,7 +187,7 @@ define List(List(String))
    Reading a table from an already opened file. 
-define List(List(String))
+ define List(List(String))
    read_table
      (
        RStream file, 
@@ -218,7 +220,7 @@ define List(List(String))
    Now, here is our tool. 
-public define Maybe(List(List(String)))
+ public define Maybe(List(List(String)))
    read_table
      (
        String                                 filename,
+
+                                   The Anubis Project
+
+                       Reading a csv (Comma Separated Values) file. 
+   
+   
+read tools/basis.anubis   
+read tools/time.anubis
+read lexical_analysis/fast_lexer_4.anubis
+
+
+   The function made by the function below reads a single record from a CSV input source. 
+
+public type ReadCsvResult:
+   end_of_input,              
+   error         (String    message),                             // an error message
+   ok            (Int       offset,    List(String)  record).     // a single record and the offset of the end of 
+                                                                  // this record. 
+   
+public define One -> ReadCsvResult
+   make_read_csv_line
+   (
+     LexingStream    ls,            // lexing stream to be constructed from the input (see fast_lexer_4.anubis)
+     String          sep,           // cell separator (can be "," or ";") 
+     List(Int)       cols_to_get    // list of column numbers you want to get
+   ).
+
+   
+   
+   --- That's all for the public part ! ------------------------------------------------------------------
+   
+                                        
+type CellPrefixToken:          // reading the beginning of a cell until the first double quote or separator
+   eof, 
+   double_quote,               // if double quote, ignore the content up to here and switch to another lexer
+   separator(ByteArray).       // if separator, keep everything before this separator
+    
+   
+   This lexer if for reading the beginning of a cell. 
+   
+define List(LexerItem(CellPrefixToken,One))
+   begin_cell_description
+   (
+     String        sep
+   ) = 
+   [
+     lexer_item("[# #t]*\"",      
+       return(((Int,Int) -> ByteArray extract, Int l, LexingTools t, One u) |-> 
+           token(double_quote))),
+           
+     lexer_item("[^#"+sep+"\"#r#n]*#"+sep,      
+       return(((Int,Int) -> ByteArray extract, Int l, LexingTools t, One u) |-> 
+           token(separator(extract(0,l-1))))), 
+           
+     lexer_item("[^#"+sep+"\"#r#n]*#n"  ,      
+       return(((Int,Int) -> ByteArray extract, Int l, LexingTools t, One u) |-> 
+           token(separator(extract(0,l-1))))), 
+           
+     lexer_item("[^#"+sep+"\"#r#n]*(#r#n)"  ,      
+       return(((Int,Int) -> ByteArray extract, Int l, LexingTools t, One u) |-> 
+           token(separator(extract(0,l-2)))))           
+   ].
+                                        
+   
+    This lexer is for reading within the double quotes (if the cell is double quoted)
+    
+    
+type InToken:
+   double_quote,         // can also be the end of file
+   two_double_quotes, 
+   part(ByteArray).      // part of cell
+   
+   
+define List(LexerItem(InToken,One))
+   read_quoted_cell_description
+   (
+     String        sep
+   ) = 
+   [
+     lexer_item("[^\"]*"  ,      
+          return((ByteArray b, LexingTools t, One u) |-> token(part(b)))), 
+          
+     lexer_item("\"\""  ,      
+          return((ByteArray b, LexingTools t, One u) |-> token(two_double_quotes))), 
+                    
+     lexer_item("\"[# #t]*(("+sep+")|(#n)|(#r#n))"  ,      
+          return((ByteArray b, LexingTools t, One u) |-> token(double_quote)))
+   ].
+                                              
+   
+    The lexer described below skips a cell (and eats the trailing separator). 
+
+define List(LexerItem(One,One)) 
+   skip_cell_description
+   (
+     String sep
+   ) =
+   [
+     lexer_item("(([^\"#n#r#"+sep+"]*)|([# #t]*\"([^\"]|(\"\"))*\"[# #t]*))#"+sep,  
+                   return(((Int,Int) -> ByteArray b, Int l, LexingTools t, One u) |-> token(unique)))
+   ].
+
+   The lexer described below skips to end of line (and eats the end of line).
+
+type EOL_Token:
+   eol_offset(Int offset).    
+   
+define List(LexerItem(EOL_Token,One))
+   to_eol_description
+   = 
+   [
+     lexer_item("([^#r#n]*)((#n)|(#r#n))",  
+                return(((Int,Int) -> ByteArray b, Int l, LexingTools t, One u) |-> 
+                           token(eol_offset(t.offset(unique)))))
+   ]. 
+   
+   
+   
+global define One 
+   make_the_lexers
+   (
+     List(String) _
+   ) =
+   make_precompiled_lexer("csv_c_begin_cell",    begin_cell_description(","),         '#'); 
+   make_precompiled_lexer("csv_c_quoted_cell",   read_quoted_cell_description(","),   '#'); 
+   make_precompiled_lexer("csv_c_skip_cell",     skip_cell_description(","),          '#'); 
+   make_precompiled_lexer("csv_s_begin_cell",    begin_cell_description(";"),         '#'); 
+   make_precompiled_lexer("csv_s_quoted_cell",   read_quoted_cell_description(";"),   '#'); 
+   make_precompiled_lexer("csv_s_skip_cell",     skip_cell_description(";"),          '#'); 
+   make_precompiled_lexer("csv_to_eol",          to_eol_description,                  '#'). 
+
+   
+execute anbexec make_the_lexers
+read generated/csv_c_begin_cell.anubis
+read generated/csv_c_quoted_cell.anubis
+read generated/csv_c_skip_cell.anubis
+read generated/csv_s_begin_cell.anubis
+read generated/csv_s_quoted_cell.anubis
+read generated/csv_s_skip_cell.anubis
+read generated/csv_to_eol.anubis
+
+
+ define One
+   repeat
+   (
+     Int          n, 
+     One -> One   f
+   ) =
+   if n =< 0 then unique else f(unique); repeat(n-1,f). 
+   
+
+ define Int -> Result(String,ByteArray)
+   read_next_cell
+   (
+     One -> One                          skip_cell,
+     One -> Result(String,ByteArray)     read_cell
+   ) =
+   (Int n) |-> repeat(n,skip_cell); read_cell(unique). 
+
+
+type CB_Result: 
+   eof, 
+   error(String), 
+   skip, 
+   cell(String),
+   eol(Int offset).
+      
+   
+define One -> ReadCsvResult
+   make_read_csv_line
+   (
+     List(One -> CB_Result)    cbs,
+     One -> One                to_eol
+   ) = 
+   with f = (List(One -> CB_Result) l, List(String) so_far) |-f1->
+     if l is 
+     {
+       [ ]      then ok(0, reverse(so_far)),  
+       [f . g]  then
+         if f(unique) is 
+         {
+           eof       then end_of_input, 
+           error(e)  then to_eol(unique); error(e),
+           skip      then (ReadCsvResult)f1(g, so_far),
+           cell(c)   then 
+            //print("read cell ["+c+"] ");
+            (ReadCsvResult)f1(g, [c. so_far]),
+           eol(offset)  then 
+            if g is 
+            {
+              []      then ok(offset, reverse(so_far)),
+              [_ . _] then error("End Of Line unexpected")
+            }
+            
+         }
+     },
+   (One u) |-> f(cbs, []). 
+   
+   
+   
+define List(One -> CB_Result)
+   make_cbs
+   (
+     One -> CB_Result         skip_cell, 
+     One -> CB_Result         read_cell, 
+     One -> CB_Result         to_eol,
+     List(Int)                cols_to_get   // must be strictly increasing
+   ) =
+   if cols_to_get is 
+   {
+     [ ]      then [to_eol],
+     [n . l]  then
+       with rest = make_cbs(skip_cell,read_cell,to_eol, map((Int x) |-> x - n -1,l)), 
+         constant_list(skip_cell,n) + [read_cell . rest]
+   }.
+   
+
+    Just a test for make_cbs: 
+ global define One
+    gaga
+   (
+     List(String) args
+   ) =
+   with   skip_cell = (One u) |-> skip, 
+          read_cell = (One u) |-> (CB_Result)cell(to_byte_array("")), 
+          to_eol    = (One u) |-> (CB_Result)error(""), 
+      l = [3,5,(Int)10], 
+      with r = make_cbs(skip_cell,read_cell,to_eol,l), 
+        forget(map((One -> CB_Result f) |-> if f(unique) is 
+           {
+             error(e) then print("eol\n\n"),
+             skip     then print("skip\n"), 
+             cell(_)  then print("cell\n")
+           }, r)). 
+
+           
+           
+
+public define One -> ReadCsvResult
+   make_read_csv_line
+   (
+     LexingStream    ls, 
+     String          sep,
+     List(Int)       cols_to_get
+   ) =
+   with lex_skip   = retrieve_lexer(skip_cell_description(sep),          if sep = "," then csv_c_skip_cell else csv_s_skip_cell)(ls,unique), 
+        lex_begin  = retrieve_lexer(begin_cell_description(sep),         if sep = "," then csv_c_begin_cell else csv_s_begin_cell)(ls,unique),
+        lex_in     = retrieve_lexer(read_quoted_cell_description(sep),   if sep = "," then csv_c_quoted_cell else csv_s_quoted_cell)(ls,unique),   
+        lex_eol    = retrieve_lexer(to_eol_description,                  csv_to_eol)(ls,unique),
+        skip_cell  = (One u) |-> (CB_Result)if lex_skip(u) is 
+                       {
+                         end_of_input       then eof,
+                         error(b,line,col)  then error("skip "+line+":"+col+" :"+to_string(b)),
+                         token(t)           then skip
+                       }, 
+         begin_cell = (One u) |-> (Result(String,CellPrefixToken))if lex_begin(u) is 
+                       {
+                         end_of_input       then ok(eof),
+                         error(b,line,col)  then error("begin "+to_string(b)),
+                         token(t)           then ok(t)
+                       },                        
+         read_in_aux  = (List(ByteArray) so_far) |-aux-> (CB_Result)if lex_in(unique) is 
+                       {
+                         end_of_input       then eof,
+                         error(b,line,col)  then error("in "+to_string(b)),
+                         token(t)           then if t is 
+                           {
+                             double_quote       then cell(to_string(concat(reverse(so_far)))), 
+                             two_double_quotes  then aux([{0x22} . so_far]),
+                             part(p)            then aux([p . so_far])
+                           }
+                       },
+         read_in    = (One u) |-> read_in_aux([]), 
+         read_cell  = (One u) |-> if begin_cell(u) is 
+                       {
+                         error(e)  then error(e),  
+                         ok(b)     then if b is 
+                           {
+                             eof            then eof, 
+                             double_quote   then read_in(u), 
+                             separator(c)   then cell(to_string(c))
+                           }
+                       }, 
+         to_eol     = (One u) |-> if lex_eol(u) is 
+                       {
+                         end_of_input       then eof,
+                         error(b,line,col)  then error("eol "+to_string(b)),
+                         token(t)           then if t is eol_offset(offset) then eol(offset)
+                       }, 
+         make_read_csv_line(make_cbs(skip_cell,read_cell,to_eol,cols_to_get),
+                            (One u) |-> forget(to_eol(u))). 
+         
+                            
+                            
+   *** Command line test.                             
+                            
+define Maybe(List(Int))
+   map_to_Int
+   (
+     List(String) l
+   ) =
+   if l is 
+   {
+     [ ]     then success([ ]),
+     [h . t] then if decimal_scan(h) is 
+       {
+         failure     then failure, 
+         success(n1) then if map_to_Int(t) is 
+           {
+             failure     then failure,
+             success(l1) then success([n1 . l1])
+           }
+       }
+   }.
+         
+         
+define One 
+   print_csv_line
+   (
+     List(String)   l
+   ) = 
+   print("| "); 
+   map_forget((String b) |-> print(b+" | "),l). 
+   
+   
+define One syntax = print("Usage: anbexec read_csv_file <csv file path> <sep> <n1> ... <nk>\n"+
+                          "   where <sep> is the (double quoted) separator (can be \",\" or \";\")\n"+
+                          "   and where the integers <n1>...<nk> are the ranks of the columns to keep,\n"+
+                          "   (starting at 0).\n\n").    
+                          
+define One 
+   print_to_error
+   (
+     One -> ReadCsvResult   f
+   ) =
+   if f(unique) is 
+   {
+     end_of_input  then print("-------- end of file --------------\n"), 
+     error(e)      then print("Error ["+e+"]\n"); 
+                        print_to_error(f),  
+     ok(offset,n)  then print_csv_line(n);
+                        print("[at offset "+offset+"]\n");
+                        print_to_error(f)
+   }.
+   
+define One 
+   show_perf
+   (
+     One -> ReadCsvResult   f,
+     Int                    left,
+     Int                    read_line,
+     Int                    block_size,
+     UTime                  start_time
+   ) =  
+   if f(unique) is 
+   {
+     end_of_input  then show_duration("lines read "+read_line, start_time);
+                        print("----------------------\n"),
+     error(e)      then print("error ["+e+"]\n"); print_to_error(f),  
+     ok(o,n)       then
+      with left1 = if left = 1 then
+                    show_duration("lines read "+read_line+1, start_time);
+                    block_size
+                  else
+                    left -1,
+      show_perf(f, left1, read_line+1, block_size, start_time)
+   }.
+   
+   
+define One 
+   show_perf
+   (
+    One -> ReadCsvResult  f,
+    Int                   block_size
+   )=
+   show_perf(f, block_size, 0, block_size, unow)
+   .
+   
+global define One
+   read_csv_file
+   (
+     List(String) args
+   ) =
+  if args is 
+  {
+    [ ]         then syntax, 
+    [path . t]  then 
+      println("file "+path);
+      if t is
+      {
+        
+        [ ] then syntax, 
+        [sep . l] then if sep:[",",";"] 
+          then 
+          if map_to_Int(l) is 
+          {
+            failure                 then syntax 
+            success(List(Int) cols) then
+              if file(path,read) is 
+              {
+                failure     then print("File '"+path+"' not found.\n"), 
+                success(f)  then
+                  if make_lexing_stream("",f,10,10) is 
+                  {
+                    failure      then print("Error while reading file '"+path+"'.\n"), 
+                    success(ls)  then   
+                      with cs = no_doubles(qsort(cols,(Int x, Int y) |-> x < y)), 
+                        read_line = make_read_csv_line(ls,sep,cs), 
+//                          show_perf(read_line, 10000)
+                          print_to_error(read_line)
+                  }
+               }
+           }
+           else syntax
+       }
+   }. 
+
+
+   This file is obsoleted by data_base/read_csv.anubis
+   
    Try it !
-read import_csv.anubis   
+ read import_csv.anubis   
-define One
+ define One
    table_print
      (
        List(String) l
@@ -18,7 +20,7 @@ define One
                     table_print(t)
      }.
-define One
+ define One
    table_print
      (
        List(List(String)) t
@@ -29,7 +31,7 @@ define One
        [h . t2] then table_print(h); table_print(t2)
      }.
-define One
+ define One
    table_print
      (
        Maybe(List(List(String))) t
@@ -40,7 +42,7 @@ define One
        success(l) then table_print(l)
      }.
-global define One
+ global define One
    read_csv_table
      (
        List(String) args
@@ -618,7 +618,7 @@ public type DFA_state($Token,$Aux):
    variant of 'make_lexer':
 public define Result(RegExprError, 
-                     (LexingStream -> One -> LexerOutput($Token),     // the lexer
+                     ((LexingStream,$Aux) -> One -> LexerOutput($Token),     // the lexer
                       List(DFA_state($Token,$Aux))))                        // the automaton
    make_lexer_and_automaton
      (
@@ -651,7 +651,7 @@ public define One
       *** (4.3) How to use a lexer.
-   Applying the function of type 'LexingStream($Aux) -> One -> LexerOutput($Token)' returned by
+   Applying the function of type 'LexingStream -> One -> LexerOutput($Token)' returned by
    'make_lexer' to a lexing stream is understood as 'plugging' the lexer onto this lexing
    stream. The result is a function of type:
@@ -1314,10 +1314,34 @@ define Result(RegExprError,RegExpr)
+    Debugging tools:
+define String
+   format
+   (
+     List(Word8)  l
+   ) =
+   concat(map((Word8 c) |-> to_decimal(c) ,l)," ").
-   
-   
+define String
+   format
+   (
+     RegExpr e
+   ) =
+   if e is 
+   {
+     char(Word8 _0)              then "char("+constant_string(1,_0)+")",
+     choice(List(Word8) _0)      then "choice("+format(_0)+")",
+     plus(RegExpr _0)            then "plus("+format(_0)+")",
+     star(RegExpr _0)            then "star("+format(_0)+")",
+     cat(RegExpr _0,RegExpr _1)  then "cat("+format(_0)+","+format(_1)+")",
+     or(RegExpr _0,RegExpr _1)   then "or("+format(_0)+","+format(_1)+")",
+     dot                         then "dot",
+     question_mark(RegExpr _0)   then "question_mark("+format(_0)+")"
+   }. 
+         
+         
          *** [1.6.3] The tool for parsing regular expressions. 
+         
 public define Result(RegExprError,RegExpr)
   parse_regular_expression
@@ -1328,13 +1352,12 @@ public define Result(RegExprError,RegExpr)
    if read_regexpr(s,escape_char,[],end_of_regexpr) is 
      {
        error(msg)  then error(msg), 
-       ok(re)      then ok(re)
+       ok(re)      then //print("["+format(re)+"]\n"); 
+                        ok(re)
      }.
-   
-   
       *** [1.7] Transforming a regular expression into a basic one.    
          *** [1.7.1] Expanding a 'choice' of characters. 
@@ -1427,7 +1450,7 @@ public define String
       *** [1.1] The type 'LexingStream'. 
    A lexing stream provides tools which are adhoc for using low level fast lexers as
-   defined in section 13 of predefined.anubis:
+   defined in section 13 of predefined.anubis.
    The type below records the information needed to come back to the state just after the 
    last or penultimate token was read. 
@@ -1440,9 +1463,10 @@ type TokenState:
       Int         col
     ).      
-   There is a ``penultimate token'' when at least one token has been successfully read since the 
+   There is a ``penultimate token'' when at least two token has been successfully read since the 
    creation of the lexing stream. If it is not the case, the value of the ``penultimate state''
-   defaults to the very initial state.
+   defaults to the state after the very first token was read or to the very initial state if no
+   tokan was read.
    When the buffer is reloaded, part of the current buffer is kept. One reason for this is that
    when we encounter the end of the buffer it can be the case that we are currently reading a token
@@ -1464,7 +1488,6 @@ type TokenState:
    state informations for token1 and token2, the last two tokens successfully read.
-        
 public type LexingStream:
    lexing_stream
      (
@@ -1546,18 +1569,18 @@ public type LexingStream:
            -- the new current buffer "source text."
            -- last accepted: (s,3), because 'sou' has been accepted in state 's' and
-              ends at offset 0 within the new buffer,
+              ends at offset 3 within the new buffer,
            -- current_v receives the value 3, because 'sou' is already read, 
            -- token_start_v receives the value 0, because the token we are currently 
               reading begins at offset 0. 
-           -- state s, because we want to try to read the sequel of 'sou'. 
+           -- restart in state s, because we want to try to read the sequel of 'sou'. 
    Notice that if the low level lexer had returned 'rejected(s,at_end_of_input,12,15)'
    instead of 'accepted(s,at_end_of_input,12,15)', the scenario is the same one except
-   that last accepted is 'none'.
+   that last accepted will be 'none'.
    The low level lexer will now return 'accepted(s,not_at_end_of_input,0,6)', meaning that
    it has recognized the token 'source' between positions 0 (included) and 6 (not
@@ -1574,6 +1597,7 @@ public type LexingStream:
 define LexingTools
   make_tools
   (
+    Var(Int)                     token_start_v,    // actually not used in this function
     Var(Int)                     current_v, 
     Var(Int)                     line_v,
     Var(Int)                     col_v,
@@ -1590,12 +1614,11 @@ define LexingTools
     (One _) |-> *col_v,
     // get current offset:
+    // This is the number of bytes which are no more in the buffer plus the current position.
     (One _) |-> *past_v + *current_v,
     // go back one char:
     // don't go beyond the beginning of the buffer
-    // No need to update line_v and col_v because they
-    // refer to the beginning of the token.
     (Int n) |-> current_v <- max(*current_v - n, 0),
     // comming back to the state just after the last token was read
@@ -1603,7 +1626,6 @@ define LexingTools
                 current_v <- cur; 
                 line_v <- l; 
                 col_v <- c;
-                last_tok_v <- *penult_tok_v;
                 last_accept_v <- none, 
     // comming back to the state just after the penultimate token was read
@@ -1624,7 +1646,8 @@ public define LexingStream
        String      preambule, 
        ByteArray   b
      ) = 
-   with    b1_v = var(if length(preambule) = 0 then b else to_byte_array(preambule)+b), 
+   with    b1_v = var(if length(preambule) = 0 then b else to_byte_array(preambule)+b),
+  token_start_v = var((Int)0),  
       current_v = var((Int)0), 
          line_v = var((Int)0), 
           col_v = var((Int)0), 
@@ -1633,7 +1656,7 @@ public define LexingStream
    penult_tok_v = var(tstate(0,0,0)), 
   last_accept_v = var((FastLexerLastAccepted)none), 
    lexing_stream(b1_v,                       // buffer
-                 var((Int)0),                // starting position
+                 token_start_v,              // starting position
                  current_v,                  // current position
                  last_accept_v,              // last accepting position
                  last_tok_v,                 // last token state
@@ -1641,8 +1664,8 @@ public define LexingStream
                  (One u) |-> failure,        // buffer is never reloaded
                  line_v,                     // current line
                  col_v,                      // current column
-                 past_v,                     // past bytes
-                 make_tools(current_v,line_v,col_v,past_v,last_tok_v,penult_tok_v,last_accept_v)). 
+                 past_v,                     // past bytes (will remain always 0 in this case)
+                 make_tools(token_start_v,current_v,line_v,col_v,past_v,last_tok_v,penult_tok_v,last_accept_v)). 
@@ -1700,7 +1723,7 @@ public define Maybe(LexingStream)
          //print("Keeping this from previous buffer: ["+to_string(extract(old_buffer,dropped,old_length))+"]\n");
                               buffer_v <- extract(old_buffer,dropped,old_length)+more; 
          //print("New buffer: ["+to_string(*buffer_v)+"] size: "+to_decimal(length(*buffer_v))+"\n");
-                              token_start_v <- 0; 
+                              token_start_v <- *token_start_v - dropped; 
          //print("Next token starting position: "+to_decimal(*token_start_v)+"\n");
                               current_v <- old_length - dropped; 
          //print("New current reading position: "+to_decimal(*current_v)+"\n");
@@ -1726,7 +1749,7 @@ public define Maybe(LexingStream)
                                line_v,
                                col_v,
                                past_bytes_v, 
-                               make_tools(current_v,line_v,col_v,past_bytes_v,last_tok_v,penult_tok_v,last_accepted_v)))
+                               make_tools(token_start_v,current_v,line_v,col_v,past_bytes_v,last_tok_v,penult_tok_v,last_accepted_v)))
      }.
@@ -1785,7 +1808,7 @@ public define Maybe(LexingStream)
                                      min(min(current(*penult_tok_v),current(*last_tok_v)),*token_start_v), 
                               buffer_v <- extract(old_buffer,dropped,old_length)+more; 
-                              token_start_v <- 0; 
+                              token_start_v <- *token_start_v - dropped; 
                               current_v <- old_length - dropped; 
                               past_bytes_v <- *past_bytes_v + dropped; 
                               last_tok_v <- (if *last_tok_v is tstate(cur,l,c) then tstate(cur - dropped,l,c));
@@ -1809,7 +1832,7 @@ public define Maybe(LexingStream)
                                line_v,
                                col_v,
                                past_bytes_v, 
-                               make_tools(current_v,line_v,col_v,past_bytes_v,last_tok_v,penult_tok_v,last_accepted_v)))
+                               make_tools(token_start_v,current_v,line_v,col_v,past_bytes_v,last_tok_v,penult_tok_v,last_accepted_v)))
      }.
@@ -1841,7 +1864,9 @@ define (Int, Int, Int)              // returns new (start,line,col)
        Int           line,          // current line
        Int           col            // current column
      ) =
-   if old_start >= new_start then (new_start,line,col) else
+   //print("old_start = "+old_start+"\n"); 
+   if old_start >= new_start then //print("======== new col: "+col+"\n");
+           (new_start,line,col) else
    with c = force_nth(old_start,buffer), 
    if ((c >> 6) = 2)   
      /*
@@ -1864,6 +1889,7 @@ define One
        Var(Int)      line_v, 
        Var(Int)      col_v
      ) =
+   //print("new_start = "+new_start+"\n"); 
    if compute_start_line_col(buffer,*token_start_v,new_start,*line_v,*col_v) is (s,l,c) then 
    token_start_v <- s; 
    line_v <- l; 
@@ -1891,7 +1917,7 @@ public define LexerOutput($Token)
      ) =
    if lstream is lexing_stream(buffer_v,token_start_v,current_v,last_accept_v,last_tok_v,penult_tok_v,reload_buffer,
                                line_v,col_v,offset_v,tools) then 
- //print("starting at offset "+to_decimal(*current_v)+" with token start at "+to_decimal(*token_start_v)+"\n"); 
+   //print("starting at offset "+to_decimal(*current_v)+" with token start at "+to_decimal(*token_start_v)+"\n"); 
    with lgbuf = length(*buffer_v), 
    if low_level_lexer(*buffer_v,*last_accept_v,*current_v,*token_start_v,starting_state) is 
      {
@@ -1908,7 +1934,7 @@ public define LexerOutput($Token)
          else
          (
              /* the lexeme may still be accepted after the buffer is reloaded */
-             update_start_line_col(*buffer_v,start,token_start_v,line_v,col_v);
+             //update_start_line_col(*buffer_v,start,token_start_v,line_v,col_v);
              if reload_buffer(unique) is 
                {
                  failure then 
@@ -1928,13 +1954,13 @@ public define LexerOutput($Token)
        /* almost the same thing for accepted */
        accepted(s,start,end) then 
-  //print("low level accepted start = "+to_decimal(start)+" end = "+to_decimal(end)+"\n");
+       //print("low level accepted start = "+to_decimal(start)+" end = "+to_decimal(end)+"\n");
          last_accept_v <- last(s,end); 
          current_v <- end; 
          if end /= lgbuf then 
          (
              /* the lexeme just read must be accepted: the action is applied */
-               update_start_line_col(*buffer_v,end,token_start_v,line_v,col_v); 
+                
                last_accept_v <- none; 
                if *actions(word32(s,0)) is 
                  {
@@ -1945,18 +1971,25 @@ public define LexerOutput($Token)
                      // We must update some variables
                       penult_tok_v <- *last_tok_v; 
                       last_tok_v <- tstate(end,*line_v,*col_v); 
-                      f(extract(*buffer_v,start,end),tools,aux),
+                      with result = f(extract(*buffer_v,start,end),tools,aux),
+                        update_start_line_col(*buffer_v,*current_v,token_start_v,line_v,col_v);
+                        result, 
+                      
                    return(f)  then 
                       penult_tok_v <- *last_tok_v; 
                       last_tok_v <- tstate(end,*line_v,*col_v); 
-                      f((Int k, Int l) |-> extract(*buffer_v,start+k,start+l),
-                        end-start,tools,aux),
+                      with result = f((Int k, Int l) |-> extract(*buffer_v,start+k,start+l),
+                                                 end-start,tools,aux),
+                        //print("*token_start_v = "+*token_start_v+"\n"); 
+                        //print("*current_v = "+*current_v+"\n");
+                        update_start_line_col(*buffer_v,*current_v,token_start_v,line_v,col_v);
+                        result
                  }
          )
          else 
          (               
              /* the lexeme may still be accepted after the buffer is reloaded */
-             update_start_line_col(*buffer_v,start,token_start_v,line_v,col_v); 
+              
              if reload_buffer(unique) is 
                {
                  failure then 
@@ -1970,11 +2003,15 @@ public define LexerOutput($Token)
                        ignore      then should_not_happen(end_of_input), 
                        return(f)   then penult_tok_v <- *last_tok_v; 
                                         last_tok_v <- tstate(end,*line_v,*col_v); 
-                                        f(extract(*buffer_v,start,end),tools,aux),
+                                        with result = f(extract(*buffer_v,start,end),tools,aux),
+                                          update_start_line_col(*buffer_v,*current_v,token_start_v,line_v,col_v);
+                                          result,
                        return(f)   then penult_tok_v <- *last_tok_v; 
                                         last_tok_v <- tstate(end,*line_v,*col_v); 
-                                        f((Int k, Int l) |-> extract(*buffer_v,start+k,start+l),
-                                           end-start,tools,aux)
+                                        with result = f((Int k, Int l) |-> extract(*buffer_v,start+k,start+l),
+                                                        end-start,tools,aux),
+                                          update_start_line_col(*buffer_v,*current_v,token_start_v,line_v,col_v);
+                                          result
                      }, 
                  success(_) then 
@@ -1988,7 +2025,7 @@ public define LexerOutput($Token)
         ignored_to_end then 
     //print("low level ignored_to_end\n");
              /* we are at end of input buffer */
-             update_start_line_col(*buffer_v,lgbuf,token_start_v,line_v,col_v); 
+             //update_start_line_col(*buffer_v,lgbuf,token_start_v,line_v,col_v); 
              if reload_buffer(unique) is 
                {
                  failure then 
@@ -2010,7 +2047,7 @@ public define LexerOutput($Token)
    *** [3] Constructing the automaton. 
-   The description of a lexer is given as a list of 'LexerItem($Token)', where the
+   The description of a lexer is given as a list of 'LexerItem($Token,$Aux)', where the
    parameter '$Token' represents the type of tokens.  Each lexer item is made of a regular
    expression and an action. If the action is 'ignore', the token just read is ignored and
    the lexer tries to read the next one.  Otherwise, the action is applied to the lexeme
@@ -3032,20 +3069,47 @@ define List(Int)
        ignoring(name,transitions)               then [-1   . actions_ranks(t)]
      }
   }. 
+  
+  
+   Get a characteristic serializable datum from a lexer description (used to avoid
+   reconstructing the lexer when the description did not change). The signature is 
+   changed if any of the regular expressions is changed or if their order is changed
+   or if the sort of action is changed (this last point ensures that the list of 
+   action ranks remains correct in the generated file). 
+define List(ByteArray)
+   extract_regexprs
+   (
+     List(LexerItem($Token,$Aux))  l
+   ) = 
+   with asign = (LexerAction($Token,$Aux) a) |-> if a is 
+     {
+       ignore      then "(*i)",     // something which is illegal as a regular expression
+       return(_0)  then "(*r1)",
+       return(_0)  then "(*r2)"
+     },
+   map((LexerItem($Token,$Aux) i) |-> if i is 
+           {
+             lexer_item(regular_expression,action)  then to_byte_array(regular_expression+asign(action)), 
+             lexer_item(literal,action)             then literal+to_byte_array(asign(action))
+           },l).
+           
+           
 public define One
-  make_precompiled_lexer
+  make_precompiled_lexer_aux
   (
+    String                         signature,
     String                         directory, 
     String                         lexer_name, 
     List(LexerItem($Token,$Aux))   lexer_description,
     Word8                          escape_char    
-  ) =
+  ) =  
   with file_name = directory/lexer_name+".anubis", 
   if file(file_name,new) is 
   {
     failure       then print("Cannot create file '"+file_name+"'.\n"), 
     success(file) then
+      print(weaken(file),"   "+signature+"\n\n"); 
       if make_DFA(lexer_description,escape_char) is 
       {
         error(msg) then print(to_English(msg)+"\n"), 
@@ -3060,6 +3124,46 @@ public define One
   }.
+define Maybe(String)
+   read_signature
+   (
+     String file_name
+   ) =
+   if file(file_name,read) is 
+   {
+     failure     then failure,
+     success(f)  then if read(f,43,10) is     // read the first 3 (blanks) + 40 (sha1 hash) characters
+       {
+         error    then failure, 
+         timeout  then failure, 
+         ok(ba)   then success(to_string(extract(ba,3,43))) 
+       }
+   }.
+  
+public define One
+  make_precompiled_lexer
+  (
+    String                         directory, 
+    String                         lexer_name, 
+    List(LexerItem($Token,$Aux))   lexer_description,
+    Word8                          escape_char    
+  ) =  
+  // avoid to reconstruct the lexer if not needed
+  with signature = to_hexa(sha1(extract_regexprs(lexer_description))),
+       file_name = directory/lexer_name+".anubis", 
+           do_it = (One u) |-> 
+             print("Creating '"+file_name+"'. Please wait ... "); forget(flush(stdout)); 
+             make_precompiled_lexer_aux(signature,directory,lexer_name,lexer_description,escape_char);
+             print("Done.\n"); forget(flush(stdout)),
+  if read_signature(file_name) is 
+  {
+    failure    then do_it(unique), 
+    success(s) then if s = signature
+                    then unique
+                    else do_it(unique)
+  }. 
+  
+  
 public define One
   make_precompiled_lexer
   (
@@ -1020,14 +1020,14 @@ define Maybe(Int)
   find_the_first
   (
     Data_IO   io,         
-    String    looking_for,  //String to search
+    ByteArray looking_for,  //String to search
     Int       size,         //size of the string to search
-    String    buffer,
+    ByteArray buffer,
     Int       current_pos,
     Int       buf_size,
     Int       buf_pos
   )=
-  //println("buf_size :"+buf_size+ " buf_pos :"+buf_pos + " size : "+size);
+  //println("general current pos: "+current_pos+" | buffer size: "+buf_size+ " | buffer pos: "+buf_pos + " | search size: "+size);
   if (buf_size - buf_pos) < size then
     //println("New buffer request current pos "+current_pos+" buffer_pos "+buf_pos);
     if read_bytes(io, 65536) is // <- block size is 64k
@@ -1035,23 +1035,30 @@ define Maybe(Int)
       failure   then  println("read_bytes failure");failure, //finish
       time_out  then  println("read_bytes timeout");failure, //finish
       success(ba) then  
-        with new_buffer = to_string(extract(to_byte_array(buffer), buf_pos, buf_size) + ba),
+        //println("length of ba "+length(ba));
+        with ex_ba = extract(buffer, buf_pos, buf_size),
+        //println("length of ex_ba "+length(ex_ba));
+        with new_ba = ex_ba + ba,
+        //println("length of new_ba "+length(new_ba));
+        with new_buffer = ex_ba + ba,
         //println("SUCCESS New buffer length "+length(new_buffer)+" new current_pos "+current_pos);
         find_the_first(io, looking_for, size, new_buffer, current_pos + buf_pos, length(new_buffer), 0),
       truncated(ba) then 
         if length(ba) = 0 then
+          //println("last buffer current position ["+current_pos+"]");
           failure //finish
         else
-          with new_buffer = to_string(extract(to_byte_array(buffer), buf_pos, buf_size) + ba),
+          with new_buffer = extract(buffer, buf_pos, buf_size) + ba,
         //  println("TRUNCATED New buffer length "+length(new_buffer)+" new current_pos "+current_pos);
           find_the_first(io, looking_for, size, new_buffer, current_pos + buf_pos, length(new_buffer), 0)
     }
   else
-    if find_string(buffer, looking_for, buf_pos) is
+    if find_byte_array(buffer, looking_for, buf_pos) is
     {
-      failure       then find_the_first(io, looking_for, size, buffer, current_pos, buf_size, buf_size - (size-1)),
+      failure       then 
+        find_the_first(io, looking_for, size, buffer, current_pos, buf_size, buf_size - (size-1)),
       success(pos)  then  
-        println("pattern ["+looking_for+"] found at offset "+(current_pos+pos)); 
+        //println("pattern ["+to_string(looking_for)+"] found at offset "+(current_pos+pos)); 
         success(current_pos + pos)
     }
   .
@@ -1126,7 +1133,7 @@ public define Maybe(Int)
     Data_IO io,  
     String  search_string
   ) =
-  find_the_first(io, search_string, length(search_string), "", 0, 0, 0).
+  find_the_first(io, to_byte_array(search_string), length(search_string), constant_byte_array(0,0), 0, 0, 0).
 public define Maybe(Int)
   find_the_first
@@ -1138,7 +1145,7 @@ public define Maybe(Int)
   {
     failure    then failure,
     success(f) then 
-      find_the_first(make_data_io(f), search_string, length(search_string), "", 0, 0, 0)
+      find_the_first(make_data_io(f), to_byte_array(search_string), length(search_string), constant_byte_array(0,0), 0, 0, 0)
   }.
 public define Maybe(Int)
@@ -1153,7 +1160,8 @@ public define Maybe(Int)
     failure    then failure,
     success(f) then 
       with size = file_size(filename),
-      find_the_first(make_data_io(f, start_position, size - start_position), search_string, length(search_string), "", 0, 0, 0)
+      //println("file size "+size);
+      find_the_first(make_data_io(f, start_position, size - start_position), to_byte_array(search_string), length(search_string), constant_byte_array(0,0), 0, 0, 0)
   }.
 public define Maybe(Int)
@@ -1168,6 +1176,6 @@ public define Maybe(Int)
   {
     failure    then failure,
     success(f) then 
-      find_the_first(make_data_io(f, start_position, end_position - start_position), search_string, length(search_string), "", 0, 0, 0)
+      find_the_first(make_data_io(f, start_position, end_position - start_position), to_byte_array(search_string), length(search_string), constant_byte_array(0,0), 0, 0, 0)
   }.
@@ -50,6 +50,16 @@ define One
    anubis_format_aux(s,b,bpl,i+1,c+1,ind). 
-
+public define ByteArray
+   concat
+   (
+     List(ByteArray)  l
+   ) =
+   if l is 
+   {
+     [ ] then constant_byte_array(0,0), 
+     [h . t] then h + concat(t)
+   }. 
+  
@@ -14,15 +14,15 @@
    *** (1) Absolute value.
- public define macro       Int      abs(Int x).                    returns the absolute value of x
+ public define inline       Int      abs(Int x).                    returns the absolute value of x
    *** (2) Max and min. 
    The 'max' and 'min' functions respectively return the greatest and the smallest of their arguments. 
-public define              Int      max(Int x, Int y).             returns the greatest of x and y
-public define              Int      min(Int x, Int y).             returns the smallest of x and y
+public define inline       Int      max(Int x, Int y).             returns the greatest of x and y
+public define inline       Int      min(Int x, Int y).             returns the smallest of x and y
  public define              Int      max(NonEmptyList(Int) l).      returns the greatest element of the list (which is non empty) 
  public define              Int      min(NonEmptyList(Int) l).      returns the smallest element of the list (which is non empty) 
@@ -63,20 +63,25 @@ public define              Int       Int x ^ Int y.            raises x to the p
 public define              Bool      odd     (Int x).       
 public define              Bool      even    (Int x).       
+   *** (7) Greatest common divisor, ... 
+ 
+public define              Int       gcd     (Int x, Int y).     
+public define              Int       lcm     (Int x, Int y).    least common multiple
+
    --- That's all for the public part ! -----------------------------------------------
-public define macro Int
+public define inline Int
    abs
      (
       Int  x
      ) =
    if x < 0 then -x else x.
-public define Int         // must not be macro because x and y are always computed
+public define inline Int         // must not be macro because x and y are always computed
                           // and one of them would be computed twice
   max
     (
@@ -116,7 +121,7 @@ public define Int
-public define Int
+public define inline Int
   min
     (
       Int x, 
@@ -226,6 +231,38 @@ public define Bool
   }. 
+public define Int       
+   gcd     
+   (
+     Int     x, 
+     Int     y
+   ) =
+   /* Euclid's algorithm */ 
+   if x > y then gcd(y,x) else
+   if x = 0 then abs(y) else
+   if x < 0 then gcd(-x,y) else
+   if y/x is
+    {
+      failure    then should_not_happen(0), 
+      success(p) then if p is (q,r) then gcd(r,x)
+    }. 
+
+
+
+public define Int       
+  lcm     
+  (
+    Int x, 
+    Int y
+  ) =
+  with d = gcd(x,y), 
+  if d = 0
+  then 0
+  else if (x*y)/d is 
+  {
+    failure    then should_not_happen(0), 
+    success(p) then if p is (q,_) then q 
+  }.
@@ -38,7 +38,7 @@
 //---------------------------------------------------------------------------
-read lexical_analysis/fast_lexer_3.anubis   
+read lexical_analysis/fast_lexer_4.anubis   
 type Token:
@@ -46,13 +46,13 @@ type Token:
    eol.
 public type LineReaderLexer:
-  line_reader_lexer(LexingStream(One) -> One -> LexerOutput(Token) /*lexer_base*/).
+  line_reader_lexer((LexingStream, One) -> One -> LexerOutput(Token) /*lexer_base*/).
 public type LineReader:
   line_reader(One -> LexerOutput(Token)   /*lexer*/,
               One -> Int                  /*offset*/,
               LineReaderLexer             /*lexer_base*/,
-              LexingStream(One)           lexing_stream).
+              LexingStream           lexing_stream).
 public define Int
   current_offset
@@ -96,30 +96,30 @@ public define Maybe(String)
 public define Maybe(LineReaderLexer)
   make_line_reader_lexer
   =
-  if make_lexer_and_automaton([
-                                lexer_item("#r?#n", return((ByteArray b, LexingTools t, One aux) |-> token(eol))),
-                                lexer_item("#r", return((ByteArray b, LexingTools t, One aux) |-> token(eol))),
-                                lexer_item("[^\r\n]*", return((ByteArray b, LexingTools t, One aux) |-> token(line(to_string(b))))),
-                              ],
+  if make_lexer([
+                  lexer_item("#r?#n", return((ByteArray b, LexingTools t, One aux) |-> token(eol))),
+                  lexer_item("#r", return((ByteArray b, LexingTools t, One aux) |-> token(eol))),
+                  lexer_item("[^\r\n]*", return((ByteArray b, LexingTools t, One aux) |-> token(line(to_string(b))))),
+                ],
                               '#') is
   {
     error(msg) then print("Syntax error in regular expression: "+to_English(msg)+"\n"); failure,
-    ok(p) then if p is (lexer, automaton) then success(line_reader_lexer(lexer))
+    ok(lexer)  then success(line_reader_lexer(lexer))
   }.
 public define LineReader
   make_line_reader
   (
-    LexingStream(One)    ls,
+    LexingStream    ls,
     LineReaderLexer make_lexer
   ) =
   if make_lexer is line_reader_lexer(lexer) then
-  line_reader(lexer(ls), (One u) |-> offset(ls), make_lexer, ls).
+  line_reader(lexer(ls, unique), (One u) |-> offset(ls), make_lexer, ls).
 public define Maybe(LineReader)
   make_line_reader
   (
-    LexingStream(One)  ls,
+    LexingStream  ls,
   ) =
   if make_line_reader_lexer is
   {
@@ -135,7 +135,7 @@ public define Maybe(LineReader)
   (
     String  s,
   ) =
-  make_line_reader(make_lexing_stream("", s, unique)).
+  make_line_reader(make_lexing_stream("", s)).
 public define Maybe(LineReader)
   make_line_reader
@@ -146,8 +146,7 @@ public define Maybe(LineReader)
   if make_lexing_stream("",       /* preambule */
                         f,        /* the opened file */
                         65536,    /* size of buffer for the lexing stream */
-                        timeout,  /* timeout (seconds) */
-                        unique)  
+                        timeout)  /* timeout (seconds) */
   is
   {
     failure     then print("cannot make lexing stream.\n"); failure, 
@@ -159,7 +158,7 @@ public define LineReader
   reset_line_reader
   (
     LineReader    lr,
-    LexingStream(One)  ls,
+    LexingStream  ls,
   ) =
   if lr is line_reader(lexer, offset, make_lexer, _) then
   make_line_reader(ls, make_lexer).
@@ -922,3 +922,33 @@ public define (List($T1), List($T2)) unzip(List(($T1, $T2)) l) = unzip(reverse(l
+   Compute the list of all sublists of a list (beware: for a list of length n, 
+   this gives a list of length 2^n). 
+   
+public define List(List($T))
+   sublists
+   (
+     List($T)  l
+   ) =
+   if l is 
+   {
+     [ ] then [[ ]], 
+     [h . t] then with p = sublists(t),
+       map((List($T) u) |-> [h . u],p) + p
+   }.
+
+
+   Construct a 'constant' list containing any number of times the same datum. 
+   
+   constant_list(x,6) is just:  [x,x,x,x,x,x] 
+   
+public define List($T)
+  constant_list
+  (
+    $T     x, 
+    Int    n 
+  ) =
+  if n =< 0 then [ ] else [x . constant_list(x,n-1)]. 
+  
+  
+  
-transmit data_base/import_csv.anubis    (since version 1.6)
+  Obsolete file. 
+
+ transmit data_base/import_csv.anubis    (since version 1.6)
@@ -1011,6 +1011,12 @@ independently of the many definitions you may give to the symbol \cod{[~]} and t
 This is enough on syntax for reading the subsequent sections. We will say more on syntax later.
+\section{\cod{\_\_LINE\_\_}, \cod{\_\_FILE\_\_}, \cod{\_\_DIR\_\_} and \cod{\_\_TIME\_\_}}
+The keyword \cod{\_\_LINE\_\_} represents the line number (of the source file) where this keyword is written. This
+is a datum of type \cod{Word32}. Similarly, \cod{\_\_FILE\_\_} and \cod{\_\_DIR\_\_} represent the absolute file path and
+absolute directory path of the source file they are written within. They are of type \cod{String}. The keyword
+\cod{\_\_TIME\_\_} instantiates to the current time (of type \cod{Word32}) at
+which it is found in the source file during the compilation.
 \section{Automatically generated files}
@@ -1019,7 +1025,7 @@ an ``automatically generated file&#39;&#39;. There are several examples
 of metaprograms in the library such as~: \APG\ (see section \myref{sec:APG}), \cod{fast\_lexer\_3} (see section
     \myref{sec:fastlexer}), \cod{metaSQL} (see section \myref{sec:metaSQL}), \dots\ 
-Autmatically generated files should never be modified because any modification is destroyed by a new execution of the
+Automatically generated files should never be modified because any modification is destroyed by a new execution of the
 metaprogram. Only true source files should be modified. As a consequence, generated files encomber your working
 directory, and we have adopted the policy of putting them into a subdirectory (of the current directory), uniformly named \fn{generated}. 
 Putting generated files into the \fn{generated} subdirectory must be the default behavior of metaprograms. An option of the