Anubis / Anubis

Commit 1ad9dbea8cbfc36f9bc5abaa3594b58bb3719a6a

Authored by Alain Prouté
1 parent eb3ab906

-

Showing 2 changed files with 1917 additions and 24 deletions   Show diff stats
Inline Side-by-side
anubis_dev/library/syntactic_analysis/anubis_output.anubis
  Show/Hide comments   View file @1ad9dbe
... ... @@ -833,16 +833,14 @@ define One
833 833 define One
834 834 print_token_type_alts
835 835 (
836   - Stream s,
837   - List(APG_Token) tokens
  836 + Stream s,
  837 + List(String) all_tokens
838 838 ) =
839   - if tokens is
  839 + if all_tokens is
840 840 {
841 841 [ ] then unique,
842   - [h . t ] then with nh = name(h),
843   - print(s,"\n "+nh+(if t is [] then "." else ","));
844   - print(s,constant_string(30-length(nh),' '));
845   - print(s,if h is unused(_) then " // WARNING: this token is not used !" else "");
  842 + [h . t ] then
  843 + print(s,"\n "+h+(if t is [] then "." else ","));
846 844 print_token_type_alts(s,t)
847 845 }.
848 846  
... ... @@ -851,26 +849,24 @@ define One
851 849 (
852 850 Stream s,
853 851 String parser_name,
854   - List(APG_Token) tokens
  852 + List(String) all_tokens
855 853 ) =
856 854 print(s,"\n\npublic type Token_"+parser_name+":");
857   - print_token_type_alts(s,tokens).
  855 + print_token_type_alts(s,all_tokens).
858 856  
859 857  
860 858 define One
861 859 print_token_value_type_alts
862 860 (
863 861 Stream s,
864   - List(APG_Token) tokens,
  862 + List(String) all_tokens,
865 863 List(TypeEntry) type_table
866 864 ) =
867   - if tokens is
  865 + if all_tokens is
868 866 {
869 867 [ ] then unique,
870   - [h . t ] then with nh = name(h),
871   - print(s,"\n "+nh+"("+get_type(nh,type_table)+")"+(if t is [] then "." else ","));
872   - print(s,constant_string(30-length(nh),' '));
873   - print(s,if h is unused(_) then " // WARNING: this token is not used !" else "");
  868 + [h . t ] then
  869 + print(s,"\n "+h+"("+get_type(h,type_table)+")"+(if t is [] then "." else ","));
874 870 print_token_value_type_alts(s,t,type_table)
875 871 }.
876 872  
... ... @@ -879,11 +875,11 @@ define One
879 875 (
880 876 Stream s,
881 877 String parser_name,
882   - List(APG_Token) tokens,
  878 + List(String) all_tokens,
883 879 List(TypeEntry) type_table
884 880 ) =
885 881 print(s,"\n\npublic type Token_Value_"+parser_name+":");
886   - print_token_value_type_alts(s,tokens,type_table).
  882 + print_token_value_type_alts(s,all_tokens,type_table).
887 883  
888 884  
889 885  
... ... @@ -1764,6 +1760,7 @@ define One
1764 1760 read make_automaton.anubis
1765 1761  
1766 1762  
  1763 +
1767 1764 define One
1768 1765 do_output
1769 1766 (
... ... @@ -1775,9 +1772,6 @@ define One
1775 1772 if g is grammar(preambule,parser_name,prec_decs,type_decs,rules,mb_extra,postambule) then
1776 1773 with non_terminals = (List(String))["start" . all_non_terminals(rules)],
1777 1774 all_tokens = (List(String))["eof" . all_symbols(rules) - non_terminals],
1778   - unused_tokens = no_doubles((merge(tokens_from_type_decs(type_decs),
1779   - tokens_from_precs(prec_decs)) - all_tokens) - non_terminals),
1780   - used_and_unused_tokens = qsort(map(used,all_tokens)+map(unused,unused_tokens),alphaless),
1781 1775 prec_table = make_precedence_table(prec_decs),
1782 1776 assoc_table = make_association_table(prec_decs),
1783 1777 _A = if get_rule(1,rules) is grammar_rule(_,_A,_,_) then _A,
... ... @@ -1796,8 +1790,8 @@ define One
1796 1790 print(s,"\n The public types 'Token_"+parser_name+"' and 'Token_Value_"+parser_name+"'\n");
1797 1791 print(s," are defined below in this file.\n\n");
1798 1792 print(s,preambule);
1799   - print_token_type(s,parser_name,used_and_unused_tokens);
1800   - print_token_value_type(s,parser_name,used_and_unused_tokens,type_table);
  1793 + print_token_type(s,parser_name,all_tokens);
  1794 + print_token_value_type(s,parser_name,all_tokens,type_table);
1801 1795 print_non_terminals_type(s,non_terminals,type_table);
1802 1796 print_type_Ret(s,parser_name);
1803 1797 print_Lexer_type(s,parser_name);
... ... @@ -1810,7 +1804,7 @@ define One
1810 1804 type_table,parser_name,mb_extra,options);
1811 1805  
1812 1806 map_forget((APG_State st) |->
1813   - if st is state(state_id,classes,transitions) then
  1807 + if st is state(state_id,classes,transitions) then
1814 1808 with scs = flat_classes(classes),
1815 1809 print_state_function_declaration(s,
1816 1810 state_id,
... ... @@ -1825,7 +1819,7 @@ define One
1825 1819 map_forget((APG_State state) |->
1826 1820 print_state(s,
1827 1821 state,
1828   - map(name,used_and_unused_tokens), // all_tokens,
  1822 + all_tokens,
1829 1823 prec_table,
1830 1824 assoc_table,
1831 1825 type_table,
... ...
anubis_dev/library/syntactic_analysis/anubis_output.anubis.test 0 → 100644
  Show/Hide comments   View file @1ad9dbe
  1 +
  2 +
  3 +
  4 +
  5 + *Project* Anubis
  6 +
  7 + *Title* Outputing a parser automaton in Anubis.
  8 +
  9 + *Copyright* Copyright (c) Alain Proute' 2006.
  10 +
  11 +
  12 +
  13 + *Author* Alain Proute'
  14 +
  15 + *Created* March 2006
  16 + *Revised* March 2006
  17 +
  18 +
  19 + *Overview*
  20 +
  21 + The tool defined in this file is part of the Anubis Parser Generator (APG). Given an
  22 + abstract parser automaton (as computed by `make_APG_automaton`; see
  23 + `make_automaton.anubis`), it outputs the automaton in the form of an Anubis program.
  24 +
  25 +
  26 +
  27 + *Public*
  28 +
  29 +read common.anubis
  30 +read tools/streams.anubis
  31 +
  32 +
  33 +
  34 + *Name* anubis_output
  35 +
  36 + *Description*
  37 +
  38 + This function takes a grammar and outputs a corresponding parser in Anubis into a file.
  39 +
  40 +public define One
  41 + anubis_output
  42 + (
  43 + APG_Grammar g,
  44 + String output_filename,
  45 + List(APG_Option) options
  46 + ).
  47 +
  48 +
  49 +
  50 +
  51 + *Private*
  52 +
  53 +read tools/basis.anubis
  54 +
  55 + ------------------------------- Table of Contents -------------------------------------
  56 +
  57 + *** [1] Precedence, association mode and type tables.
  58 + *** [1.1] The precedence table.
  59 + *** [1.1.1] Computing the precedence table.
  60 + *** [1.1.2] Using the precedence table.
  61 + *** [1.1.3] Printing the precedence table.
  62 + *** [1.2] The association mode table.
  63 + *** [1.2.1] Computing the association mode table.
  64 + *** [1.2.2] Using the association mode table.
  65 + *** [1.2.3] Printing the association mode table.
  66 + *** [1.3] The type table.
  67 + *** [1.3.1] Computing the type table.
  68 + *** [1.3.2] Using the type table.
  69 + *** [1.3.3] Printing the type table.
  70 + *** [1.4] Flattening classes.
  71 + *** [1.5] Getting the longuest stack for a state.
  72 + *** [1.6] Getting a grammar rule by its id.
  73 + *** [1.7] Computing the precedence level of a grammar rule.
  74 +
  75 + *** [2] Resolving conflicts.
  76 + *** [2.1] Computing the list of behaviors for a state.
  77 + *** [2.2] Making the list of conflicts for a state.
  78 + *** [2.2.1] Computing the resolution of a shift/reduce conflict.
  79 + *** [2.2.2] Computing all conflicts and their resolutions.
  80 + *** [2.2.3] Getting the resolution of a conflict.
  81 + *** [2.3] Printing conflicts.
  82 +
  83 + *** [3] Outputing the parser program.
  84 + *** [3.1] Outputing parser specific types.
  85 + *** [3.1.1] The types of tokens.
  86 + *** [3.1.2] The type of non terminals.
  87 + *** [3.1.3] The type 'Ret'.
  88 + *** [3.1.4] Outputing the 'Lexer' type.
  89 + *** [3.2] Outputing the declaration of the function 'vmsg'.
  90 + *** [3.3] Outputing the 'reduce_n' functions.
  91 + *** [3.5] Outputing states.
  92 + *** [3.5.1] Printing the state header.
  93 + *** [3.5.2] Printing scenarios.
  94 + *** [3.5.3] Printing the transitions.
  95 + *** [3.5.4] Printing the reductions.
  96 + *** [3.5.5] Printing acceptable tokens.
  97 + *** [3.5.6] Printing the restart function
  98 + *** [3.5.7] Printing the state function.
  99 + *** [3.5.8] Printing a whole state.
  100 +
  101 + *** [4] The interface.
  102 +
  103 + *** [5] Testing.
  104 +
  105 + ---------------------------------------------------------------------------------------
  106 +
  107 +
  108 +define One print(Stream s, String t) = forget(write_string(s,t)).
  109 +
  110 +
  111 + *** [1] Precedence, association mode and type tables.
  112 +
  113 + The grammar contains precedence/association declarations and type declarations. We must
  114 + organize these data into several tables (actually association lists):
  115 +
  116 + - a 'precedence table' with entries of the form (symbol,precedence_level),
  117 + - a 'association mode table' with entries of the form (precedence_level,mode),
  118 + - a 'type table' with entries of the form (symbol,type).
  119 +
  120 +
  121 +
  122 + *** [1.1] The precedence table.
  123 +
  124 + *** [1.1.1] Computing the precedence table.
  125 +
  126 +type PrecEntry:
  127 + prec_entry(String symbol,
  128 + Word32 level).
  129 +
  130 +define List(PrecEntry)
  131 + make_precedence_table
  132 + (
  133 + List(APG_Precedence_Dec) decs
  134 + ) =
  135 + if decs is
  136 + {
  137 + [ ] then [ ],
  138 + [h . t] then
  139 + with n = level(h),
  140 + names = symbol_names(h),
  141 + rest = make_precedence_table(t),
  142 + map((String s) |-> prec_entry(s,n),
  143 + names)
  144 + + rest
  145 + }.
  146 +
  147 +
  148 +
  149 +
  150 + *** [1.1.2] Using the precedence table.
  151 +
  152 +define Maybe(Word32)
  153 + get_precedence
  154 + (
  155 + String symbol,
  156 + List(PrecEntry) prec_table
  157 + ) =
  158 + if prec_table is
  159 + {
  160 + [ ] then failure,
  161 + [h . t] then if h is prec_entry(s,l) then
  162 + if symbol = s
  163 + then success(l)
  164 + else get_precedence(symbol,t)
  165 + }.
  166 +
  167 +
  168 +
  169 +
  170 + *** [1.1.3] Printing the precedence table.
  171 +
  172 +define One
  173 + print
  174 + (
  175 + Stream s,
  176 + List(PrecEntry) prec_table
  177 + ) =
  178 + print(s,"\n --- Precedence table ---");
  179 + map_forget((PrecEntry e) |-> if e is prec_entry(sym,level) then
  180 + print(s,"\n "+right_pad(sym,15)+" "+level),
  181 + prec_table).
  182 +
  183 +
  184 +
  185 +
  186 +
  187 +
  188 +
  189 + *** [1.2] The association mode table.
  190 +
  191 + *** [1.2.1] Computing the association mode table.
  192 +
  193 +type AssocMode:
  194 + left,
  195 + right,
  196 + non_assoc.
  197 +
  198 +
  199 +define String
  200 + to_string
  201 + (
  202 + AssocMode m
  203 + ) =
  204 + if m is
  205 + {
  206 + left then "left",
  207 + right then "right",
  208 + non_assoc then "non_assoc"
  209 + }.
  210 +
  211 +type AssocEntry:
  212 + assoc_entry(Word32 level,
  213 + AssocMode mode).
  214 +
  215 +define List(AssocEntry)
  216 + make_association_table
  217 + (
  218 + List(APG_Precedence_Dec) decs
  219 + ) =
  220 + if decs is
  221 + {
  222 + [ ] then [ ],
  223 + [h . t] then
  224 + [if h is
  225 + {
  226 + left(level,names) then assoc_entry(level,left),
  227 + right(level,names) then assoc_entry(level,right),
  228 + non_assoc(level,names) then assoc_entry(level,non_assoc),
  229 + }
  230 + . make_association_table(t)]
  231 + }.
  232 +
  233 +
  234 +
  235 + *** [1.2.2] Using the association mode table.
  236 +
  237 +define Maybe(AssocMode)
  238 + get_association_mode
  239 + (
  240 + Word32 level,
  241 + List(AssocEntry) assoc_table
  242 + ) =
  243 + if assoc_table is
  244 + {
  245 + [ ] then failure,
  246 + [h . t] then if h is assoc_entry(l,mode) then
  247 + if l = level
  248 + then success(mode)
  249 + else get_association_mode(level,t)
  250 + }.
  251 +
  252 +
  253 +
  254 + *** [1.2.3] Printing the association mode table.
  255 +
  256 +define One
  257 + print
  258 + (
  259 + Stream s,
  260 + List(AssocEntry) assoc_table
  261 + ) =
  262 + print(s,"\n --- Association mode table ---");
  263 + map_forget((AssocEntry e) |-> if e is assoc_entry(level,mode) then
  264 + print(s,"\n "+right_pad(""+level,15)+" "+to_string(mode)),
  265 + assoc_table).
  266 +
  267 +
  268 +
  269 + *** [1.3] The type table.
  270 +
  271 +type TypeEntry:
  272 + type_entry(String symbol,
  273 + String type).
  274 +
  275 +
  276 +
  277 + *** [1.3.1] Using the type table.
  278 +
  279 +define String
  280 + get_type
  281 + (
  282 + String symbol,
  283 + List(TypeEntry) type_table
  284 + ) =
  285 + if type_table is
  286 + {
  287 + [ ] then "One",
  288 + [h . t] then if h is type_entry(s,type) then
  289 + if symbol = s
  290 + then type
  291 + else get_type(symbol,t)
  292 + }.
  293 +
  294 +
  295 +
  296 + *** [1.3.2] Computing the type table.
  297 +
  298 +define List(TypeEntry)
  299 + make_type_table_1
  300 + (
  301 + List(APG_Type_Dec) decs
  302 + ) =
  303 + if decs is
  304 + {
  305 + [ ] then [ ],
  306 + [h . t] then if h is type_dec(type,names) then
  307 + map((String n) |-> type_entry(n,type),
  308 + names)
  309 + + make_type_table_1(t)
  310 + }.
  311 +
  312 +define List(TypeEntry)
  313 + make_type_table
  314 + (
  315 + List(APG_Type_Dec) decs,
  316 + String axiom
  317 + ) =
  318 + with tt = make_type_table_1(decs),
  319 + t = get_type(axiom,tt),
  320 + [type_entry("start",t) . tt].
  321 +
  322 +
  323 +
  324 +
  325 + *** [1.3.3] Printing the type table.
  326 +
  327 +define One
  328 + print
  329 + (
  330 + Stream s,
  331 + List(TypeEntry) type_table
  332 + ) =
  333 + print(s,"\n --- Type table ---\n");
  334 + map_forget((TypeEntry e) |-> if e is type_entry(sym,type) then
  335 + print(s,"\n "+right_pad(sym,15)+" "+type),
  336 + type_table).
  337 +
  338 +
  339 +
  340 +
  341 +
  342 +
  343 + *** [1.4] Flattening classes.
  344 +
  345 + We do no more need classes of scenarios, and put all scenarios of a set of classes into
  346 + a single list of scenarios.
  347 +
  348 +define List(APG_Scenario)
  349 + flat_classes
  350 + (
  351 + List(APG_Class) classes
  352 + ) =
  353 + if classes is
  354 + {
  355 + [ ] then [ ],
  356 + [c1 . others] then
  357 + scenarios(c1) + flat_classes(others)
  358 + }.
  359 +
  360 +
  361 +
  362 +
  363 +
  364 + *** [1.5] Getting the longuest stack for a state.
  365 +
  366 + For each state, there is a longuest sequence of grammar symbols before the dot in the
  367 + scenarios of the state. This sequence is the 'longuest stack' for that state.
  368 +
  369 +define List(String)
  370 + get_longuest_stack
  371 + (
  372 + List(APG_Scenario) l
  373 + ) =
  374 + if l is
  375 + {
  376 + [ ] then [ ],
  377 + [sc1 . other_scenarios] then
  378 + if sc1 is scenario(_,_,l1,_,_,_,_) then
  379 + with l2 = get_longuest_stack(other_scenarios),
  380 + if length(l1) > length(l2) then l1 else l2
  381 + }.
  382 +
  383 +
  384 +
  385 +
  386 + *** [1.6] Getting a grammar rule by its id.
  387 +
  388 +define APG_Grammar_Rule
  389 + get_rule
  390 + (
  391 + Word32 rule_id,
  392 + List(APG_Grammar_Rule) rules
  393 + ) =
  394 + if rules is
  395 + {
  396 + [ ] then should_not_happen(grammar_rule(0,symbol_value("",""),[],failure)),
  397 + [g1 . others] then
  398 + if g1 is grammar_rule(id,_,_,_) then
  399 + if id = rule_id
  400 + then g1
  401 + else get_rule(rule_id,others)
  402 + }.
  403 +
  404 +
  405 +
  406 +
  407 +
  408 + *** [1.7] Computing the precedence level of a grammar rule.
  409 +
  410 + If the rule has a declared precedence, this is the precedence of the rule. Otherwise,
  411 + it is the precedence of the rightmost token in its body (if there a token with a
  412 + declared precedence in the body). Otherwise, there is no precedence level.
  413 +
  414 + The first function computes the precedence using the body of rule only.
  415 +
  416 +define Maybe(Word32)
  417 + compute_rule_precedence
  418 + (
  419 + List(String) body, // in reverse order
  420 + List(PrecEntry) prec_table
  421 + ) =
  422 + if body is
  423 + {
  424 + [ ] then failure,
  425 + [last . others] then
  426 + if get_precedence(last,prec_table) is
  427 + {
  428 + failure then compute_rule_precedence(others,prec_table),
  429 + success(level) then success(level)
  430 + }
  431 + }.
  432 +
  433 +define Maybe(Word32)
  434 + compute_rule_precedence
  435 + (
  436 + List(String) body,
  437 + Maybe(String) mb_prec,
  438 + List(PrecEntry) prec_table
  439 + ) =
  440 + if mb_prec is
  441 + {
  442 + failure then compute_rule_precedence(reverse(body),prec_table)
  443 + success(sym) then
  444 + if get_precedence(sym,prec_table) is
  445 + {
  446 + failure then success(-1), // a token used for a rule precedence should have a precedence
  447 + success(level) then success(level)
  448 + }
  449 + }.
  450 +
  451 +
  452 +
  453 +
  454 +
  455 +
  456 +
  457 +
  458 +
  459 +
  460 +
  461 + *** [2] Resolving conflicts.
  462 +
  463 + When it reads a token 't' from the input in some state 'S', the automaton must choose
  464 + between 3 possible behaviors:
  465 +
  466 + - 'shifting' the token and making a transition to some state,
  467 + - 'reducing' using some grammar rule,
  468 + - reporting an error.
  469 +
  470 + Possible behaviors are represented by the following type. We don't record error
  471 + behaviors in our list of behaviors for a state, because by convention, if a token is
  472 + not recorded in the list, it generates an error. Furthermore, we record restarting
  473 + transitions, not only shifting transitions.
  474 +
  475 +type Behavior:
  476 + restart (Word32 target_state_id),
  477 + shift (Word32 target_state_id),
  478 + reduce (Word32 grammar_rule_id).
  479 +
  480 +type BehaviorEntry:
  481 + behavior (String symbol,
  482 + Behavior behavior).
  483 +
  484 + We will compute a list of 'BehaviorEntry' below for each state.
  485 +
  486 +
  487 + A 'conflict' arises when the automaton has several choices. The conflicts are of two
  488 + sorts:
  489 +
  490 + - 'shift/reduce' conflict: the automaton may either shift or reduce,
  491 + - 'reduce/reduce' conflict: the automaton has several different grammar rules for
  492 + reducing.
  493 +
  494 + Shift/reduce conflict may always be resolved by declaring appropriate precedence levels
  495 + and association modes. On the contrary, reduce/reduce conflicts cannot be resolved that
  496 + way. They are in general the result of a bad design of the grammar itself.
  497 +
  498 +
  499 + The resolution of a shift/reduce conflict is of the following type:
  500 +
  501 +type Resolved_As:
  502 + non_assoc_error, // cannot be resolved because of non_assoc declaration
  503 + unresolved, // unresolved because some precedence rule is missing
  504 + no_token_level(Word32 rule_level), // idem
  505 + no_rule_level(Word32 token_level), // idem
  506 + shift1(Word32 tok_level, Word32 rule_level), // resolved as 'shift'
  507 + shift2(Word32 level, AssocMode mode), // idem
  508 + reduce1(Word32 tok_level, Word32 rule_level), // resolved as 'reduce'
  509 + reduce2(Word32 level, AssocMode mode). // idem
  510 +
  511 +
  512 + Conflicts are represented as follows:
  513 +
  514 +type Conflict:
  515 + reduce_reduce(String token),
  516 + shift_reduce (String token,
  517 + Resolved_As resolution).
  518 +
  519 +
  520 +define (Word32,Word32)
  521 + number_of_conflicts
  522 + (
  523 + List(Conflict) l,
  524 + Word32 sr_so_far,
  525 + Word32 rr_so_far
  526 + ) =
  527 + if l is
  528 + {
  529 + [ ] then (sr_so_far,rr_so_far),
  530 + [h . t] then if h is
  531 + {
  532 + reduce_reduce(_) then number_of_conflicts(t,sr_so_far,rr_so_far+1),
  533 + shift_reduce(_,ra) then if ra is
  534 + {
  535 + non_assoc_error then number_of_conflicts(t,sr_so_far,rr_so_far),
  536 + unresolved then number_of_conflicts(t,sr_so_far+1,rr_so_far),
  537 + no_token_level(_) then number_of_conflicts(t,sr_so_far+1,rr_so_far),
  538 + no_rule_level(_) then number_of_conflicts(t,sr_so_far+1,rr_so_far),
  539 + shift1(_,_) then number_of_conflicts(t,sr_so_far,rr_so_far),
  540 + shift2(_,_) then number_of_conflicts(t,sr_so_far,rr_so_far),
  541 + reduce1(_,_) then number_of_conflicts(t,sr_so_far,rr_so_far),
  542 + reduce2(_,_) then number_of_conflicts(t,sr_so_far,rr_so_far)
  543 + }
  544 + }
  545 + }.
  546 +
  547 +
  548 +
  549 + *** [2.1] Computing the list of behaviors for a state.
  550 +
  551 + The behaviors are comming from two sources:
  552 +
  553 + - the reducing scenarios, which produce one behavior per lookahead,
  554 + - transitions, which produce either shifting behaviors or restarting behaviors
  555 +
  556 +define List(BehaviorEntry)
  557 + compute_behaviors
  558 + (
  559 + List(APG_Scenario) scs, // all scenarios in the state
  560 + List(APG_Transition) transitions, // all transitions of the state
  561 + List(String) all_tokens // all symbols which are tokens
  562 + ) =
  563 + if scs is
  564 + {
  565 + [ ] then map((APG_Transition tr) |->
  566 + if tr is transition(sym,target_id) then
  567 + if sym:all_tokens
  568 + then behavior(sym,shift(target_id))
  569 + else behavior(sym,restart(target_id)),
  570 + transitions),
  571 + [sc1 . others] then
  572 + if sc1 is scenario(rule_id,head,bd,ad,prop,hg,lh_v) then
  573 + if ad is
  574 + {
  575 + [ ] then map((String lookahead) |->
  576 + behavior(lookahead,reduce(rule_id)),
  577 + *lh_v)
  578 + + compute_behaviors(others,transitions,all_tokens),
  579 + [_ . _] then compute_behaviors(others,transitions,all_tokens)
  580 + }
  581 + }.
  582 +
  583 +
  584 +
  585 +
  586 +define Bool
  587 + has_restarts
  588 + (
  589 + List(BehaviorEntry) behaviors
  590 + ) =
  591 + if behaviors is
  592 + {
  593 + [ ] then false,
  594 + [h . t] then
  595 + if h is behavior(sym,b) then
  596 + if b is
  597 + {
  598 + restart(_) then true,
  599 + shift(_) then has_restarts(t),
  600 + reduce(_) then has_restarts(t)
  601 + }
  602 + }.
  603 +
  604 +
  605 +
  606 +
  607 +
  608 + *** [2.2] Making the list of conflicts for a state.
  609 +
  610 +
  611 + *** [2.2.1] Computing the resolution of a shift/reduce conflict.
  612 +
  613 +define Resolved_As
  614 + compute_resolution // of shift/reduce conflict
  615 + (
  616 + String token, // the token that may be shifted
  617 + Word32 rule_id, // the id of the rule by which we may reduce
  618 + List(APG_Grammar_Rule) rules,
  619 + List(PrecEntry) prec_table,
  620 + List(AssocEntry) assoc_table
  621 + ) =
  622 + if get_rule(rule_id,rules) is grammar_rule(_,_,body,mb_prec) then
  623 + if compute_rule_precedence(map(name,body),mb_prec,prec_table) is
  624 + {
  625 + failure then if get_precedence(token,prec_table) is
  626 + {
  627 + failure then unresolved,
  628 + success(tl) then no_rule_level(tl)
  629 + },
  630 + success(rule_level) then
  631 + if get_precedence(token,prec_table) is
  632 + {
  633 + failure then no_token_level(rule_level),
  634 + success(token_level) then
  635 + if token_level +< rule_level then reduce1(token_level,rule_level) else
  636 + if token_level >+ rule_level then shift1 (token_level,rule_level) else
  637 + if get_association_mode(token_level,assoc_table) is
  638 + {
  639 + failure then should_not_happen(unresolved),
  640 + // if there is a precedence, there is an association mode
  641 + success(mode) then if mode is
  642 + {
  643 + left then reduce2(token_level,mode),
  644 + right then shift2 (token_level,mode),
  645 + non_assoc then non_assoc_error
  646 + }
  647 + }
  648 + }
  649 + }.
  650 +
  651 +
  652 +
  653 + *** [2.2.2] Computing all conflicts and their resolutions.
  654 +
  655 +define List(Conflict)
  656 + compute_conflicts
  657 + (
  658 + BehaviorEntry b1,
  659 + List(BehaviorEntry) others,
  660 + List(APG_Grammar_Rule) rules,
  661 + List(PrecEntry) prec_table,
  662 + List(AssocEntry) assoc_table
  663 + ) =
  664 + if others is
  665 + {
  666 + [ ] then [ ],
  667 + [b2 . rest] then
  668 + if b1 is behavior(sym1,a1) then
  669 + if b2 is behavior(sym2,a2) then
  670 + if sym1 = sym2
  671 + then if a1 is
  672 + {
  673 + restart (target_state_id1) then compute_conflicts(b1,rest,rules,prec_table,assoc_table),
  674 + shift (target_state_id1) then if a2 is
  675 + {
  676 + restart (target_state_id2) then compute_conflicts(b1,rest,rules,prec_table,assoc_table),
  677 + shift (target_state_id2) then compute_conflicts(b1,rest,rules,prec_table,assoc_table),
  678 + reduce (grammar_rule_id2) then
  679 + [shift_reduce(sym1,
  680 + compute_resolution(sym1,grammar_rule_id2,rules,prec_table,assoc_table))
  681 + . compute_conflicts(b1,rest,rules,prec_table,assoc_table)]
  682 + }
  683 + reduce (grammar_rule_id1) then if a2 is
  684 + {
  685 + restart (target_state_id2) then compute_conflicts(b1,rest,rules,prec_table,assoc_table),
  686 + shift (target_state_id2) then
  687 + [shift_reduce(sym1,
  688 + compute_resolution(sym1,grammar_rule_id1,rules,prec_table,assoc_table))
  689 + . compute_conflicts(b1,rest,rules,prec_table,assoc_table)],
  690 + reduce (grammar_rule_id2) then
  691 + [reduce_reduce(sym1)
  692 + . compute_conflicts(b1,rest,rules,prec_table,assoc_table)]
  693 + }
  694 + }
  695 + else compute_conflicts(b1,rest,rules,prec_table,assoc_table)
  696 + }.
  697 +
  698 +define List(Conflict)
  699 + compute_conflicts
  700 + (
  701 + List(BehaviorEntry) behaviors,
  702 + List(APG_Grammar_Rule) rules,
  703 + List(PrecEntry) prec_table,
  704 + List(AssocEntry) assoc_table
  705 + ) =
  706 + if behaviors is
  707 + {
  708 + [ ] then [ ],
  709 + [b1 . others] then
  710 + compute_conflicts(b1,others,rules,prec_table,assoc_table) +
  711 + compute_conflicts(others,rules,prec_table,assoc_table)
  712 + }.
  713 +
  714 +
  715 +
  716 + *** [2.2.3] Getting the resolution of a conflict.
  717 +
  718 +define Resolved_As
  719 + get_conflict_resolution
  720 + (
  721 + String token,
  722 + List(Conflict) conflicts
  723 + ) =
  724 + if conflicts is
  725 + {
  726 + [ ] then unresolved,
  727 + [h . t] then if h is
  728 + {
  729 + reduce_reduce(_) then get_conflict_resolution(token,t),
  730 + shift_reduce(tok,resol) then
  731 + if tok = token
  732 + then resol
  733 + else get_conflict_resolution(token,t)
  734 + }
  735 + }.
  736 +
  737 +
  738 +
  739 +
  740 +
  741 + *** [2.3] Printing conflicts.
  742 +
  743 + The function below prints the conflicts for a state (it prints nothing if no conflict).
  744 +
  745 +define One
  746 + print_conflicts
  747 + (
  748 + Stream s,
  749 + List(Conflict) conflicts
  750 + ) =
  751 + if conflicts is [] then unique else
  752 + print(s,"\n\n --- Conflicts ---\n");
  753 + map_forget((Conflict c) |-> if c is
  754 + {
  755 + reduce_reduce(token) then
  756 + print(s," "+right_pad(token,21)+" reduce/reduce\n"),
  757 +
  758 + shift_reduce(token,resol) then
  759 + print(s," "+right_pad(token,21)+" shift/reduce ");
  760 + if resol is
  761 + {
  762 + non_assoc_error then print(s,"(produces a 'non_assoc' syntax error)\n"),
  763 + unresolved then print(s,"%(* * * unresolved * * * ?/?)\n"),
  764 + no_token_level(rl) then print(s,"%(* * * unresolved * * * ?/"+rl+")\n"),
  765 + no_rule_level(tl) then print(s,"%(* * * unresolved * * * "+tl+"/?)\n"),
  766 + shift1(tl,rl) then print(s,"(resolved as 'shift' "+tl+"/"+rl+")\n"),
  767 + shift2(l,m) then print(s,"(resolved as 'shift' "+l+"/"+to_string(m)+")\n"),
  768 + reduce1(tl,rl) then print(s,"(resolved as 'reduce' "+tl+"/"+rl+")\n")
  769 + reduce2(l,m) then print(s,"(resolved as 'reduce' "+l+"/"+to_string(m)+")\n")
  770 + }
  771 + },conflicts).
  772 +
  773 +
  774 +
  775 +
  776 +
  777 +
  778 +
  779 +
  780 +
  781 +
  782 +
  783 +
  784 +
  785 +
  786 + *** [3] Outputing the parser program.
  787 +
  788 +
  789 + The LALR1 parser constructed by APG needs a stack. In this stack we have to push two
  790 + sorts of things:
  791 +
  792 + - return adresses (actually addresses of states of the automaton),
  793 + - values of grammar symbols.
  794 +
  795 + However, unlike YACC/BISON, APG does not implement a stack. On the contrary, it uses
  796 + the Anubis system stack as a stack for the automaton. This is a logical consequence of
  797 + the rigidity of the typing system of Anubis. An 'implemented' stack would have been an
  798 + array (or list) of heterogeneous data (i.e. data of some sum type). When working with
  799 + such a datum, a conditional would be necessary to determine its actual type. However,
  800 + the type of the datum is implicitly known by the automaton. Hence, this conditional
  801 + (even if mandatory if we use this method) is clearly a waste of time, and always the
  802 + same case would be used. In order to avoid this incongruity, we have designed a system
  803 + for using the Anubis system stack. The consequence is that the type of data in the
  804 + stack is always known at compile time. Note: In C there is no such problem, because the
  805 + designer may always cast to an appropriate type, which is not possible in Anubis for
  806 + safety reasons.
  807 +
  808 + For each state (say state number 'n'), we construct two functions: 'state_n' and
  809 + 'restart_n'. The function 'state_n' reads the next token and decides what to do with it
  810 + (either shift, reduce or report an error). When a reduction occurs, the 'state_n'
  811 + function returns a value, which is designed in such a way that the calling functions
  812 + will know at which state the returns must end. Then the state which ends the returns
  813 + (i.e. which captures in some sens the result of the reduction) calls its 'restart_?'
  814 + function in order to restart parsing from the right state.
  815 +
  816 + Reductions are performed by a set of 'reduce_n' functions (one per grammar rule).
  817 +
  818 +
  819 +
  820 +
  821 + *** [3.1] Outputing parser specific types.
  822 +
  823 + Our parser program works with several data types whose definitions must be output.
  824 +
  825 +
  826 +
  827 + *** [3.1.1] The types of tokens.
  828 +
  829 + We need two types of tokens, both having one alternative per token. The first one is an
  830 + enumeration. In the second one, each alternative has a component for holding the value
  831 + of the token.
  832 +
  833 +define One
  834 + print_token_type_alts
  835 + (
  836 + Stream s,
  837 + List(APG_Token) tokens
  838 + ) =
  839 + if tokens is
  840 + {
  841 + [ ] then unique,
  842 + [h . t ] then with nh = name(h),
  843 + print(s,"\n "+nh+(if t is [] then "." else ","));
  844 + print(s,constant_string(30-length(nh),' '));
  845 + print(s,if h is unused(_) then " // WARNING: this token is not used !" else "");
  846 + print_token_type_alts(s,t)
  847 + }.
  848 +
  849 +define One
  850 + print_token_type
  851 + (
  852 + Stream s,
  853 + String parser_name,
  854 + List(APG_Token) tokens
  855 + ) =
  856 + print(s,"\n\npublic type Token_"+parser_name+":");
  857 + print_token_type_alts(s,tokens).
  858 +
  859 +
  860 +define One
  861 + print_token_value_type_alts
  862 + (
  863 + Stream s,
  864 + List(APG_Token) tokens,
  865 + List(TypeEntry) type_table
  866 + ) =
  867 + if tokens is
  868 + {
  869 + [ ] then unique,
  870 + [h . t ] then with nh = name(h),
  871 + print(s,"\n "+nh+"("+get_type(nh,type_table)+")"+(if t is [] then "." else ","));
  872 + print(s,constant_string(30-length(nh),' '));
  873 + print(s,if h is unused(_) then " // WARNING: this token is not used !" else "");
  874 + print_token_value_type_alts(s,t,type_table)
  875 + }.
  876 +
  877 +define One
  878 + print_token_value_type
  879 + (
  880 + Stream s,
  881 + String parser_name,
  882 + List(APG_Token) tokens,
  883 + List(TypeEntry) type_table
  884 + ) =
  885 + print(s,"\n\npublic type Token_Value_"+parser_name+":");
  886 + print_token_value_type_alts(s,tokens,type_table).
  887 +
  888 +
  889 +
  890 +
  891 + *** [3.1.2] The type of non terminals.
  892 +
  893 + We need a type with an alternative per non terminal with a component holding the value.
  894 +
  895 +
  896 +define One
  897 + print_non_terminals_type_alts
  898 + (
  899 + Stream s,
  900 + List(String) non_terminals,
  901 + List(TypeEntry) type_table
  902 + ) =
  903 + if non_terminals is
  904 + {
  905 + [ ] then unique,
  906 + [h . t] then
  907 + print(s,"\n "+h+"("+get_type(h,type_table)+")"+(if t is [] then "." else ","));
  908 + print_non_terminals_type_alts(s,t,type_table)
  909 + }.
  910 +
  911 +define One
  912 + print_non_terminals_type
  913 + (
  914 + Stream s,
  915 + List(String) non_terminals,
  916 + List(TypeEntry) type_table
  917 + ) =
  918 + print(s,"\n\ntype Non_Terminal_Value:");
  919 + print_non_terminals_type_alts(s,non_terminals,type_table).
  920 +
  921 +
  922 +
  923 +
  924 + *** [3.1.3] The type 'Ret'.
  925 +
  926 +define One
  927 + print_type_Ret
  928 + (
  929 + Stream s,
  930 + String parser_name
  931 + ) =
  932 + print(s,"\n\ntype Ret($T):");
  933 + print(s,"\n error(Token_Value_"+parser_name+",List(Token_"+parser_name+")),");
  934 + print(s,"\n end_ret($T),");
  935 + print(s,"\n do_ret(Ret($T)).").
  936 +
  937 +
  938 +
  939 +
  940 +
  941 +
  942 +
  943 + *** [3.1.4] Outputing the 'Lexer' type.
  944 +
  945 + The parser gets the tokens from a 'lexer'. This lexer is an object of the following
  946 + type:
  947 +
  948 +define One
  949 + print_Lexer_type
  950 + (
  951 + Stream s,
  952 + String parser_name
  953 + ) =
  954 + print(s,"\n\ntype Lexer:");
  955 + print(s,"\n lexer(One -> Token_Value_"+parser_name+" read_token,");
  956 + print(s,"\n Token_Value_"+parser_name+" -> One unput_token).").
  957 +
  958 +
  959 +
  960 +
  961 +
  962 + *** [3.1.5] Outputing the parser function.
  963 +
  964 +define One
  965 + print_parser_function
  966 + (
  967 + Stream s,
  968 + String parser_name,
  969 + List(TypeEntry) type_table,
  970 + Maybe(Extra) mb_extra
  971 + ) =
  972 + print(s,"\n\npublic define Result((Token_Value_"+parser_name+",List(Token_"+parser_name+")),\n"+
  973 + " "+get_type("start",type_table)+")\n");
  974 + print(s," "+parser_name+"\n");
  975 + print(s," (\n");
  976 + if mb_extra is
  977 + {
  978 + failure then unique,
  979 + success(e) then if e is extra(t,n) then
  980 + print(s," "+t+" "+n+",\n")
  981 + };
  982 + print(s," One -> Token_Value_"+parser_name+" read_token\n");
  983 + print(s," ) =\n");
  984 + print(s," with unput_list = var((List(Token_Value_"+parser_name+"))[]),\n");
  985 + print(s," input = lexer((One u) |-> if *unput_list is\n");
  986 + print(s," {\n");
  987 + print(s," [ ] then read_token(unique),\n");
  988 + print(s," [h . t] then \n");
  989 + print(s," unput_list <- t;\n");
  990 + print(s," h\n");
  991 + print(s," },\n");
  992 + print(s," (Token_Value_"+parser_name+" t) |->\n");
  993 + print(s," unput_list <- [t . *unput_list]),\n");
  994 + if mb_extra is
  995 + {
  996 + failure then print(s," if state_0(input) is\n")
  997 + success(e) then if e is extra(t,n) then
  998 + print(s," if state_0("+n+",input) is\n")
  999 + };
  1000 + print(s," {\n");
  1001 + print(s," error(a,b) then error((a,b)),\n");
  1002 + print(s," end_ret(r) then if r is start(value)\n");
  1003 + print(s," then ok(value)\n");
  1004 + print(s," else should_not_happen(error((eof(unique),[]))),\n");
  1005 + print(s," do_ret(_) then should_not_happen(error((eof(unique),[])))\n");
  1006 + print(s," }.").
  1007 +
  1008 +
  1009 +define One
  1010 + print_parser_function_declaration
  1011 + (
  1012 + Stream s,
  1013 + String parser_name,
  1014 + List(TypeEntry) type_table,
  1015 + Maybe(Extra) mb_extra
  1016 + ) =
  1017 + print(s,"\n\n public define Result((Token_Value_"+parser_name+",List(Token_"+parser_name+")),\n"+
  1018 + " "+get_type("start",type_table)+")\n");
  1019 + print(s," "+parser_name+"\n");
  1020 + print(s," (\n");
  1021 + if mb_extra is
  1022 + {
  1023 + failure then unique,
  1024 + success(e) then if e is extra(t,n) then
  1025 + print(s," "+t+" "+n+",\n")
  1026 + };
  1027 + print(s," One -> Token_Value_"+parser_name+" read_token\n");
  1028 + print(s," ).\n").
  1029 +
  1030 +
  1031 +
  1032 + *** [3.2] Outputing the declaration of the function 'vmsg'.
  1033 +
  1034 + When the 'trace' option is used, APG outputs terms of the form 'vmsg("..."). These
  1035 + terms send messages which allow to follow the behavior of the parser. This is used only
  1036 + for debugging purpose. The function 'vmsg' must be provided by the user. The next
  1037 + function outputs the declaration of 'vmsg'.
  1038 +
  1039 +define One
  1040 + print_vmsg_declaration
  1041 + (
  1042 + Stream s
  1043 + ) =
  1044 + print(s,
  1045 + "\n\n Declaration of 'vmsg'. This function must be provided by the user of APG. ");
  1046 + print(s,"\n\ndefine One vmsg(String text).").
  1047 +
  1048 +
  1049 +
  1050 +
  1051 +
  1052 + *** [3.3] Outputing the 'reduce_n' functions.
  1053 +
  1054 + Reductions are performed by the 'reduce_n' functions (one per grammar rule).
  1055 +
  1056 +define String
  1057 + put_do_ret
  1058 + (
  1059 + String s,
  1060 + Int n
  1061 + ) =
  1062 + if n =< 0 then "\n end_ret("+s+")\n " else "do_ret("+put_do_ret(s,n-1)+")".
  1063 +
  1064 +
  1065 +define One
  1066 + print_reduce_function_args
  1067 + (
  1068 + Stream s,
  1069 + List(APG_Symbol_Value) rbody, // body of rule in reverse order
  1070 + List(TypeEntry) type_table
  1071 + ) =
  1072 + if rbody is
  1073 + {
  1074 + [ ] then unique,
  1075 + [h . t] then
  1076 + if h is symbol_value(sym,val) then
  1077 + print(s," "+get_type(sym,type_table)+" "+val+
  1078 + (if t is [] then "" else ",")+"\n");
  1079 + print_reduce_function_args(s,t,type_table)
  1080 + }.
  1081 +
  1082 +
  1083 +define One
  1084 + print_reduce_functions
  1085 + (
  1086 + Stream s,
  1087 + List(APG_Grammar_Rule) rules,
  1088 + List(TypeEntry) type_table,
  1089 + String parser_name,
  1090 + Maybe(Extra) mb_extra,
  1091 + List(APG_Option) options
  1092 + ) =
  1093 + if rules is
  1094 + {
  1095 + [ ] then unique,
  1096 + [rule1 . other_rules] then
  1097 + if rule1 is grammar_rule(id,head,body,prec) then
  1098 + print(s,"\n\ndefine Ret(Non_Terminal_Value)\n");
  1099 + print(s," reduce_"+id+"\n");
  1100 + print(s," (\n");
  1101 + if mb_extra is
  1102 + {
  1103 + failure then unique,
  1104 + success(e) then if e is extra(t,n) then
  1105 + print(s," "+t+" "+n+",\n")
  1106 + };
  1107 + if head is symbol_value(name,val) then
  1108 + if body is
  1109 + {
  1110 + [ ] then
  1111 + print(s," ) =\n");
  1112 + (if trace:options
  1113 + then print(s," vmsg(\"Reducing using rule "+to_decimal(id)+"\");\n")
  1114 + else unique);
  1115 + print(s," end_ret("+name+"("+val+")).\n"),
  1116 +
  1117 + [_ . _] then
  1118 + print_reduce_function_args(s,reverse(body),type_table);
  1119 + print(s," ) =\n");
  1120 + (if trace:options
  1121 + then print(s," vmsg(\"Reducing using rule "+to_decimal(id)+"\");\n")
  1122 + else unique);
  1123 + print(s," "+put_do_ret(name+"("+val+")",
  1124 + length(body))+".\n")
  1125 + };
  1126 + print_reduce_functions(s,other_rules,type_table,parser_name,mb_extra,options)
  1127 + }.
  1128 +
  1129 +
  1130 +
  1131 +
  1132 +
  1133 + *** [3.5] Outputing states.
  1134 +
  1135 +
  1136 +
  1137 + *** [3.5.1] Printing the state header.
  1138 +
  1139 +define One
  1140 + print_state_header
  1141 + (
  1142 + Stream s,
  1143 + Word32 state_id
  1144 + ) =
  1145 + print(s,"\n\n === State "+state_id+" ============================\n\n").
  1146 +
  1147 +
  1148 +
  1149 +
  1150 + *** [3.5.2] Printing scenarios.
  1151 +
  1152 + Printing a single scenario.
  1153 +
  1154 +define One
  1155 + print
  1156 + (
  1157 + Stream s,
  1158 + APG_Scenario sc
  1159 + ) =
  1160 + if sc is scenario(rid,head,bd,ad,prop,hg,lh_v) then
  1161 + print(s,right_pad(" ("+rid+")",10)+" ");
  1162 + print(s,right_pad(head+":",15)+" ");
  1163 + map_forget((String x) |-> print(s,x+" "),reverse(bd));
  1164 + print(s,". ");
  1165 + map_forget((String x) |-> print(s,x+" "),ad);
  1166 + print(s,"\n").
  1167 +
  1168 +
  1169 +
  1170 +
  1171 +
  1172 + Printing a list of classes of scenarios.
  1173 +
  1174 +define One
  1175 + print_scenarios
  1176 + (
  1177 + Stream s,
  1178 + List(APG_Class) classes
  1179 + ) =
  1180 + map_forget((APG_Class c) |-> if c is class(_,scs) then
  1181 + map_forget((APG_Scenario sc) |-> print(s,sc),scs),
  1182 + classes).
  1183 +
  1184 +
  1185 +
  1186 +
  1187 +
  1188 +
  1189 + *** [3.5.3] Printing the transitions.
  1190 +
  1191 +define One
  1192 + print_transitions
  1193 + (
  1194 + Stream s,
  1195 + List(APG_Transition) transitions,
  1196 + List(String) all_tokens
  1197 + ) =
  1198 + print(s,"\n --- Transitions/Reductions ---");
  1199 + map_forget((APG_Transition tr) |->
  1200 + if tr is transition(sym,id) then
  1201 + if sym:all_tokens
  1202 + then print(s,"\n "+right_pad(sym,20)+" shift and go to state "+id)
  1203 + else print(s,"\n "+right_pad(sym,20)+" restart from state "+id),
  1204 + transitions).
  1205 +
  1206 +
  1207 +
  1208 +
  1209 +
  1210 + *** [3.5.4] Printing the reductions.
  1211 +
  1212 +define One
  1213 + print_reductions
  1214 + (
  1215 + Stream s,
  1216 + List(APG_Scenario) scs
  1217 + ) =
  1218 + if scs is
  1219 + {
  1220 + [ ] then unique,
  1221 + [sc1 . others] then
  1222 + if sc1 is scenario(rid,head,bd,ad,prop,hg,lh_v) then
  1223 + if ad is
  1224 + {
  1225 + [ ] then
  1226 + map_forget((String lh) |->
  1227 + print(s,"\n "+right_pad(lh,20)+" reduce using rule "+rid),
  1228 + *lh_v),
  1229 + [_ . _] then print_reductions(s,others)
  1230 + }
  1231 + }.
  1232 +
  1233 +
  1234 +
  1235 +
  1236 + *** [3.5.5] Printing acceptable tokens.
  1237 +
  1238 +define One
  1239 + print_acceptable_tokens
  1240 + (
  1241 + Stream s,
  1242 + List(String) tokens // with possible repetitions
  1243 + ) =
  1244 + if tokens is
  1245 + {
  1246 + [ ] then unique,
  1247 + [h . t] then
  1248 + if h:t
  1249 + then print_acceptable_tokens(s,t)
  1250 + else print(s,"\n "+h+(if t is [] then "" else ","));
  1251 + print_acceptable_tokens(s,t)
  1252 + }.
  1253 +
  1254 +
  1255 +define One
  1256 + print_acceptable_tokens
  1257 + (
  1258 + Stream s,
  1259 + Word32 state_id,
  1260 + String parser_name,
  1261 + List(BehaviorEntry) behaviors,
  1262 + List(String) non_terminals
  1263 + ) =
  1264 + print(s,"\n\ndefine List(Token_"+parser_name+") token_list_"+state_id+" =");
  1265 + print(s,"\n [");
  1266 + print_acceptable_tokens(s,map(symbol,behaviors) - non_terminals);
  1267 + print(s,"\n ].").
  1268 +
  1269 +
  1270 +
  1271 +
  1272 + *** [3.5.6] Printing the restart function
  1273 +
  1274 +
  1275 +define Maybe(Word32)
  1276 + find_transition
  1277 + (
  1278 + String symbol,
  1279 + List(APG_Transition) transitions
  1280 + ) =
  1281 + if transitions is
  1282 + {
  1283 + [ ] then failure,
  1284 + [tr1 . others] then if tr1 is transition(sym,target_id) then
  1285 + if symbol = sym
  1286 + then success(target_id)
  1287 + else find_transition(symbol,others)
  1288 + }.
  1289 +
  1290 +
  1291 +
  1292 +define String
  1293 + format_restart_case_args
  1294 + (
  1295 + Int n
  1296 + ) =
  1297 + if n < 0 then "" else
  1298 + format_restart_case_args(n-1)+(if n = 0 then "" else ",")+"_"+abs_to_decimal(n).
  1299 +
  1300 +
  1301 +define List(String)
  1302 + get_longuest_stack_for
  1303 + (
  1304 + String tok_name,
  1305 + List(APG_Scenario) scs
  1306 + ) =
  1307 + if scs is
  1308 + {
  1309 + [ ] then [ ],
  1310 + [h . t] then
  1311 + with rest = get_longuest_stack_for(tok_name,t),
  1312 + if h is scenario(id,head,bd,ad,prop,hg,lh_v) then
  1313 + if ad is
  1314 + {
  1315 + [ ] then rest,
  1316 + [sym . _] then
  1317 + if sym = tok_name
  1318 + then if length(bd) > length(rest)
  1319 + then bd
  1320 + else rest
  1321 + else rest
  1322 + }
  1323 + }.
  1324 +
  1325 +
  1326 +
  1327 +
  1328 +define One
  1329 + print_restart_args
  1330 + (
  1331 + Stream s,
  1332 + List(String) stack,
  1333 + List(TypeEntry) type_table,
  1334 + Word32 i
  1335 + ) =
  1336 + if stack is
  1337 + {
  1338 + [ ] then unique,
  1339 + [h . t] then
  1340 + print(s,"\n "+right_pad(get_type(h,type_table),20)+right_pad(" _"+i,4)+
  1341 + (if t is [] then " " else ",")+" // "+h);
  1342 + print_restart_args(s,t,type_table,i+1)
  1343 + }.
  1344 +
  1345 +
  1346 +define One
  1347 + print_restart_cases
  1348 + (
  1349 + Stream s,
  1350 + Word32 state_id,
  1351 + List(APG_Scenario) scs,
  1352 + List(String) non_terminals,
  1353 + List(APG_Transition) transitions,
  1354 + Int num_args,
  1355 + List(APG_Option) options,
  1356 + Maybe(Extra) mb_extra
  1357 + ) =
  1358 + if non_terminals is
  1359 + {
  1360 + [ ] then unique,
  1361 + [h . t] then
  1362 + print(s,"\n "+h+"(value)"+" then");
  1363 + (if trace:options
  1364 + then print(s,"\n vmsg(\"Got a '"+h+"'\");")
  1365 + else unique);
  1366 + if find_transition(h,transitions) is
  1367 + {
  1368 + failure then if (state_id = 0 & h = "start")
  1369 + then print(s,"\n end_ret(start(value))")
  1370 + else print(s,"\n should_not_happen(error(eof(unique),[]))"),
  1371 +
  1372 + success(target_id) then
  1373 + with n = length(get_longuest_stack_for(h,scs)),
  1374 + print(s,"\n if state_"+target_id+"("+
  1375 + if mb_extra is
  1376 + {
  1377 + failure then "",
  1378 + success(e) then if e is extra(_,n1) then n1+","
  1379 + } +"input,value"+
  1380 + (if n = 0 then "" else ",")+
  1381 + format_restart_case_args(n-1)+") is");
  1382 + print(s,"\n {");
  1383 + print(s,"\n error(a,b) then error(a,b),");
  1384 + print(s,"\n end_ret(v) then /* 1 */ restart_"+state_id+"("+
  1385 + if mb_extra is
  1386 + {
  1387 + failure then "",
  1388 + success(e) then if e is extra(_,n1) then n1+","
  1389 + }+"input,v"+
  1390 + (if num_args = 0 then "" else ",")+
  1391 + format_restart_case_args(num_args-1)+"),");
  1392 + print(s,"\n do_ret(v) then v");
  1393 + print(s,"\n }")
  1394 + };
  1395 + print(s,if t is [] then "" else ",");
  1396 + print_restart_cases(s,state_id,scs,t,transitions,num_args,options,mb_extra)
  1397 + }.
  1398 +
  1399 +
  1400 +
  1401 +define One
  1402 + print_restart_function
  1403 + (
  1404 + Stream s,
  1405 + Word32 state_id,
  1406 + String parser_name,
  1407 + List(String) stack,
  1408 + List(APG_Scenario) scs,
  1409 + List(TypeEntry) type_table,
  1410 + List(String) non_terminals,
  1411 + List(APG_Transition) transitions,
  1412 + List(APG_Option) options,
  1413 + Maybe(Extra) mb_extra
  1414 + ) =
  1415 + print(s,"\n\ndefine Ret(Non_Terminal_Value)");
  1416 + print(s,"\n restart_"+state_id);
  1417 + print(s,"\n (");
  1418 + if mb_extra is
  1419 + {
  1420 + failure then unique,
  1421 + success(e) then if e is extra(t,n) then
  1422 + print(s,"\n "+t+" "+n+",")
  1423 + };
  1424 + print(s,"\n Lexer input,");
  1425 + print(s,"\n Non_Terminal_Value result"+(if stack is [] then "" else ","));
  1426 + print_restart_args(s,stack,type_table,0);
  1427 + print(s,"\n ) =");
  1428 + (if trace:options
  1429 + then print(s,"\n vmsg(\"Restarting\");")
  1430 + else unique);
  1431 + print(s,"\n if result is");
  1432 + print(s,"\n {");
  1433 + print_restart_cases(s,state_id,scs,non_terminals,transitions,length(stack),options,mb_extra);
  1434 + print(s,"\n }.").
  1435 +
  1436 +
  1437 +
  1438 +
  1439 +
  1440 +
  1441 + *** [3.5.7] Printing the state function.
  1442 +
  1443 +
  1444 + Printing the declarations of the arguments of the state function.
  1445 +
  1446 + There is one argument per symbol in the (top of) stack. Types of arguments are
  1447 + obtained from the type table, and names of arguments are of the form: "_0", "_1", "_2"
  1448 + etc...
  1449 +
  1450 +define One
  1451 + print_state_function_args
  1452 + (
  1453 + Stream s,
  1454 + List(String) stack,
  1455 + List(TypeEntry) type_table,
  1456 + Word32 rank // of next argument to be printed
  1457 + ) =
  1458 + if stack is
  1459 + {
  1460 + [ ] then unique,
  1461 + [name . t] then
  1462 + print(s," "+right_pad(get_type(name,type_table),20)+" _"+
  1463 + right_pad(rank+(if t is [] then " " else ","),5)+" // "+name+"\n");
  1464 + print_state_function_args(s,t,type_table,rank+1)
  1465 + }.
  1466 +
  1467 +
  1468 +
  1469 + Printing the beginning of the state function (which is common to the declaration and
  1470 + the definition). The state function begins like this:
  1471 +
  1472 + define Ret(Non_Terminal_Value)
  1473 + state_n
  1474 + (
  1475 + Lexer input,
  1476 + Type0 _0,
  1477 + Type1 _1
  1478 + ...
  1479 + )
  1480 +
  1481 +
  1482 +define One
  1483 + print_state_function_beginning
  1484 + (
  1485 + Stream s,
  1486 + Word32 state_id,
  1487 + String parser_name,
  1488 + List(String) stack,
  1489 + List(TypeEntry) type_table,
  1490 + Maybe(Extra) mb_extra
  1491 + ) =
  1492 + print(s,"\n\ndefine Ret(Non_Terminal_Value)\n");
  1493 + print(s," state_"+state_id+"\n");
  1494 + print(s," (\n");
  1495 + if mb_extra is
  1496 + {
  1497 + failure then unique,
  1498 + success(e) then if e is extra(t,n) then
  1499 + print(s," "+t+" "+n+",\n")
  1500 + };
  1501 + print(s," Lexer input"+(if stack is [] then "" else ",")+"\n");
  1502 + print_state_function_args(s,stack,type_table,0);
  1503 + print(s," )").
  1504 +
  1505 +
  1506 +
  1507 + Printing the declaration of the state function. This amounts to print the beginning and
  1508 + a dot.
  1509 +
  1510 +define One
  1511 + print_state_function_declaration
  1512 + (
  1513 + Stream s,
  1514 + Word32 state_id,
  1515 + String parser_name,
  1516 + List(String) stack,
  1517 + List(TypeEntry) type_table,
  1518 + Maybe(Extra) mb_extra
  1519 + ) =
  1520 + print_state_function_beginning(s,state_id,parser_name,stack,type_table,mb_extra);
  1521 + print(s,".").
  1522 +
  1523 +
  1524 +
  1525 +
  1526 +
  1527 +
  1528 + Printing the definition of the state function. We have to print the beginning, an
  1529 + 'equal' sign and the body of the function. The body is a conditional with one case per
  1530 + token of the grammar. First of all, here is a function for printing the cases of this
  1531 + conditional.
  1532 +
  1533 +
  1534 +define Maybe(APG_Scenario)
  1535 + find_reduction
  1536 + (
  1537 + String token,
  1538 + List(APG_Scenario) scs
  1539 + ) =
  1540 + if scs is
  1541 + {
  1542 + [ ] then failure,
  1543 + [sc1 . others] then
  1544 + if sc1 is scenario(id,head,bd,ad,prop,hg,lh_v) then
  1545 + if token : *lh_v
  1546 + then if ad is
  1547 + {
  1548 + [ ] then success(sc1),
  1549 + [_ . _] then find_reduction(token,others)
  1550 + }
  1551 + else find_reduction(token,others)
  1552 + }.
  1553 +
  1554 +
  1555 +define One
  1556 + print_reduce_body
  1557 + (
  1558 + Stream s,
  1559 + String token,
  1560 + List(APG_Scenario) scs,
  1561 + Word32 state_id,
  1562 + List(APG_Option) options,
  1563 + Maybe(Extra) mb_extra
  1564 + ) =
  1565 + if find_reduction(token,scs) is
  1566 + {
  1567 + failure then (if trace:options
  1568 + then print(s," vmsg(\"Unexpected token '"+token+"'\");\n")
  1569 + else unique);
  1570 + print(s," error(next,token_list_"+state_id+")"),
  1571 +
  1572 + success(sc) then if sc is scenario(rid,_,bd,_,_,_,_) then
  1573 + print(s," unput_token(input)(next);\n");
  1574 + with nargs = length(bd),
  1575 + (if nargs = 0
  1576 + then print(s," if reduce_"+rid+
  1577 + if mb_extra is
  1578 + {
  1579 + failure then "",
  1580 + success(e) then if e is extra(t,n) then "("+n+")"
  1581 + }+" is\n")
  1582 + else print(s," if reduce_"+rid+"("+
  1583 + if mb_extra is
  1584 + {
  1585 + failure then "",
  1586 + success(e) then if e is extra(t,n) then n+","
  1587 + }+format_restart_case_args(nargs-1)+") is\n"));
  1588 + print(s," {\n");
  1589 + print(s," error(a,b) then error(a,b),\n");
  1590 + (if nargs = 0
  1591 + then print(s," end_ret(v) then /* 2 */ restart_"+state_id+"("+
  1592 + if mb_extra is
  1593 + {
  1594 + failure then "",
  1595 + success(e) then if e is extra(t,n) then n+","
  1596 + }+"input,v"+
  1597 + (with n = length(get_longuest_stack(scs)), (if n = 0 then "" else ",")+
  1598 + format_restart_case_args(n-1))+"),\n")
  1599 + else print(s," end_ret(v) then end_ret(v),\n"));
  1600 + print(s," do_ret(v) then v\n");
  1601 + print(s," }")
  1602 + }.
  1603 +
  1604 +
  1605 +
  1606 +
  1607 +
  1608 +
  1609 +
  1610 +define One
  1611 + print_state_function_cases
  1612 + (
  1613 + Stream s,
  1614 + List(String) all_tokens,
  1615 + List(APG_Scenario) scs,
  1616 + List(APG_Transition) transitions,
  1617 + Word32 state_id,
  1618 + List(String) stack,
  1619 + List(Conflict) conflicts,
  1620 + List(APG_Option) options,
  1621 + Bool has_restarts,
  1622 + Maybe(Extra) mb_extra
  1623 + ) =
  1624 + if all_tokens is
  1625 + {
  1626 + [ ] then unique,
  1627 + [tok1 . others] then
  1628 + print(s," "+tok1+"(value)"+" then\n");
  1629 + with resol = get_conflict_resolution(tok1,conflicts),
  1630 + (if resol is reduce1(Word32 tl, Word32 rl) then
  1631 + print_reduce_body(s,tok1,scs,state_id,options,mb_extra) else
  1632 + if resol is reduce2(Word32 l, AssocMode m) then
  1633 + print_reduce_body(s,tok1,scs,state_id,options,mb_extra) else
  1634 + if find_transition(tok1,transitions) is
  1635 + {
  1636 + failure then print_reduce_body(s,tok1,scs,state_id,options,mb_extra),
  1637 + success(target_state_id) then
  1638 + (if trace:options
  1639 + then print(s," vmsg(\"Shifting token '"+tok1+"'\");\n")
  1640 + else unique);
  1641 + with n = length(get_longuest_stack_for(tok1,scs)),
  1642 + print(s," if state_"+target_state_id+"("+
  1643 + if mb_extra is
  1644 + {
  1645 + failure then "",
  1646 + success(e) then if e is extra(_,n1) then n1+","
  1647 + }+"input,value"+
  1648 + (if n = 0 then "" else ",")+
  1649 + format_restart_case_args(n-1)+") is\n");
  1650 + print(s," {\n");
  1651 + print(s," error(a,b) then error(a,b),\n");
  1652 + with m = length(stack),
  1653 + (if has_restarts
  1654 + then print(s," end_ret(value1) then /* 3 */ restart_"+state_id+
  1655 + "("+if mb_extra is
  1656 + {
  1657 + failure then "",
  1658 + success(e) then if e is extra(_,n1) then n1+","
  1659 + }+"input,value1"+
  1660 + (if m = 0 then "" else ",")+
  1661 + format_restart_case_args(m-1)+"),\n")
  1662 + else print(s," end_ret(value1) then should_not_happen(error(eof(unique),[])),\n"));
  1663 + print(s," do_ret(value1) then ");
  1664 + (if trace:options
  1665 + then print(s,"vmsg(\"Ignoring state "+state_id+
  1666 + "\");\n ")
  1667 + else unique);
  1668 + print(s,"value1\n");
  1669 + print(s," }")
  1670 + });
  1671 + print(s,if others is [] then "\n" else ",\n");
  1672 + print_state_function_cases(s,others,scs,transitions,state_id,
  1673 + stack,conflicts,options,has_restarts,mb_extra)
  1674 + }.
  1675 +
  1676 +
  1677 +
  1678 +define One
  1679 + print_state_function
  1680 + (
  1681 + Stream s,
  1682 + Word32 state_id,
  1683 + String parser_name,
  1684 + List(String) stack,
  1685 + List(TypeEntry) type_table,
  1686 + List(String) all_tokens,
  1687 + List(APG_Transition) transitions,
  1688 + List(APG_Scenario) scs,
  1689 + List(Conflict) cfls,
  1690 + List(APG_Option) options,
  1691 + Bool has_restarts,
  1692 + Maybe(Extra) mb_extra
  1693 + ) =
  1694 + print_state_function_beginning(s,state_id,parser_name,stack,type_table,mb_extra);
  1695 + print(s," =\n");
  1696 + (if trace:options
  1697 + then print(s," vmsg(\"Entering state "+state_id+"\");\n")
  1698 + else unique);
  1699 + print(s," with next = read_token(input)(unique),\n");
  1700 + print(s," if next is\n");
  1701 + print(s," {\n");
  1702 + print_state_function_cases(s,all_tokens,scs,transitions,state_id,stack,cfls,options,
  1703 + has_restarts,mb_extra);
  1704 + print(s," }.\n").
  1705 +
  1706 +
  1707 +
  1708 +
  1709 +
  1710 +
  1711 + *** [3.5.8] Printing a whole state.
  1712 +
  1713 +define One
  1714 + print_state
  1715 + (
  1716 + Stream s,
  1717 + APG_State st,
  1718 + List(String) all_tokens,
  1719 + List(PrecEntry) prec_table,
  1720 + List(AssocEntry) assoc_table,
  1721 + List(TypeEntry) type_table,
  1722 + List(String) non_terminals,
  1723 + List(APG_Grammar_Rule) rules,
  1724 + String parser_name,
  1725 + List(APG_Option) options,
  1726 + Var(Word32) count_shift_reduce_conflicts,
  1727 + Var(Word32) count_reduce_reduce_conflicts,
  1728 + Maybe(Extra) mb_extra
  1729 + ) =
  1730 + if st is state(st_id,classes,transitions) then
  1731 + with
  1732 + scs = flat_classes(classes),
  1733 + stack = get_longuest_stack(scs),
  1734 + behaviors = compute_behaviors(scs,transitions,all_tokens),
  1735 + conflicts = compute_conflicts(behaviors,rules,prec_table,assoc_table),
  1736 + has_rest = has_restarts(behaviors),
  1737 +
  1738 + if number_of_conflicts(conflicts,0,0) is (sr,rr) then
  1739 + (
  1740 + count_shift_reduce_conflicts <- *count_shift_reduce_conflicts + sr;
  1741 + count_reduce_reduce_conflicts <- *count_reduce_reduce_conflicts + rr
  1742 + );
  1743 +
  1744 + print_state_header(s,st_id);
  1745 + print_scenarios(s,classes);
  1746 + print_transitions(s,transitions,all_tokens);
  1747 + print_reductions(s,scs);
  1748 + print_conflicts(s,conflicts);
  1749 + //print_conflicts(make_stream(stdout),conflicts);
  1750 + print_acceptable_tokens(s,st_id,parser_name,behaviors,non_terminals);
  1751 + (if has_rest
  1752 + then print_restart_function(s,st_id,parser_name,stack,scs,
  1753 + type_table,non_terminals,transitions,options,mb_extra)
  1754 + else unique);
  1755 + print_state_function(s,st_id,parser_name,stack,type_table,
  1756 + all_tokens,transitions,scs,conflicts,options,has_rest,mb_extra).
  1757 +
  1758 +
  1759 +
  1760 +
  1761 +
  1762 + *** [4] The interface.
  1763 +
  1764 +read make_automaton.anubis
  1765 +
  1766 +
  1767 +define One
  1768 + do_output
  1769 + (
  1770 + APG_Grammar g,
  1771 + List(APG_State) auto,
  1772 + Stream s,
  1773 + List(APG_Option) options
  1774 + ) =
  1775 + if g is grammar(preambule,parser_name,prec_decs,type_decs,rules,mb_extra,postambule) then
  1776 + with non_terminals = (List(String))["start" . all_non_terminals(rules)],
  1777 + all_tokens = (List(String))["eof" . all_symbols(rules) - non_terminals],
  1778 + unused_tokens = no_doubles((merge(tokens_from_type_decs(type_decs),
  1779 + tokens_from_precs(prec_decs)) - all_tokens) - non_terminals),
  1780 + used_and_unused_tokens = qsort(map(used,all_tokens)+map(unused,unused_tokens),alphaless),
  1781 + prec_table = make_precedence_table(prec_decs),
  1782 + assoc_table = make_association_table(prec_decs),
  1783 + _A = if get_rule(1,rules) is grammar_rule(_,_A,_,_) then _A,
  1784 + type_table = make_type_table(type_decs,name(_A)),
  1785 + sr_count_v = var((Word32)0),
  1786 + rr_count_v = var((Word32)0),
  1787 +
  1788 + (if verbose:options
  1789 + then print("\nOutputting the result ... ")
  1790 + else unique);
  1791 + print(s,"\n\n This file was generated by APG (the Anubis Parser Generator)\n");
  1792 + print(s," (to find unresolved shift/reduce conflicts, search for 'unresolved')\n");
  1793 + print(s," (to find reduce/reduce conflicts, search for 'reduce/reduce')\n\n");
  1794 + print(s," Your parser is declared as follows below in this file:");
  1795 + print_parser_function_declaration(s,parser_name,type_table,mb_extra);
  1796 + print(s,"\n The public types 'Token_"+parser_name+"' and 'Token_Value_"+parser_name+"'\n");
  1797 + print(s," are defined below in this file.\n\n");
  1798 + print(s,preambule);
  1799 + print_token_type(s,parser_name,used_and_unused_tokens);
  1800 + print_token_value_type(s,parser_name,used_and_unused_tokens,type_table);
  1801 + print_non_terminals_type(s,non_terminals,type_table);
  1802 + print_type_Ret(s,parser_name);
  1803 + print_Lexer_type(s,parser_name);
  1804 +
  1805 + (if trace:options then print_vmsg_declaration(s) else unique);
  1806 +
  1807 + print_reduce_functions(s,
  1808 + [grammar_rule(0,symbol_value("start","_0"),
  1809 + [symbol_value(name(_A),"_0")],failure) . rules],
  1810 + type_table,parser_name,mb_extra,options);
  1811 +
  1812 + map_forget((APG_State st) |->
  1813 + if st is state(state_id,classes,transitions) then
  1814 + with scs = flat_classes(classes),
  1815 + print_state_function_declaration(s,
  1816 + state_id,
  1817 + parser_name,
  1818 + get_longuest_stack(scs),
  1819 + type_table,
  1820 + mb_extra),
  1821 + auto);
  1822 +
  1823 + print_parser_function(s,parser_name,type_table,mb_extra);
  1824 +
  1825 + map_forget((APG_State state) |->
  1826 + print_state(s,
  1827 + state,
  1828 + map(name,used_and_unused_tokens), // all_tokens,
  1829 + prec_table,
  1830 + assoc_table,
  1831 + type_table,
  1832 + non_terminals,
  1833 + rules,
  1834 + parser_name,
  1835 + options,
  1836 + sr_count_v,
  1837 + rr_count_v,
  1838 + mb_extra),
  1839 + auto);
  1840 +
  1841 + print(s,postambule);
  1842 +
  1843 + (if verbose:options
  1844 + then print("Done. \n")
  1845 + else unique);
  1846 +
  1847 + (if *sr_count_v /= 0
  1848 + then print("\nThere are "+(*sr_count_v)+" shift/reduce conflicts.")
  1849 + else unique);
  1850 +
  1851 + (if *rr_count_v /= 0
  1852 + then print("\nThere are "+(*rr_count_v)+" reduce/reduce conflicts.")
  1853 + else unique);
  1854 +
  1855 + (if (*rr_count_v + *sr_count_v) /= 0
  1856 + then print("\n\n")
  1857 + else unique).
  1858 +
  1859 +public define One
  1860 + anubis_output
  1861 + (
  1862 + APG_Grammar g,
  1863 + String output_filename,
  1864 + List(APG_Option) options
  1865 + ) =
  1866 + with auto = make_APG_automaton(g,options),
  1867 + if file(output_filename,new) is
  1868 + {
  1869 + failure then print("Cannot create file '"+output_filename+"'.\n"),
  1870 + success(f) then with s = make_stream(f),
  1871 + if time:options
  1872 + then show_time("do_output: ",(One u) |-> do_output(g,auto,s,options))
  1873 + else do_output(g,auto,s,options)
  1874 + }.
  1875 +
  1876 +
  1877 +
  1878 +
  1879 + *** [5] Testing.
  1880 +
  1881 +read read_grammar.anubis
  1882 +
  1883 + global define One
  1884 + apg_test
  1885 + (
  1886 + List(String) args
  1887 + ) =
  1888 + if read_APG_grammar(make_stream(example_grammar),[]) is
  1889 + {
  1890 + error(msg) then en_print(msg),
  1891 + ok(g) then anubis_output(g,make_stream(stdout),
  1892 + [
  1893 + //trace
  1894 + ])
  1895 + }.
  1896 +
  1897 +
  1898 +
  1899 +
... ...