Anubis / Anubis

Commit 92c7ed18c2ee6e10f09865d2d577af4a0fcf26a9

Authored by Cédric RICARD
1 parent bde59c8c

Fix LineReader bug (infinite loop) 

Add UnitTest to FastLexer revealing bug
Showing 9 changed files with 347 additions and 225 deletions   Show diff stats
Inline Side-by-side
anubis_dev/library/lexical_analysis/fast_lexer.anubis
  Show/Hide comments   View file @92c7ed1
@@ -447,24 +447,24 @@ public define Maybe(LexingStream) make_lexing_stream(SSL_Connection stream, @@ -447,24 +447,24 @@ public define Maybe(LexingStream) make_lexing_stream(SSL_Connection stream,
447 file, in particular at the actual definition of type 'LexingStream', and write down 447 file, in particular at the actual definition of type 'LexingStream', and write down
448 another such function (in a file of yours within which you put a 'read 448 another such function (in a file of yours within which you put a 'read
449 fast_lexer.anubis'). 449 fast_lexer.anubis').
450 -  
451 -  
452 - To each lexing stream is attached a function for counting of type 'One -> Int'. When  
453 - applied to 'unique', this function returns the number of bytes already read from the  
454 - lexing stream, i.e. the position from which the reading of te next token will  
455 - occur. This function is obtained as follows: 450 +
  451 +
  452 + To each lexing stream is attached a function for counting of type 'One -> Int'. When
  453 + applied to 'unique', this function returns the number of bytes already read from the
  454 + lexing stream, i.e. the position from which the reading of te next token will
  455 + occur. This function is obtained as follows:
456 456
457 -public define One -> Int  
458 - offset_counter  
459 - (  
460 - LexingStream ls  
461 - ). 457 +public define One -> Int
  458 + offset_counter
  459 + (
  460 + LexingStream ls
  461 + ).
  462 +
  463 + If youn need this function, it is recommended to call 'offset_counter' only once just
  464 + after the lexing stream is created.
462 465
463 - If youn need this function, it is recommended to call 'offset_counter' only once just  
464 - after the lexing stream is created.  
465 466
466 467
467 -  
468 *** (4) Constructing a lexer. 468 *** (4) Constructing a lexer.
469 469
470 *** (4.1) Construction. 470 *** (4.1) Construction.
@@ -580,7 +580,7 @@ public define Printable_tree @@ -580,7 +580,7 @@ public define Printable_tree
580 580
581 read tools/basis.anubis 581 read tools/basis.anubis
582 read tools/streams.anubis 582 read tools/streams.anubis
583 - 583 +read system/convert.anubis
584 584
585 -------------------------------- Table of Contents ------------------------------------ 585 -------------------------------- Table of Contents ------------------------------------
586 586
@@ -740,13 +740,7 @@ define Maybe(ExChar) @@ -740,13 +740,7 @@ define Maybe(ExChar)
740 740
741 *** [1.5.1] Truncating a Word32 to a Word8. 741 *** [1.5.1] Truncating a Word32 to a Word8.
742 742
743 -define Word8  
744 - truncate_to_Word8  
745 - (  
746 - Word32 x  
747 - ) =  
748 - if x is word32(l1,_) then if l1 is word16(l2,_) then l2.  
749 - 743 + moved to system/convert.anubis
750 744
751 745
752 *** [1.5.2] Creating a range of consecutive characters. 746 *** [1.5.2] Creating a range of consecutive characters.
@@ -1223,24 +1217,24 @@ public define String @@ -1223,24 +1217,24 @@ public define String
1223 public type LexingStream: 1217 public type LexingStream:
1224 lexing_stream 1218 lexing_stream
1225 ( 1219 (
1226 - Var(ByteArray) buffer_v, // the current buffer  
1227 - Var(Int) start_v, // start of lexem in buffer  
1228 - Var(FastLexerLastAccepted) last_accept_v, // last accepting position (if any)  
1229 - Var(Int) current_v, // position of reading in buffer  
1230 - Int -> Maybe(One) reload_buffer, // command for loading the sequel in the buffer  
1231 - Var(Int) current_offset_v // accumulator for current offset 1220 + Var(ByteArray) buffer_v, // the current buffer
  1221 + Var(Int) start_v, // start of lexem in buffer
  1222 + Var(FastLexerLastAccepted) last_accept_v, // last accepting position (if any)
  1223 + Var(Int) current_v, // position of reading in buffer
  1224 + Int -> Maybe(One) reload_buffer, // command for loading the sequel in the buffer
  1225 + Var(Int) current_offset_v // accumulator for current offset
1232 ). 1226 ).
1233 1227
1234 -  
1235 -public define One -> Int  
1236 - offset_counter  
1237 - (  
1238 - LexingStream ls  
1239 - ) =  
1240 - if ls is lexing_stream(_,_,_,_,_,v) then  
1241 - (One u) |-> *v.  
1242 -  
1243 - 1228 +
  1229 +public define One -> Int
  1230 + offset_counter
  1231 + (
  1232 + LexingStream ls
  1233 + ) =
  1234 + if ls is lexing_stream(_,_,_,_,_,v) then
  1235 + (One u) |-> *v.
  1236 +
  1237 +
1244 While we are reading a lexeme, we keep the starting position (offset of first character 1238 While we are reading a lexeme, we keep the starting position (offset of first character
1245 of the current lexeme) in 'start_v' so as to be able to extract the lexeme. We also 1239 of the current lexeme) in 'start_v' so as to be able to extract the lexeme. We also
1246 keep the last position at which a lexeme was accepted. This is because the lexer always 1240 keep the last position at which a lexeme was accepted. This is because the lexer always
@@ -1272,8 +1266,8 @@ public define LexingStream @@ -1272,8 +1266,8 @@ public define LexingStream
1272 var(0), // starting position 1266 var(0), // starting position
1273 var(none), // last accepting position 1267 var(none), // last accepting position
1274 var(0), // current position 1268 var(0), // current position
1275 - (Int u) |-> failure, // buffer cannot be reloaded  
1276 - var(0)). 1269 + (Int u) |-> failure, // buffer cannot be reloaded
  1270 + var(0)).
1277 1271
1278 1272
1279 1273
@@ -1308,7 +1302,7 @@ public define Maybe(LexingStream) @@ -1308,7 +1302,7 @@ public define Maybe(LexingStream)
1308 start_v = var((Int)0), 1302 start_v = var((Int)0),
1309 last_accepted_v = var((FastLexerLastAccepted)none), 1303 last_accepted_v = var((FastLexerLastAccepted)none),
1310 current_v = var((Int)0), 1304 current_v = var((Int)0),
1311 - offset_v = var((Int)0), 1305 + offset_v = var((Int)0),
1312 reload_buffer = (Int i) |-> 1306 reload_buffer = (Int i) |->
1313 if read(stream,buffer_size,timeout) is 1307 if read(stream,buffer_size,timeout) is
1314 { 1308 {
@@ -1317,13 +1311,13 @@ public define Maybe(LexingStream) @@ -1317,13 +1311,13 @@ public define Maybe(LexingStream)
1317 ok(more) then 1311 ok(more) then
1318 //print("Buffer reloaded ("+abs_to_decimal(length(more))+" bytes).\n"); 1312 //print("Buffer reloaded ("+abs_to_decimal(length(more))+" bytes).\n");
1319 if length(more) = 0 1313 if length(more) = 0
1320 - then (with old_buffer = *buffer_v,  
1321 - old_length = length(old_buffer),  
1322 - dropped = *start_v, // number of bytes dropped from old buffer 1314 + then (with old_buffer = *buffer_v,
  1315 + old_length = length(old_buffer),
  1316 + dropped = *start_v, // number of bytes dropped from old buffer
1323 buffer_v <- extract(old_buffer,dropped,old_length); 1317 buffer_v <- extract(old_buffer,dropped,old_length);
1324 start_v <- 0; 1318 start_v <- 0;
1325 current_v <- *current_v - dropped; 1319 current_v <- *current_v - dropped;
1326 - /* reloading does not change the current offset */ 1320 + /* reloading does not change the current offset */
1327 last_accepted_v <- 1321 last_accepted_v <-
1328 if *last_accepted_v is 1322 if *last_accepted_v is
1329 { 1323 {
@@ -1337,7 +1331,7 @@ public define Maybe(LexingStream) @@ -1337,7 +1331,7 @@ public define Maybe(LexingStream)
1337 buffer_v <- extract(old_buffer,dropped,old_length)+more; 1331 buffer_v <- extract(old_buffer,dropped,old_length)+more;
1338 start_v <- 0; 1332 start_v <- 0;
1339 current_v <- *current_v - dropped; 1333 current_v <- *current_v - dropped;
1340 - /* reloading does not change the current offset */ 1334 + /* reloading does not change the current offset */
1341 last_accepted_v <- 1335 last_accepted_v <-
1342 if *last_accepted_v is 1336 if *last_accepted_v is
1343 { 1337 {
@@ -1350,8 +1344,8 @@ public define Maybe(LexingStream) @@ -1350,8 +1344,8 @@ public define Maybe(LexingStream)
1350 start_v, 1344 start_v,
1351 last_accepted_v, 1345 last_accepted_v,
1352 current_v, 1346 current_v,
1353 - reload_buffer,  
1354 - offset_v)) 1347 + reload_buffer,
  1348 + offset_v))
1355 }. 1349 }.
1356 1350
1357 1351
@@ -1390,7 +1384,7 @@ public define Maybe(LexingStream) @@ -1390,7 +1384,7 @@ public define Maybe(LexingStream)
1390 start_v = var((Int)0), 1384 start_v = var((Int)0),
1391 last_accepted_v = var((FastLexerLastAccepted)none), 1385 last_accepted_v = var((FastLexerLastAccepted)none),
1392 current_v = var((Int)0), 1386 current_v = var((Int)0),
1393 - offset_v = var((Int)0), 1387 + offset_v = var((Int)0),
1394 reload_buffer = (Int i) |-> 1388 reload_buffer = (Int i) |->
1395 if (Maybe(ByteArray))read(stream,buffer_size,timeout) is 1389 if (Maybe(ByteArray))read(stream,buffer_size,timeout) is
1396 { 1390 {
@@ -1403,7 +1397,7 @@ public define Maybe(LexingStream) @@ -1403,7 +1397,7 @@ public define Maybe(LexingStream)
1403 buffer_v <- extract(old_buffer,dropped,old_length); 1397 buffer_v <- extract(old_buffer,dropped,old_length);
1404 start_v <- 0; 1398 start_v <- 0;
1405 current_v <- *current_v - dropped; 1399 current_v <- *current_v - dropped;
1406 - /* reloading does not change the current offset */ 1400 + /* reloading does not change the current offset */
1407 last_accepted_v <- 1401 last_accepted_v <-
1408 if *last_accepted_v is 1402 if *last_accepted_v is
1409 { 1403 {
@@ -1417,7 +1411,7 @@ public define Maybe(LexingStream) @@ -1417,7 +1411,7 @@ public define Maybe(LexingStream)
1417 buffer_v <- extract(old_buffer,dropped,old_length)+more; 1411 buffer_v <- extract(old_buffer,dropped,old_length)+more;
1418 start_v <- 0; 1412 start_v <- 0;
1419 current_v <- *current_v - dropped; 1413 current_v <- *current_v - dropped;
1420 - /* reloading does not change the current offset */ 1414 + /* reloading does not change the current offset */
1421 last_accepted_v <- 1415 last_accepted_v <-
1422 if *last_accepted_v is 1416 if *last_accepted_v is
1423 { 1417 {
@@ -1430,8 +1424,8 @@ public define Maybe(LexingStream) @@ -1430,8 +1424,8 @@ public define Maybe(LexingStream)
1430 start_v, 1424 start_v,
1431 last_accepted_v, 1425 last_accepted_v,
1432 current_v, 1426 current_v,
1433 - reload_buffer,  
1434 - offset_v)) 1427 + reload_buffer,
  1428 + offset_v))
1435 }. 1429 }.
1436 1430
1437 1431
@@ -2289,9 +2283,9 @@ define One -&gt; LexerOutput($Token) @@ -2289,9 +2283,9 @@ define One -&gt; LexerOutput($Token)
2289 Word16 starting_state) -> FastLexerOutput lexer, 2283 Word16 starting_state) -> FastLexerOutput lexer,
2290 MVar(LexerAction($Token)) actions 2284 MVar(LexerAction($Token)) actions
2291 ) = 2285 ) =
2292 - if stream is lexing_stream(buffer_v,start_v,last_accept_v,current_v,reload_buffer,offset_v) then  
2293 - (One _) |-l-> with old_current = *current_v,  
2294 - if lexer(*buffer_v, 2286 + if stream is lexing_stream(buffer_v,start_v,last_accept_v,current_v,reload_buffer,offset_v) then
  2287 + (One _) |-l-> with old_current = *current_v,
  2288 + if lexer(*buffer_v,
2295 *last_accept_v, 2289 *last_accept_v,
2296 *current_v, 2290 *current_v,
2297 0) // reading a new token always starts in state 0 2291 0) // reading a new token always starts in state 0
@@ -2303,7 +2297,7 @@ define One -&gt; LexerOutput($Token) @@ -2303,7 +2297,7 @@ define One -&gt; LexerOutput($Token)
2303 not_at_end_of_input then 2297 not_at_end_of_input then
2304 with result = (LexerOutput($Token))error(extract(*buffer_v,*start_v,end)), 2298 with result = (LexerOutput($Token))error(extract(*buffer_v,*start_v,end)),
2305 current_v <- end+1; 2299 current_v <- end+1;
2306 - offset_v <- *offset_v + (end + 1 - old_current); 2300 + offset_v <- *offset_v + (end + 1 - old_current);
2307 start_v <- end+1; 2301 start_v <- end+1;
2308 last_accept_v <- none; 2302 last_accept_v <- none;
2309 result, 2303 result,
@@ -2326,7 +2320,7 @@ define One -&gt; LexerOutput($Token) @@ -2326,7 +2320,7 @@ define One -&gt; LexerOutput($Token)
2326 { 2320 {
2327 ignore then 2321 ignore then
2328 current_v <- end; 2322 current_v <- end;
2329 - offset_v <- *offset_v + (end - old_current); 2323 + offset_v <- *offset_v + (end - old_current);
2330 start_v <- end; 2324 start_v <- end;
2331 last_accept_v <- none; 2325 last_accept_v <- none;
2332 l(unique), // ignore and try to read the next token 2326 l(unique), // ignore and try to read the next token
@@ -2334,7 +2328,7 @@ define One -&gt; LexerOutput($Token) @@ -2334,7 +2328,7 @@ define One -&gt; LexerOutput($Token)
2334 return(f) then 2328 return(f) then
2335 with result = f(extract(*buffer_v,*start_v,end)), 2329 with result = f(extract(*buffer_v,*start_v,end)),
2336 current_v <- end; 2330 current_v <- end;
2337 - offset_v <- *offset_v + (end - old_current); 2331 + offset_v <- *offset_v + (end - old_current);
2338 start_v <- end; 2332 start_v <- end;
2339 last_accept_v <- none; 2333 last_accept_v <- none;
2340 result 2334 result
@@ -2351,7 +2345,7 @@ define One -&gt; LexerOutput($Token) @@ -2351,7 +2345,7 @@ define One -&gt; LexerOutput($Token)
2351 return(f) then 2345 return(f) then
2352 with result = f(extract(*buffer_v,*start_v,end)), 2346 with result = f(extract(*buffer_v,*start_v,end)),
2353 current_v <- end; 2347 current_v <- end;
2354 - offset_v <- *offset_v + (end - old_current); 2348 + offset_v <- *offset_v + (end - old_current);
2355 start_v <- end; 2349 start_v <- end;
2356 last_accept_v <- none; 2350 last_accept_v <- none;
2357 result 2351 result
anubis_dev/library/system/convert.anubis
  Show/Hide comments   View file @92c7ed1
@@ -92,18 +92,18 @@ public define inline ByteArray @@ -92,18 +92,18 @@ public define inline ByteArray
92 92
93 // Obsolete name, should be removed 93 // Obsolete name, should be removed
94 public define inline Word8 94 public define inline Word8
95 - truncate_to_word8  
96 - (  
97 - Word32 value  
98 - ) =  
99 - truncate_to_Word8(to_Int(value)). 95 + truncate_to_word8
  96 + (
  97 + Word32 x
  98 + ) =
  99 + if x is word32(l1,_) then if l1 is word16(l2,_) then l2.
100 100
101 public define inline Word8 101 public define inline Word8
102 - truncate_to_Word8  
103 - (  
104 - Word32 value  
105 - ) =  
106 - truncate_to_Word8(to_Int(value)). 102 + truncate_to_Word8
  103 + (
  104 + Word32 x
  105 + ) =
  106 + if x is word32(l1,_) then if l1 is word16(l2,_) then l2.
107 107
108 public define inline Word32 108 public define inline Word32
109 word8_to_Word32 109 word8_to_Word32
anubis_dev/library/test/Anubis UnitTest.aproj
  Show/Hide comments   View file @92c7ed1
@@ -32,6 +32,7 @@ @@ -32,6 +32,7 @@
32 <Import Project="$(AnubisBinPath)\Anubis.Build.targets" /> 32 <Import Project="$(AnubisBinPath)\Anubis.Build.targets" />
33 <ItemGroup> 33 <ItemGroup>
34 <Compile Include="all_unit_test.anubis" /> 34 <Compile Include="all_unit_test.anubis" />
  35 + <Compile Include="lexical_analysis\fast_lexer.ut.anubis" />
35 <Compile Include="predefined\date_and_time.unit_test.anubis" /> 36 <Compile Include="predefined\date_and_time.unit_test.anubis" />
36 <Compile Include="predefined\decimal_scan.unit_test.anubis" /> 37 <Compile Include="predefined\decimal_scan.unit_test.anubis" />
37 <Compile Include="predefined\fast_lexer.unit_test.anubis" /> 38 <Compile Include="predefined\fast_lexer.unit_test.anubis" />
@@ -48,6 +49,7 @@ @@ -48,6 +49,7 @@
48 <Compile Include="tools\utf-8.unit_test.anubis" /> 49 <Compile Include="tools\utf-8.unit_test.anubis" />
49 </ItemGroup> 50 </ItemGroup>
50 <ItemGroup> 51 <ItemGroup>
  52 + <Folder Include="lexical_analysis" />
51 <Folder Include="predefined" /> 53 <Folder Include="predefined" />
52 <Folder Include="system" /> 54 <Folder Include="system" />
53 <Folder Include="tools" /> 55 <Folder Include="tools" />
anubis_dev/library/test/all_unit_test.anubis
  Show/Hide comments   View file @92c7ed1
@@ -10,6 +10,7 @@ read test/predefined/sqlite.unit_test.anubis @@ -10,6 +10,7 @@ read test/predefined/sqlite.unit_test.anubis
10 read test/predefined/fast_lexer.unit_test.anubis 10 read test/predefined/fast_lexer.unit_test.anubis
11 11
12 read test/system/convert.unit_test.anubis 12 read test/system/convert.unit_test.anubis
  13 +read test/lexical_analysis/fast_lexer.ut.anubis
13 read test/system/message_queue.unit_test.anubis 14 read test/system/message_queue.unit_test.anubis
14 read test/system/message_transceiver.unit_test.anubis 15 read test/system/message_transceiver.unit_test.anubis
15 read test/system/string.unit_test.anubis 16 read test/system/string.unit_test.anubis
@@ -37,7 +38,8 @@ define List(UnitTestSuite) @@ -37,7 +38,8 @@ define List(UnitTestSuite)
37 make_SQLite_test_suite, 38 make_SQLite_test_suite,
38 make_UTF8_test_suite, 39 make_UTF8_test_suite,
39 make_line_reader_test_suite, 40 make_line_reader_test_suite,
40 - make_fast_lexer_test_suite 41 + make_fast_lexer_test_suite,
  42 + make_fast_lexer_test_suite2,
41 ] 43 ]
42 . 44 .
43 45
anubis_dev/library/test/lexical_analysis/fast_lexer.ut.anubis 0 → 100644
  Show/Hide comments   View file @92c7ed1
  1 +/*
  2 + * Created by PyramIDE.
  3 + * User: ricard
  4 + * Date: 19/09/2008
  5 + * Time: 14:24
  6 + *
  7 + */
  8 +
  9 +read tools/unit_test.anubis
  10 +read lexical_analysis/fast_lexer.anubis
  11 +
  12 +type InfLoopToken:
  13 + line(String),
  14 + eol.
  15 +
  16 +define String
  17 + token_to_string
  18 + (
  19 + LexerOutput(InfLoopToken) output
  20 + ) =
  21 + if output is
  22 + {
  23 + end_of_input then /* no more token: exit the main loop */
  24 + "End of input",
  25 +
  26 + error(b) then
  27 + /* should never happen with this lexer (see the above comment) */
  28 + "Lexer error: ["+to_string(b)+"]",
  29 +
  30 + token(t) then
  31 + /* a token has been recognized */
  32 + if t is
  33 + {
  34 + line(l) then
  35 + "token: line("+l+")"
  36 + eol then
  37 + "token: eol"
  38 + }
  39 + }.
  40 +
  41 +
  42 +define One
  43 + infinite_loop_test
  44 + (
  45 + UnitTestContext ut
  46 + ) =
  47 + if make_lexer_and_automaton([
  48 + lexer_item("#r?#n", return((ByteArray b) |-> token(eol))),
  49 + lexer_item("[^\r\n]*", return((ByteArray b) |-> token(line(to_string(b))))),
  50 + ],
  51 + '#') is
  52 + {
  53 + error(msg) then print("Syntax error in regular expression: "+to_English(msg)+"\n"); assertIsTrue(ut, false, "Can't create lexer"),
  54 + ok(p) then if p is (lexer, automaton) then
  55 + with next_token = lexer(make_lexing_stream("mon texte\n\r\nla suite")),
  56 + assertIsSame(ut, next_token(unique), token(line("mon texte")), token_to_string, "#1 1st line");
  57 + assertIsSame(ut, next_token(unique), token(eol), token_to_string, "#1 1st EOF");
  58 + //assertIsSame(ut, next_token(unique), token(line("")), token_to_string, "#1 2nd line");
  59 + assertIsSame(ut, next_token(unique), token(eol), token_to_string, "#1 2nd EOF");
  60 + assertIsSame(ut, next_token(unique), token(line("la suite")), token_to_string, "#1 3rd line");
  61 + //assertIsSame(ut, next_token(unique), token(eol), token_to_string, "#1 3rd EOF");
  62 + assertIsSame(ut, next_token(unique), end_of_input, token_to_string, "#1 EOF");
  63 +
  64 +
  65 + with next_token = lexer(make_lexing_stream("mon texte\r\n\rla suite")),
  66 + assertIsSame(ut, next_token(unique), token(line("mon texte")), token_to_string, "#2 1st line");
  67 + assertIsSame(ut, next_token(unique), token(eol), token_to_string, "#2 1st EOF");
  68 + assertIsSame(ut, next_token(unique), error(to_byte_array("\r")), token_to_string, "#2 error");
  69 + //assertIsSame(ut, next_token(unique), token(line("")), token_to_string, "#2 2nd line");
  70 + //assertIsSame(ut, next_token(unique), token(eol), token_to_string, "#2 2nd EOF");
  71 + //assertIsSame(ut, next_token(unique), token(line("la suite")), token_to_string, "#2 3rd line");
  72 + //assertIsSame(ut, next_token(unique), token(eol), token_to_string, "#2 3rd EOF");
  73 + //assertIsSame(ut, next_token(unique), end_of_input, token_to_string, "#2 EOF");
  74 + unique
  75 + }.
  76 +
  77 +
  78 +
  79 +public define UnitTestSuite
  80 + make_fast_lexer_test_suite2
  81 + =
  82 + ut_suite("lexical_analysis.fast_lexer",
  83 + [
  84 + ut_fixture("infinite_loop", infinite_loop_test)
  85 + ]).
  86 +
  87 +
anubis_dev/library/test/predefined/fast_lexer.unit_test.anubis
  Show/Hide comments   View file @92c7ed1
1 -  
2 -  
3 -read tools/unit_test.anubis  
4 -  
5 -  
6 -define String  
7 - format  
8 - (  
9 - AtEndOfInput a  
10 - ) =  
11 - if a is  
12 - {  
13 - not_at_end_of_input then "not_at_end_of_input",  
14 - at_end_of_input then "at_end_of_input"  
15 - }.  
16 -  
17 -  
18 -define FastLexerOutput  
19 - test_fast_lexer  
20 - (  
21 - List(FastLexerState) lexer,  
22 - ByteArray input,  
23 - FastLexerLastAccepted last_accepted,  
24 - Int position,  
25 - Word16 state  
26 - ) =  
27 - if make_fast_lexer(lexer) is  
28 - {  
29 - unknown_state(n) then print("fast lexer unknown state: "+to_decimal(n)+"\n"); alert,  
30 - too_many_states then print("too many states.\n"); alert,  
31 - ok(fl) then with result = fl(input,last_accepted,position,state),  
32 -  
33 - if result is  
34 - {  
35 - rejected(w,e,a) then  
36 - print("\nrejected("+to_decimal(w)+","+abs_to_decimal(e)+","+format(a)+")\n"),  
37 - accepted(w,e,a,_) then  
38 - print("\naccepted("+to_decimal(w)+","+abs_to_decimal(e)+","+format(a)+")\n")  
39 - };  
40 -  
41 - result  
42 - }.  
43 -  
44 -  
45 -  
46 - A lexer accepting "g+abu", "g+abuzo",  
47 -  
48 -define List(FastLexerState)  
49 - lexer_1  
50 - =  
51 - [  
52 - /* state 0 */ rejecting([transition('g',1)]),  
53 - /* state 1 */ rejecting([transition('a',2),transition('g',1)]),  
54 - /* state 2 */ rejecting([transition('b',3)]),  
55 - /* state 3 */ rejecting([transition('u',4)]),  
56 - /* state 4 */ accepting([transition('z',5)]),  
57 - /* state 5 */ rejecting([transition('o',6)]),  
58 - /* state 6 */ accepting([ ])  
59 - ].  
60 -  
61 -  
62 -define One  
63 - fast_lexer_test  
64 - (  
65 - UnitTestContext ut  
66 - ) =  
67 -  
68 - /* starting in state 0, position 0 */  
69 - assertIsSame(ut,  
70 - test_fast_lexer(lexer_1,to_byte_array("gab"),none,0,0),  
71 - rejected(3,3,at_end_of_input), "1");  
72 - unique.  
73 -  
74 -  
75 - assertIsSame(ut,  
76 - test_fast_lexer(lexer_1,to_byte_array("gabu"),none,bol,neol,0,0),  
77 - accepted(4,4,at_end_of_input), "2");  
78 - assertIsSame(ut,  
79 - test_fast_lexer(lexer_1,to_byte_array("ggggabu"),none,bol,neol,0,0),  
80 - accepted(4,7,at_end_of_input), "2bis");  
81 - assertIsSame(ut,  
82 - test_fast_lexer(lexer_1,to_byte_array("gabuz"),none,bol,neol,0,0),  
83 - accepted(4,4,at_end_of_input), "3");  
84 - assertIsSame(ut,  
85 - test_fast_lexer(lexer_1,to_byte_array("gmbuz"),none,bol,neol,0,0),  
86 - rejected(1,1,not_at_end_of_input),"4");  
87 - assertIsSame(ut,  
88 - test_fast_lexer(lexer_1,to_byte_array("gabuzo"),none,bol,neol,0,0),  
89 - accepted(6,6,at_end_of_input), "5");  
90 - assertIsSame(ut,  
91 - test_fast_lexer(lexer_1,to_byte_array("gabuzobof"),none,bol,neol,0,0),  
92 - accepted(6,6,not_at_end_of_input), "6");  
93 - assertIsSame(ut,  
94 - test_fast_lexer(lexer_1,to_byte_array("gggggabuzobof"),none,bol,neol,0,0),  
95 - accepted(6,10,not_at_end_of_input), "6bis");  
96 -  
97 - /* restarting from some other state (with or without an already accepted position) */  
98 -  
99 - assertIsSame(ut,  
100 - test_fast_lexer(lexer_1,to_byte_array("bu"),none,bol,neol,0,2),  
101 - accepted(4,2,at_end_of_input), "7");  
102 - assertIsSame(ut,  
103 - test_fast_lexer(lexer_1,to_byte_array("gabuzfff"),last(4,4),bol,neol,5,5),  
104 - accepted(4,4,not_at_end_of_input), "8");  
105 - assertIsSame(ut,  
106 - test_fast_lexer(lexer_1,to_byte_array("gabuzomeu"),last(4,4),bol,neol,5,5),  
107 - accepted(6,6,not_at_end_of_input), "9");  
108 -  
109 - /* testing bol and eol */  
110 - assertIsSame(ut,  
111 - test_fast_lexer(lexer_1,to_byte_array("meu"),none,bol,neol,0,0),  
112 - accepted(6,6,not_at_end_of_input), "9");  
113 -  
114 -  
115 - unique.  
116 -  
117 -  
118 -  
119 -  
120 -  
121 -  
122 -  
123 -public define UnitTestSuite  
124 - make_fast_lexer_test_suite  
125 - =  
126 - ut_suite("predefined.fast_lexer",  
127 - [  
128 - ut_fixture("fast lexer", fast_lexer_test)  
129 - ]).  
130 -  
131 -  
132 -global define One  
133 - fast_lexer_unit_test  
134 - (  
135 - List(String) args  
136 - )=  
137 - execute_tests([make_fast_lexer_test_suite], args).  
138 -  
139 -  
140 \ No newline at end of file 1 \ No newline at end of file
  2 +
  3 +
  4 +read tools/unit_test.anubis
  5 +
  6 +
  7 +define String
  8 + format
  9 + (
  10 + AtEndOfInput a
  11 + ) =
  12 + if a is
  13 + {
  14 + not_at_end_of_input then "not_at_end_of_input",
  15 + at_end_of_input then "at_end_of_input"
  16 + }.
  17 +
  18 +
  19 +define FastLexerOutput
  20 + test_fast_lexer
  21 + (
  22 + List(FastLexerState) lexer,
  23 + ByteArray input,
  24 + FastLexerLastAccepted last_accepted,
  25 + Int position,
  26 + Word16 state
  27 + ) =
  28 + if make_fast_lexer(lexer) is
  29 + {
  30 + unknown_state(n) then print("fast lexer unknown state: "+to_decimal(n)+"\n"); alert,
  31 + too_many_states then print("too many states.\n"); alert,
  32 + ok(fl) then with result = fl(input,last_accepted,position,state),
  33 +
  34 + if result is
  35 + {
  36 + rejected(w,e,a) then
  37 + print("\nrejected("+to_decimal(w)+","+abs_to_decimal(e)+","+format(a)+")\n"),
  38 + accepted(w,e,a,_) then
  39 + print("\naccepted("+to_decimal(w)+","+abs_to_decimal(e)+","+format(a)+")\n")
  40 + };
  41 +
  42 + result
  43 + }.
  44 +
  45 +
  46 +
  47 + A lexer accepting "g+abu", "g+abuzo",
  48 +
  49 +define List(FastLexerState)
  50 + lexer_1
  51 + =
  52 + [
  53 + /* state 0 */ rejecting([transition('g',1)]),
  54 + /* state 1 */ rejecting([transition('a',2),transition('g',1)]),
  55 + /* state 2 */ rejecting([transition('b',3)]),
  56 + /* state 3 */ rejecting([transition('u',4)]),
  57 + /* state 4 */ accepting([transition('z',5)]),
  58 + /* state 5 */ rejecting([transition('o',6)]),
  59 + /* state 6 */ accepting([ ])
  60 + ].
  61 +
  62 +
  63 +define One
  64 + fast_lexer_test
  65 + (
  66 + UnitTestContext ut
  67 + ) =
  68 +
  69 + /* starting in state 0, position 0 */
  70 + assertIsSame(ut,
  71 + test_fast_lexer(lexer_1,to_byte_array("gab"),none,0,0),
  72 + rejected(3,3,at_end_of_input), "1");
  73 + unique.
  74 +
  75 +
  76 + assertIsSame(ut,
  77 + test_fast_lexer(lexer_1,to_byte_array("gabu"),none,bol,neol,0,0),
  78 + accepted(4,4,at_end_of_input), "2");
  79 + assertIsSame(ut,
  80 + test_fast_lexer(lexer_1,to_byte_array("ggggabu"),none,bol,neol,0,0),
  81 + accepted(4,7,at_end_of_input), "2bis");
  82 + assertIsSame(ut,
  83 + test_fast_lexer(lexer_1,to_byte_array("gabuz"),none,bol,neol,0,0),
  84 + accepted(4,4,at_end_of_input), "3");
  85 + assertIsSame(ut,
  86 + test_fast_lexer(lexer_1,to_byte_array("gmbuz"),none,bol,neol,0,0),
  87 + rejected(1,1,not_at_end_of_input),"4");
  88 + assertIsSame(ut,
  89 + test_fast_lexer(lexer_1,to_byte_array("gabuzo"),none,bol,neol,0,0),
  90 + accepted(6,6,at_end_of_input), "5");
  91 + assertIsSame(ut,
  92 + test_fast_lexer(lexer_1,to_byte_array("gabuzobof"),none,bol,neol,0,0),
  93 + accepted(6,6,not_at_end_of_input), "6");
  94 + assertIsSame(ut,
  95 + test_fast_lexer(lexer_1,to_byte_array("gggggabuzobof"),none,bol,neol,0,0),
  96 + accepted(6,10,not_at_end_of_input), "6bis");
  97 +
  98 + /* restarting from some other state (with or without an already accepted position) */
  99 +
  100 + assertIsSame(ut,
  101 + test_fast_lexer(lexer_1,to_byte_array("bu"),none,bol,neol,0,2),
  102 + accepted(4,2,at_end_of_input), "7");
  103 + assertIsSame(ut,
  104 + test_fast_lexer(lexer_1,to_byte_array("gabuzfff"),last(4,4),bol,neol,5,5),
  105 + accepted(4,4,not_at_end_of_input), "8");
  106 + assertIsSame(ut,
  107 + test_fast_lexer(lexer_1,to_byte_array("gabuzomeu"),last(4,4),bol,neol,5,5),
  108 + accepted(6,6,not_at_end_of_input), "9");
  109 +
  110 + /* testing bol and eol */
  111 + assertIsSame(ut,
  112 + test_fast_lexer(lexer_1,to_byte_array("meu"),none,bol,neol,0,0),
  113 + accepted(6,6,not_at_end_of_input), "9");
  114 +
  115 +
  116 + unique.
  117 +
  118 +
  119 +
  120 +
  121 +
  122 +
  123 +public define UnitTestSuite
  124 + make_fast_lexer_test_suite
  125 + =
  126 + ut_suite("predefined.fast_lexer",
  127 + [
  128 + ut_fixture("fast lexer", fast_lexer_test)
  129 + ]).
  130 +
  131 +
anubis_dev/library/test/tools/line_reader.ut.anubis
  Show/Hide comments   View file @92c7ed1
@@ -31,11 +31,39 @@ define One @@ -31,11 +31,39 @@ define One
31 } 31 }
32 }. 32 }.
33 33
  34 +define One
  35 + infinite_loop_test
  36 + (
  37 + UnitTestContext ut
  38 + ) =
  39 + if make_line_reader("mon texte\n\r\nla suite") is
  40 + {
  41 + failure then assertIsTrue(ut, false, "#1 make_line_reader"),
  42 + success(lr) then
  43 + assertIsSuccessString(ut, read_line(lr), "mon texte", "#1 1st line");
  44 + assertIsSuccessString(ut, read_line(lr), "", "#1 2nd line");
  45 + assertIsSuccessString(ut, read_line(lr), "la suite", "#1 3rd line");
  46 + assertIsFailure(ut, read_line(lr), "#1 EOF");
  47 + unique
  48 + };
  49 +
  50 + if make_line_reader("mon texte\r\n\rla suite") is
  51 + {
  52 + failure then assertIsTrue(ut, false, "#2 make_line_reader"),
  53 + success(lr) then
  54 + assertIsSuccessString(ut, read_line(lr), "mon texte", "#2 1st line");
  55 + assertIsSuccessString(ut, read_line(lr), "", "#2 2nd line");
  56 + assertIsSuccessString(ut, read_line(lr), "la suite", "#2 3rd line");
  57 + assertIsFailure(ut, read_line(lr), "#2 EOF");
  58 + unique
  59 + }.
  60 +
34 public define UnitTestSuite 61 public define UnitTestSuite
35 make_line_reader_test_suite 62 make_line_reader_test_suite
36 = 63 =
37 ut_suite("tools.line_reader", 64 ut_suite("tools.line_reader",
38 [ 65 [
39 ut_fixture("line_reader", line_reader_test), 66 ut_fixture("line_reader", line_reader_test),
  67 + ut_fixture("infinite_loop", infinite_loop_test, false),
40 ]). 68 ]).
41 69
anubis_dev/library/tools/line_reader.anubis
  Show/Hide comments   View file @92c7ed1
@@ -30,7 +30,7 @@ public define Maybe(String) @@ -30,7 +30,7 @@ public define Maybe(String)
30 ( 30 (
31 LineReader lr, 31 LineReader lr,
32 ) = 32 ) =
33 - if lr is line_reader(lexer, _) then 33 + if lr is line_reader(lexer, offset) then
34 if lexer(unique) is 34 if lexer(unique) is
35 { 35 {
36 end_of_input then /* no more token: exit the main loop */ 36 end_of_input then /* no more token: exit the main loop */
@@ -46,7 +46,7 @@ public define Maybe(String) @@ -46,7 +46,7 @@ public define Maybe(String)
46 if t is 46 if t is
47 { 47 {
48 line(l) then 48 line(l) then
49 - //print("tk: line("+l+")\n"); 49 + //print("tk: line("+l+") @ "+abs_to_decimal(offset(unique))+"\n");
50 forget(lexer(unique)); // reading EOL 50 forget(lexer(unique)); // reading EOL
51 success(l), 51 success(l),
52 eol then 52 eol then
@@ -64,6 +64,7 @@ public define Maybe(LineReader) @@ -64,6 +64,7 @@ public define Maybe(LineReader)
64 ) = 64 ) =
65 if make_lexer_and_automaton([ 65 if make_lexer_and_automaton([
66 lexer_item("#r?#n", return((ByteArray b) |-> token(eol))), 66 lexer_item("#r?#n", return((ByteArray b) |-> token(eol))),
  67 + lexer_item("#r", return((ByteArray b) |-> token(eol))),
67 lexer_item("[^\r\n]*", return((ByteArray b) |-> token(line(to_string(b))))), 68 lexer_item("[^\r\n]*", return((ByteArray b) |-> token(line(to_string(b))))),
68 ], 69 ],
69 '#') is 70 '#') is
anubis_dev/library/tools/unit_test.anubis
  Show/Hide comments   View file @92c7ed1
@@ -18,11 +18,13 @@ type UnitTestSuiteResult: @@ -18,11 +18,13 @@ type UnitTestSuiteResult:
18 ut_suite_result(String test_suite_name, 18 ut_suite_result(String test_suite_name,
19 Word32 passed_count, 19 Word32 passed_count,
20 Word32 failed_count, 20 Word32 failed_count,
  21 + Word32 ignored_count,
21 List(UnitTestResult) fixture_results). 22 List(UnitTestResult) fixture_results).
22 23
23 type UnitTestGlobalResult: 24 type UnitTestGlobalResult:
24 ut_global_result(Word32 total_passed, 25 ut_global_result(Word32 total_passed,
25 Word32 total_failed, 26 Word32 total_failed,
  27 + Word32 total_ignored,
26 List(UnitTestSuiteResult) suite_results). 28 List(UnitTestSuiteResult) suite_results).
27 29
28 public type UnitTestContext: 30 public type UnitTestContext:
@@ -51,28 +53,34 @@ define UnitTestContext @@ -51,28 +53,34 @@ define UnitTestContext
51 53
52 public type UnitTestFixture: 54 public type UnitTestFixture:
53 ut_fixture(String test_name, 55 ut_fixture(String test_name,
54 - (UnitTestContext) -> One the_test). 56 + (UnitTestContext) -> One the_test,
  57 + Bool active). // set this to false to ignore this fixture
  58 +
  59 +public define UnitTestFixture
  60 + ut_fixture(String test_name,
  61 + (UnitTestContext) -> One the_test) = ut_fixture(test_name, the_test, true).
55 62
56 public type UnitTestSuite: 63 public type UnitTestSuite:
57 ut_suite(String suite_name, 64 ut_suite(String suite_name,
58 List(UnitTestFixture) fixtures). 65 List(UnitTestFixture) fixtures).
59 66
60 -define (Word32 /*passed*/, Word32 /*failed*/, List(UnitTestResult)) 67 +define (Word32 /*passed*/, Word32 /*failed*/, Word32 /*ignored*/, List(UnitTestResult))
61 run_test_suite 68 run_test_suite
62 ( 69 (
63 List(UnitTestFixture) tests, 70 List(UnitTestFixture) tests,
64 String filter, 71 String filter,
65 Word32 passed, 72 Word32 passed,
66 Word32 failed, 73 Word32 failed,
  74 + Word32 ignored,
67 List(UnitTestResult) results 75 List(UnitTestResult) results
68 ) = 76 ) =
69 if tests is 77 if tests is
70 { 78 {
71 - [ ] then (passed, failed, reverse(results)), //print("All tests finish"), 79 + [ ] then (passed, failed, ignored, reverse(results)), //print("All tests finish"),
72 [test . t] then 80 [test . t] then
73 //print("\nTesting " + test.test_name + "... (filter = '"+filter+"')\n"); 81 //print("\nTesting " + test.test_name + "... (filter = '"+filter+"')\n");
74 with ut = make_new_context, 82 with ut = make_new_context,
75 - if length(filter) = 0 | filter = test.test_name then 83 + if (length(filter) = 0 & active(test)) | filter = test.test_name then
76 the_test(test)(ut); 84 the_test(test)(ut);
77 if *ut.failed_tests is 85 if *ut.failed_tests is
78 { 86 {
@@ -81,6 +89,7 @@ define (Word32 /*passed*/, Word32 /*failed*/, List(UnitTestResult)) @@ -81,6 +89,7 @@ define (Word32 /*passed*/, Word32 /*failed*/, List(UnitTestResult))
81 filter, 89 filter,
82 passed + 1, 90 passed + 1,
83 failed, 91 failed,
  92 + ignored,
84 [ut_result(test.test_name, ok(unique)) . results]), 93 [ut_result(test.test_name, ok(unique)) . results]),
85 94
86 [h . _] then print("F"); 95 [h . _] then print("F");
@@ -94,13 +103,15 @@ define (Word32 /*passed*/, Word32 /*failed*/, List(UnitTestResult)) @@ -94,13 +103,15 @@ define (Word32 /*passed*/, Word32 /*failed*/, List(UnitTestResult))
94 filter, 103 filter,
95 passed, 104 passed,
96 failed + 1, 105 failed + 1,
  106 + ignored,
97 map_add(*ut.failed_tests, results)) 107 map_add(*ut.failed_tests, results))
98 } 108 }
99 else 109 else
100 run_test_suite(t, 110 run_test_suite(t,
101 filter, 111 filter,
102 passed, 112 passed,
103 - failed, 113 + failed,
  114 + ignored + if active(test) then 0 else 1, // only explicitely ignored test are counted
104 results) 115 results)
105 }. 116 }.
106 117
@@ -108,18 +119,19 @@ define (Word32 /*passed*/, Word32 /*failed*/, List(UnitTestResult)) @@ -108,18 +119,19 @@ define (Word32 /*passed*/, Word32 /*failed*/, List(UnitTestResult))
108 119
109 120
110 121
111 -define (Word32 /* passed */, Word32 /* failed */, List(UnitTestSuiteResult)) 122 +define (Word32 /* passed */, Word32 /* failed */, Word32 /*ignored*/, List(UnitTestSuiteResult))
112 execute_tests 123 execute_tests
113 ( 124 (
114 List(UnitTestSuite) suites, 125 List(UnitTestSuite) suites,
115 String filter, 126 String filter,
116 Word32 passed, 127 Word32 passed,
117 Word32 failed, 128 Word32 failed,
  129 + Word32 ignored,
118 List(UnitTestSuiteResult) results 130 List(UnitTestSuiteResult) results
119 )= 131 )=
120 if suites is 132 if suites is
121 { 133 {
122 - [ ] then (passed, failed, results), 134 + [ ] then (passed, failed, ignored, results),
123 [h . t] then 135 [h . t] then
124 with filter_length = length(filter), 136 with filter_length = length(filter),
125 fixture_name_length = length(h.suite_name), 137 fixture_name_length = length(h.suite_name),
@@ -127,15 +139,16 @@ define (Word32 /* passed */, Word32 /* failed */, List(UnitTestSuiteResult)) @@ -127,15 +139,16 @@ define (Word32 /* passed */, Word32 /* failed */, List(UnitTestSuiteResult))
127 if filter_length = 0 | start_with(filter + ".", h.suite_name + ".") | start_with(h.suite_name + ".", filter + ".") then //"." is added to avoid partial name match 139 if filter_length = 0 | start_with(filter + ".", h.suite_name + ".") | start_with(h.suite_name + ".", filter + ".") then //"." is added to avoid partial name match
128 with new_filter = if filter_length > fixture_name_length then force(sub_string(filter, fixture_name_length + 1, filter_length - fixture_name_length - 1), "") 140 with new_filter = if filter_length > fixture_name_length then force(sub_string(filter, fixture_name_length + 1, filter_length - fixture_name_length - 1), "")
129 else "", 141 else "",
130 - run_test_suite(h.fixtures, new_filter, 0, 0, []) 142 + run_test_suite(h.fixtures, new_filter, 0, 0, 0, [])
131 else 143 else
132 - (0, 0, []),  
133 - if all_fixture_results is (fixture_passed, fixture_failed, fixture_results) then 144 + (0, 0, 0, []),
  145 + if all_fixture_results is (fixture_passed, fixture_failed, fixture_ignored, fixture_results) then
134 execute_tests(t, 146 execute_tests(t,
135 filter, 147 filter,
136 passed + fixture_passed, 148 passed + fixture_passed,
137 failed + fixture_failed, 149 failed + fixture_failed,
138 - [ut_suite_result(h.suite_name, fixture_passed, fixture_failed, fixture_results) 150 + ignored + fixture_ignored,
  151 + [ut_suite_result(h.suite_name, fixture_passed, fixture_failed, fixture_ignored, fixture_results)
139 . results]) 152 . results])
140 }. 153 }.
141 154
@@ -196,7 +209,7 @@ public define One @@ -196,7 +209,7 @@ public define One
196 )= 209 )=
197 with start_time = (UTime)unow, 210 with start_time = (UTime)unow,
198 filter = force(nth(0, args), ""), 211 filter = force(nth(0, args), ""),
199 - if execute_tests(suites, filter, 0, 0, []) is (passed, failed, results) then 212 + if execute_tests(suites, filter, 0, 0, 0, []) is (passed, failed, ignored, results) then
200 println(""); 213 println("");
201 output_suite_results(results); 214 output_suite_results(results);
202 println("----------------------------------------------------------------------"); 215 println("----------------------------------------------------------------------");
@@ -204,9 +217,13 @@ public define One @@ -204,9 +217,13 @@ public define One
204 println("Ran " + (passed + failed) + " tests in " + 217 println("Ran " + (passed + failed) + " tests in " +
205 to_decimal(duration.seconds) + "." + zero_pad_n(3, duration.microseconds \ 1000) + "s"); 218 to_decimal(duration.seconds) + "." + zero_pad_n(3, duration.microseconds \ 1000) + "s");
206 println(""); 219 println("");
  220 + println("Tests passed = " + passed);
  221 + println("Tests failed = " + failed);
  222 + println("Tests ignored = " + ignored);
  223 + println("");
207 ( 224 (
208 if failed >+ 0 then 225 if failed >+ 0 then
209 - println("FAILED (failures=" + failed + ")") 226 + println("FAILED")
210 else 227 else
211 println("OK") 228 println("OK")
212 ); 229 );