Anubis / Anubis

Commit 29bf7c8af26734894a8ba1c06208a6ff1da95d6b

Authored by Cédric RICARD
1 parent c471cddb

- Added 'split_line' function into string.anubis 

- Improved 'LineReader' fonctions allowing to not recreate lexer each time.
Showing 2 changed files with 84 additions and 10 deletions   Show diff stats
Inline Side-by-side
anubis_dev/library/system/string.anubis
  Show/Hide comments   View file @29bf7c8
... ... @@ -666,6 +666,49 @@ public define List(String)
666 666 )=
667 667 split_by_tokens(explode(line), [token], [], []).
668 668  
  669 +define List(String)
  670 + _split_lines
  671 + (
  672 + List(Word8) text,
  673 + List(Word8) current,
  674 + List(String) so_far
  675 + ) =
  676 + if text is
  677 + {
  678 + [] then
  679 + if length(current) > 0 then
  680 + reverse([ implode(reverse(current)) . so_far])
  681 + else
  682 + reverse(so_far),
  683 + [ char . t ] then
  684 + if char = 13 then // CR
  685 + if t is
  686 + {
  687 + [] then
  688 + if length(current) > 0 then
  689 + reverse([ implode(reverse(current)) . so_far])
  690 + else
  691 + reverse(so_far),
  692 + [ char2 . t2 ] then
  693 + if char2 = 10 then // LF
  694 + _split_lines( t2, [], [ implode(reverse(current)) . so_far])
  695 + else
  696 + _split_lines( t, [], [ implode(reverse(current)) . so_far])
  697 + }
  698 + else if char = 10 then // LF
  699 + _split_lines( t, [], [ implode(reverse(current)) . so_far])
  700 + else
  701 + _split_lines( t, [ char . current ], so_far)
  702 + }.
  703 +
  704 +/// Split a text into lines, whether the line ends are (CRLF / LF / CR)
  705 +public define List(String)
  706 + split_lines
  707 + (
  708 + String text,
  709 + )=
  710 + _split_lines(explode(text), [], []).
  711 +
669 712 define (Int, Int)
670 713 trim
671 714 (
... ...
anubis_dev/library/tools/line_reader.anubis
  Show/Hide comments   View file @29bf7c8
... ... @@ -14,23 +14,27 @@ type Token:
14 14 line(String),
15 15 eol.
16 16  
  17 +public type LineReaderLexer:
  18 + line_reader_lexer(LexingStream -> One -> LexerOutput(Token) /*lexer_base*/).
  19 +
17 20 public type LineReader:
18 21 line_reader(One -> LexerOutput(Token) /*lexer*/,
19   - One -> Int /*offset*/).
  22 + One -> Int /*offset*/,
  23 + LineReaderLexer /*lexer_base*/).
20 24  
21 25 public define Int
22 26 current_offset
23 27 (
24 28 LineReader lr,
25 29 ) =
26   - if lr is line_reader(_, offset) then offset(unique).
  30 + if lr is line_reader(_, offset, _) then offset(unique).
27 31  
28 32 public define Maybe(String)
29 33 read_line
30 34 (
31 35 LineReader lr,
32 36 ) =
33   - if lr is line_reader(lexer, offset) then
  37 + if lr is line_reader(lexer, offset, _) then
34 38 if lexer(unique) is
35 39 {
36 40 end_of_input then /* no more token: exit the main loop */
... ... @@ -57,11 +61,9 @@ public define Maybe(String)
57 61  
58 62  
59 63  
60   -public define Maybe(LineReader)
61   - make_line_reader
62   - (
63   - LexingStream ls,
64   - ) =
  64 +public define Maybe(LineReaderLexer)
  65 + make_line_reader_lexer
  66 + =
65 67 if make_lexer_and_automaton([
66 68 lexer_item("#r?#n", return((ByteArray b) |-> token(eol))),
67 69 lexer_item("#r", return((ByteArray b) |-> token(eol))),
... ... @@ -70,11 +72,32 @@ public define Maybe(LineReader)
70 72 '#') is
71 73 {
72 74 error(msg) then print("Syntax error in regular expression: "+to_English(msg)+"\n"); failure,
73   - ok(p) then if p is (lexer, automaton) then
  75 + ok(p) then if p is (lexer, automaton) then success(line_reader_lexer(lexer))
  76 + }.
  77 +
  78 +public define LineReader
  79 + make_line_reader
  80 + (
  81 + LexingStream ls,
  82 + LineReaderLexer make_lexer
  83 + ) =
  84 + if make_lexer is line_reader_lexer(lexer) then
  85 + line_reader(lexer(ls), offset_counter(ls), make_lexer).
  86 +
  87 +public define Maybe(LineReader)
  88 + make_line_reader
  89 + (
  90 + LexingStream ls,
  91 + ) =
  92 + if make_line_reader_lexer is
  93 + {
  94 + failure then failure,
  95 + success(lexer) then
74 96 /* prepare the input stream */
75   - success(line_reader(lexer(ls), offset_counter(ls)))
  97 + success(make_line_reader(ls, lexer))
76 98 }.
77 99  
  100 +
78 101 public define Maybe(LineReader)
79 102 make_line_reader
80 103 (
... ... @@ -98,3 +121,11 @@ public define Maybe(LineReader)
98 121 }.
99 122  
100 123  
  124 +public define LineReader
  125 + reset_line_reader
  126 + (
  127 + LineReader lr,
  128 + LexingStream ls,
  129 + ) =
  130 + if lr is line_reader(lexer, offset, make_lexer) then
  131 + make_line_reader(ls, make_lexer).
... ...