Anubis / Xlib

Commit d98711a4f40f03c52d0510d060ab870eddae936e

Authored by Julien Verneuil
1 parent 17019850

add support for parsing keys/values (WIP for lists and non block map/list) 

add "print_yaml_data" function to print a human readable representation of the parser result
add "null" alternative to YAML_Value type
Showing 3 changed files with 484 additions and 102 deletions   Show diff stats
Inline Side-by-side
file_processor_io/yaml/yaml.anubis
  Show/Hide comments   View file @d98711a
... ... @@ -6,11 +6,111 @@
6 6 * © Calexium
7 7 */
8 8  
  9 +read tools/basis.anubis
9 10 read tools/streams.anubis
10 11  
11 12 transmit yaml_type.anubis
12 13 read yaml_parser.anubis
13 14  
  15 +define One
  16 + print_yaml_value
  17 + (
  18 + String indent_str,
  19 + String type,
  20 + String value
  21 + )=
  22 + println(indent_str + " - Type: " + type);
  23 + if value = "" then unique else println(indent_str + " - Value: " + value).
  24 +
  25 +define One
  26 + print_yaml_map
  27 + (
  28 + String indent_str,
  29 + List(YAML_Key_Value) so_far
  30 + )=
  31 + if so_far is
  32 + {
  33 + [ ] then unique,
  34 + [map . t] then
  35 + since map is key_value(key, yaml_value),
  36 + println(indent_str + " - Name: " + key);
  37 + if *yaml_value is
  38 + {
  39 + null then
  40 + print_yaml_value(indent_str, "null", ""),
  41 +
  42 + string(str) then
  43 + print_yaml_value(indent_str, "String", *str),
  44 +
  45 + int(integer) then
  46 + print_yaml_value(indent_str, "Integer", "" + *integer),
  47 +
  48 + bool(boolean) then
  49 + with value_str =
  50 + if *boolean then
  51 + "true"
  52 + else
  53 + "false",
  54 + print_yaml_value(indent_str, "Boolean", "" + value_str),
  55 +
  56 + map(key_values) then
  57 + print_yaml_map(indent_str + " ", *key_values),
  58 +
  59 + list(yaml_values) then
  60 + println("list content (todo)")
  61 + };
  62 +
  63 + (if t is [] then unique else println(""));
  64 + print_yaml_map(indent_str, t)
  65 + }.
  66 +
  67 +define One
  68 + print_yaml_nodes
  69 + (
  70 + List(YAML_Node) so_far
  71 + )=
  72 + if so_far is
  73 + {
  74 + [ ] then unique,
  75 + [node . t] then
  76 + print(" YAML Node (");
  77 + if node is
  78 + {
  79 + map(key_values) then
  80 + println("map)");
  81 + print_yaml_map("", reverse(*key_values)),
  82 +
  83 + list(values) then
  84 + println("list (todo))")
  85 + };
  86 +
  87 + print_yaml_nodes(t)
  88 + }.
  89 +
  90 +define One
  91 + print_yaml_documents
  92 + (
  93 + List(YAML_Document) so_far,
  94 + Int n
  95 + )=
  96 + if so_far is
  97 + {
  98 + [ ] then unique,
  99 + [doc . t] then
  100 + since doc is yaml_document(nodes),
  101 + println("YAML Document " + n + ":");
  102 + print_yaml_nodes(reverse(*nodes));
  103 + print_yaml_documents(t, n + 1)
  104 + }.
  105 +
  106 +public define One
  107 + print_yaml_data
  108 + (
  109 + YAML yaml
  110 + )=
  111 + since yaml is yaml(documents),
  112 + print_yaml_documents(*documents, 1).
  113 +
14 114 public define Maybe(YAML_Value)
15 115 get_value
16 116 (
... ... @@ -144,6 +244,5 @@ global define One
144 244 (
145 245 List(String) dummy
146 246 )=
147   - forget(load(my_anubis_directory+"/IDPS/suricata/suricata.yaml"))
148   - .
  247 + print_yaml_data(load(my_anubis_directory+"/IDPS/suricata/test.yaml")).
149 248  
... ...
file_processor_io/yaml/yaml_parser.anubis
  Show/Hide comments   View file @d98711a
... ... @@ -13,131 +13,413 @@ read tools/string.anubis
13 13 read system/string.anubis
14 14  
15 15 type YAML_Parser_Info:
16   - yaml_p_info(Int indent_size, Int level).
  16 + yaml_p_info(Int parent_indent, Int context_indent, Int level).
  17 +
  18 +type YAML_Parser_Node_Info:
  19 + none,
  20 + unquoted,
  21 + in_single_quote,
  22 + in_double_quote.
17 23  
18   -define Maybe(String)
19   - _read_key
  24 +type YAML_Parser_Context_Info:
  25 + none,
  26 + line,
  27 + block_start,
  28 + block.
  29 +
  30 +type YAML_Parser_Node:
  31 + map(List(YAML_Key_Value) kvl),
  32 + list(List(YAML_Value) vl),
  33 + value(YAML_Value v).
  34 +
  35 +define Maybe(YAML_Parser_Node)
  36 + _read_node
  37 + (
  38 + Stream s,
  39 + YAML_Parser_Info p_info,
  40 + YAML_Parser_Node_Info node_info,
  41 + List(Word8) so_far
  42 + ).
  43 +
  44 +define Bool eol(Word8 c) =
  45 + if c = 10 then true // \n
  46 + else false.
  47 +
  48 +define One
  49 + skip_to_next_line
20 50 (
21   - Stream current_line,
22   - List(Word8) so_far
  51 + Stream s
23 52 )=
24   - failure.
  53 + if read_byte(s) is
  54 + {
  55 + failure then unique,
  56 + success(c) then
  57 + if eol(c) then
  58 + unique
  59 + else
  60 + skip_to_next_line(s)
  61 + }.
25 62  
26   - if read_byte(current_line) is
  63 +define One
  64 + skip_spaces
  65 + (
  66 + Stream s
  67 + )=
  68 + if read_byte(s) is
27 69 {
28   - failure then failure,
29   - success(char) then
30   - if in_word then
31   -
32   - else if in_single_quote then
33   - else if
34   - if char
35   - .
  70 + failure then unique,
  71 + success(c) then
  72 + if c = ' ' then skip_spaces(s)
  73 + else
  74 + unput_byte(c, s);
  75 + unique
  76 + }.
36 77  
37   -define Maybe(String)
38   - read_key
  78 +define One
  79 + skip_to_next_node
39 80 (
40   - Stream current_line
  81 + Stream s
41 82 )=
42   - _read_key(current_line, []).
  83 + if read_byte(s) is
  84 + {
  85 + failure then unique,
  86 + success(c) then
  87 + if eol(c) then
  88 + skip_to_next_node(s)
  89 + else if c = ' ' then
  90 + skip_to_next_node(s)
  91 + else if c = '#' then
  92 + //skip_spaces(s);
  93 + skip_to_next_line(s);
  94 + skip_to_next_node(s)
  95 + else
  96 + unput_byte(c, s);
  97 + unique
  98 + }.
43 99  
44   -define Maybe(YAML_Key_Value)
45   - read_key_value
  100 +define List(Word8)
  101 + trim_right
46 102 (
47   - Stream s,
48   - YAML_Parser_Info p_info,
49   - Stream current_line,
  103 + List(Word8) char_list,
  104 + Word8 token
50 105 )=
51   - if read_key(current_line) is
  106 + if char_list is
52 107 {
53   - failure then failure,
54   - success(key) then failure
55   -// if read_value(s, p_info, current_line) is
56   -// {
57   -// failure then failure,
58   -// success(value) then
59   -// success(key_value(key, var(value)))
60   -// }
61   - }
62   - .
  108 + [ ] then [ ],
  109 + [h . t] then
  110 + if h = token
  111 + then trim_right(t, token)
  112 + else char_list
  113 + }.
  114 +
  115 +define String
  116 + trim_right
  117 + (
  118 + String str,
  119 + Word8 token
  120 + )=
  121 + implode(reverse(trim_right(reverse(explode(str)), token))).
  122 +
  123 +define YAML_Value
  124 + get_yaml_value
  125 + (
  126 + String str
  127 + )=
  128 + with str = trim_token(str, ' '),
  129 + if str = "yes" then
  130 + bool(var(true))
  131 + else if str = "no" then
  132 + bool(var(false))
  133 + else
  134 + if decimal_scan(str) is
  135 + {
  136 + failure then
  137 + string(var(str)),
  138 + success(integer) then
  139 + int(var(integer))
  140 + }.
  141 +
  142 +define Maybe(List(YAML_Key_Value))
  143 + read_block_maps
  144 + (
  145 + Stream s,
  146 + YAML_Parser_Info p_info,
  147 + Int block_indent,
  148 + List(YAML_Key_Value) so_far
  149 + )=
  150 + since p_info is yaml_p_info(parent_indent, context_indent, level),
  151 + skip_spaces(s);
  152 + skip_to_next_node(s);
  153 + with current_indent = current_column(s),
  154 + //println("current_indent: " + current_indent);
  155 + //println("block_indent: " + block_indent);
  156 + if current_indent = block_indent then
  157 + if _read_node(s, yaml_p_info(parent_indent, current_indent, level), none, []) is
  158 + {
  159 + failure then
  160 + success([]),
  161 +
  162 + success(node_type) then
  163 + if node_type is
  164 + {
  165 + map(yaml_kvl) then
  166 + if yaml_kvl is
  167 + {
  168 + [ ] then
  169 + failure,
  170 +
  171 + [h . _] then
  172 + read_block_maps(s, p_info, block_indent, [ h . so_far ]),
  173 + }
  174 +
  175 + list(yaml_lv) then
  176 + failure,
  177 +
  178 + value(yaml_value) then
  179 + failure
  180 + }
  181 + }
  182 + else
  183 + success(so_far).
63 184  
64   -public define List(YAML_Key_Value)
65   - get_yaml_map
  185 +define Maybe(List(YAML_Value))
  186 + read_block_list
66 187 (
67   - Stream s,
68   - YAML_Parser_Info p_info,
69   - Stream current_line,
70   - List(YAML_Key_Value) so_far
  188 + Stream s,
  189 + List(YAML_Value) so_far
71 190 )=
72   - if read_key_value(s, p_info, current_line) is
  191 + failure.
  192 +
  193 +define Maybe(YAML_Value)
  194 + read_value
  195 + (
  196 + Stream s,
  197 + YAML_Parser_Info p_info,
  198 + YAML_Parser_Context_Info ctx_info,
  199 + List(Word8) so_far
  200 + )=
  201 + since p_info is yaml_p_info(parent_indent, context_indent, level),
  202 + //skip_spaces(s);
  203 +
  204 + with current_indent = current_column(s),
  205 +
  206 + if read_byte(s) is
  207 + {
  208 + failure then success(null),
  209 + success(c) then
  210 + if eol(c) then // \n
  211 + if ctx_info is
  212 + {
  213 + none then
  214 + read_value(s, p_info, block_start, so_far),
  215 +
  216 + line then
  217 + with value_str = implode(reverse(so_far)),
  218 +
  219 + //println(" value: " + value_str);
  220 + success(get_yaml_value(value_str)),
  221 +
  222 + block_start then
  223 + read_value(s, p_info, block_start, so_far),
  224 +
  225 + block then
  226 + read_value(s, p_info, block, so_far)
  227 + }
  228 + else if c = '#' then
  229 + skip_to_next_line(s);
  230 + read_value(s, p_info, ctx_info, so_far)
  231 + else
  232 + if ctx_info is
  233 + {
  234 + none then
  235 + read_value(s, p_info, line, [c . so_far]),
  236 +
  237 + line then
  238 + //println(implode(reverse([c . so_far])));
  239 + read_value(s, p_info, line, [c . so_far]),
  240 +
  241 + block_start then
  242 + unput_byte(c, s);
  243 + //println("block_start");
  244 + skip_to_next_node(s);
  245 + with current_indent = current_column(s),
  246 + if current_indent =< parent_indent then
  247 + success(null)
  248 + else
  249 + read_value(s, yaml_p_info(parent_indent, current_indent, level + 1), block, so_far),
  250 +
  251 + block then
  252 + unput_byte(c, s);
  253 + //println("block level " + level);
  254 + skip_to_next_node(s);
  255 + if _read_node(s, p_info, none, []) is
  256 + {
  257 + failure then
  258 + success(null),
  259 +
  260 + success(node_type) then
  261 + if node_type is
  262 + {
  263 + map(yaml_kvl) then
  264 + if read_block_maps(s, p_info, context_indent, yaml_kvl) is
  265 + {
  266 + failure then
  267 + success(map(var(yaml_kvl))),
  268 +
  269 + success(list_kv) then
  270 + success(map(var(reverse(list_kv))))
  271 + }
  272 +
  273 + list(yaml_lv) then
  274 + if read_block_list(s, yaml_lv) is
  275 + {
  276 + failure then
  277 + success(list(var(yaml_lv))),
  278 +
  279 + success(list_v) then
  280 + success(list(var(list_v)))
  281 + }
  282 +
  283 + value(yaml_value) then
  284 + success(yaml_value)
  285 + }
  286 + }
  287 + }
  288 + }.
  289 +
  290 +define Maybe(YAML_Parser_Node)
  291 + _read_node
  292 + (
  293 + Stream s,
  294 + YAML_Parser_Info p_info,
  295 + YAML_Parser_Node_Info node_info,
  296 + List(Word8) so_far
  297 + )=
  298 + skip_spaces(s);
  299 + if read_byte(s) is
73 300 {
74   - failure then
75   - println("can't read key value");
76   - reverse(so_far),
77   - success(key_value) then
78   - if read_line(s) is
  301 + failure then failure,
  302 + success(c) then
  303 + if node_info is
79 304 {
80   - failure then reverse(so_far),
81   - success(line) then
82   - with line_s = make_stream(line),
83   - //skip all blanks of begin of line
84   - skip_blanks(line_s);
85   - with indent = current_column(line_s),
86   - get_yaml_map(s, p_info, line_s, [key_value . so_far])
  305 + none then
  306 + if c = '-' then
  307 + todo//_read_list(s, p_info, in_double_quote, so_far)
  308 + else if c = '\"' then
  309 + _read_node(s, p_info, in_double_quote, so_far)
  310 + else if c = '\'' then
  311 + _read_node(s, p_info, in_single_quote, so_far)
  312 + else
  313 + _read_node(s, p_info, unquoted, [ c . so_far ]),
  314 +
  315 + unquoted then
  316 + if c = ':' then
  317 + if read_byte(s) is
  318 + {
  319 + failure then
  320 + with key_name = implode(reverse(so_far)),
  321 + success(map([key_value(key_name, var(null))])),
  322 +
  323 + success(next_char) then
  324 + if eol(next_char) then // \n
  325 + unput_byte(next_char, s);
  326 + with key_name = implode(reverse(so_far)),
  327 +
  328 + //println("key found [LF] \n name: " + key_name);
  329 + skip_spaces(s);
  330 + with value = read_value(s, p_info, none, []),
  331 + if value is
  332 + {
  333 + failure then failure,
  334 + success(v) then
  335 + success(map([key_value(key_name, var(v))]))
  336 + }
  337 + else if next_char = ' ' then
  338 + with key_name = implode(reverse(so_far)),
  339 + //println("key found [ ]\n name: " + key_name);
  340 + skip_spaces(s);
  341 + with value = read_value(s, p_info, none, []),
  342 + if value is
  343 + {
  344 + failure then failure,
  345 + success(v) then
  346 + success(map([key_value(key_name, var(v))]))
  347 + }
  348 +
  349 + else
  350 + todo
  351 + //_read_node(s, p_info, unquoted, [next_char . [c . so_far]])
  352 + }
  353 + else if eol(c) then
  354 + with str = implode(reverse(so_far)),
  355 + //println("value (str): " + str);
  356 + success(value(get_yaml_value(str)))
  357 + else
  358 + _read_node(s, p_info, unquoted, [ c . so_far ]),
  359 +
  360 + in_single_quote then
  361 + todo,
  362 +
  363 + in_double_quote then
  364 + todo
87 365 }
88 366 }.
89   -
  367 +
90 368 public define Maybe(YAML_Document)
91 369 get_yaml_document_content
92 370 (
93 371 Stream s,
94 372 YAML_Document doc
95 373 )=
96   - if read_line(s) is
97   - {
98   - failure then success(doc)
99   - success(line) then
100   - with line_s = make_stream(line),
101   -
102   - //skip all blanks of begin of line
103   - skip_blanks(line_s);
104   - with indent = current_column(line_s),
105   -
106   - if read_byte(line_s) is
107   - {
108   - failure then
109   - println("skipping blank line");
110   - get_yaml_document_content(s, doc), //blank line
111   -
112   - success(c) then
113   - //check for comment
114   - if c = '#' then
115   - //println("skipping comment ["+line+"]");
116   - get_yaml_document_content(s, doc)
  374 + //skip all blanks of begin of line
  375 + skip_spaces(s);
  376 + with indent = current_column(s),
  377 +
  378 + if read_byte(s) is
  379 + {
  380 + failure then
  381 + //println("skipping blank line");
  382 + //get_yaml_document_content(s, doc), //blank line
  383 + success(doc)
117 384  
118   - //check for item of the list
119   - else if c = '-' then
120   - println("list item indent["+indent+"]["+line+"]");
121   - get_yaml_document_content(s, doc)
  385 + success(c) then
  386 + //check for comment
  387 + if eol(c) then
  388 + get_yaml_document_content(s, doc)
  389 + else if c = '#' then
  390 + //println("skipping comment ["+line+"]");
  391 + skip_to_next_line(s);
  392 + get_yaml_document_content(s, doc)
  393 +
  394 + //check for entry of the map
  395 + else
  396 + unput_byte(c, s);
  397 + //println("map entry indent["+indent+"]");//["+line+"]");
  398 + with mb_node = _read_node(s, yaml_p_info(indent, indent, 0), none, /*make_stream(line)*/[]),
122 399  
123   - //check for entry of the map
124   - else
125   - println("map entry indent["+indent+"]["+line+"]");
126   - with list_key_val = get_yaml_map(s, yaml_p_info(indent, 0), line_s, []),
127   -
128   - if list_key_val is
129   - {
130   - [] then
131   - println("Can't get map entries");
132   - get_yaml_document_content(s, doc),
133   - [_ . _] then
134   - since doc is yaml_document(nodes),
135   - nodes <- [map(var(list_key_val)) . *nodes];
136   - get_yaml_document_content(s, yaml_document(nodes))
137   - }
138   -
139   - } //read_byte
140   - }. //read_line
  400 + if mb_node is
  401 + {
  402 + failure then
  403 + println("Can't get map entries");
  404 + get_yaml_document_content(s, doc),
  405 + success(node_type) then
  406 + since doc is yaml_document(nodes),
  407 + if node_type is
  408 + {
  409 + map(yaml_kv) then
  410 + nodes <- [map(var(yaml_kv)) . *nodes];
  411 + get_yaml_document_content(s, yaml_document(nodes))
  412 +
  413 + list(yaml_lv) then
  414 + nodes <- [list(var(yaml_lv)) . *nodes];
  415 + get_yaml_document_content(s, yaml_document(nodes))
  416 +
  417 + value(yaml_value) then
  418 + failure
  419 + }
  420 + }
  421 +
  422 + }. //read_byte
141 423  
142 424 public define Maybe(YAML_Document)
143 425 get_yaml_document
... ...
file_processor_io/yaml/yaml_type.anubis
  Show/Hide comments   View file @d98711a
... ... @@ -18,6 +18,7 @@ public type YAML_Key_Value:...
18 18 public type YAML_Value:
19 19 //comment(String value),
20 20 //blank,
  21 + null,
21 22 string(Var(String) s_value),
22 23 int (Var(Int) i_value),
23 24 //float (Float value),
... ... @@ -31,8 +32,8 @@ public type YAML_Key_Value:
31 32 key_value(String key, Var(YAML_Value) y_value).
32 33  
33 34 public type YAML_Node:
34   - map(Var(List(YAML_Key_Value)) key_values),
35   - list(Var(List(YAML_Value)) values).
  35 + map(Var(List(YAML_Key_Value)) key_values),
  36 + list(Var(List(YAML_Value)) values).
36 37  
37 38 public type YAML_Document:
38 39 yaml_document(Var(List(YAML_Node)) nodes).
... ...