Anubis / Xlib

Commit c6a745b40a91600997b004616732177d9e2a7fd9

Authored by Julien Verneuil
1 parent bc99be1d

improve "eol" function 

improve "print_yaml_data" function to print lists
add list parsing (WIP, fail in some conditions)
fix parsing stuff
Showing 2 changed files with 185 additions and 34 deletions   Show diff stats
Inline Side-by-side
file_processor_io/yaml/yaml.anubis
  Show/Hide comments   View file @c6a745b
... ... @@ -12,6 +12,8 @@ read tools/streams.anubis
12 12 transmit yaml_type.anubis
13 13 read yaml_parser.anubis
14 14  
  15 +define One print_yaml_map (String indent_str, List(YAML_Key_Value) so_far).
  16 +
15 17 define One
16 18 print_yaml_value
17 19 (
... ... @@ -23,6 +25,49 @@ define One
23 25 if value = "" then unique else println(indent_str + " - Value: " + value).
24 26  
25 27 define One
  28 + print_yaml_list
  29 + (
  30 + String indent_str,
  31 + List(YAML_Value) so_far
  32 + )=
  33 + if so_far is
  34 + {
  35 + [ ] then
  36 + unique,
  37 +
  38 + [l . t] then
  39 + if l is
  40 + {
  41 + null then
  42 + print_yaml_value(indent_str + " ", "null", ""),
  43 +
  44 + string(str) then
  45 + print_yaml_value(indent_str + " ", "String", *str),
  46 +
  47 + int(integer) then
  48 + print_yaml_value(indent_str + " ", "Integer", "" + *integer),
  49 +
  50 + bool(boolean) then
  51 + with value_str =
  52 + if *boolean then
  53 + "true"
  54 + else
  55 + "false",
  56 + print_yaml_value(indent_str + " ", "Boolean", "" + value_str),
  57 +
  58 + map(key_values) then
  59 + print_yaml_map(indent_str + " ", *key_values),
  60 +
  61 + list(yaml_values) then
  62 + println(indent_str + " - List: [");
  63 + print_yaml_list(indent_str + " ", *yaml_values);
  64 + println(indent_str + " ]"),
  65 + };
  66 + (if t is [] then unique else println(""));
  67 + print_yaml_list(indent_str, t)
  68 + }.
  69 +
  70 +define One
26 71 print_yaml_map
27 72 (
28 73 String indent_str,
... ... @@ -57,7 +102,9 @@ define One
57 102 print_yaml_map(indent_str + " ", *key_values),
58 103  
59 104 list(yaml_values) then
60   - println("list content (todo)")
  105 + println(indent_str + " - List: [");
  106 + print_yaml_list(indent_str + " ", *yaml_values);
  107 + println(indent_str + " ]"),
61 108 };
62 109  
63 110 (if t is [] then unique else println(""));
... ... @@ -81,7 +128,8 @@ define One
81 128 print_yaml_map("", reverse(*key_values)),
82 129  
83 130 list(values) then
84   - println("list (todo))")
  131 + println("list)");
  132 + print_yaml_list("", *values)
85 133 };
86 134  
87 135 print_yaml_nodes(t)
... ... @@ -244,5 +292,6 @@ global define One
244 292 (
245 293 List(String) dummy
246 294 )=
247   - print_yaml_data(load(my_anubis_directory+"/IDPS/suricata/test.yaml")).
  295 + with yaml_parsing_result = load(my_anubis_directory+"/IDPS/suricata/test.yaml"),
  296 + print_yaml_data(yaml_parsing_result).
248 297  
... ...
file_processor_io/yaml/yaml_parser.anubis
  Show/Hide comments   View file @c6a745b
... ... @@ -41,9 +41,19 @@ define Maybe(YAML_Parser_Node)
41 41 List(Word8) so_far
42 42 ).
43 43  
44   -define Bool eol(Word8 c) =
  44 +define Bool eol(Stream s, Word8 c) =
45 45 if c = 10 then true // LF
46   - else if c = 13 then true // CR
  46 + else if c = 13 then // CR
  47 + if read_byte(s) is // for CR LF
  48 + {
  49 + failure then true,
  50 + success(next_char) then
  51 + if next_char = 10 then
  52 + true
  53 + else
  54 + unput_byte(next_char, s);
  55 + true
  56 + }
47 57 else false.
48 58  
49 59 define One
... ... @@ -55,7 +65,7 @@ define One
55 65 {
56 66 failure then unique,
57 67 success(c) then
58   - if eol(c) then
  68 + if eol(s, c) then
59 69 unique
60 70 else
61 71 skip_to_next_line(s)
... ... @@ -85,7 +95,7 @@ define One
85 95 {
86 96 failure then unique,
87 97 success(c) then
88   - if eol(c) then
  98 + if eol(s, c) then
89 99 skip_to_next_node(s)
90 100 else if c = ' ' then
91 101 skip_to_next_node(s)
... ... @@ -152,10 +162,11 @@ define Maybe(List(YAML_Key_Value))
152 162 skip_spaces(s);
153 163 skip_to_next_node(s);
154 164 with current_indent = current_column(s),
155   - //println("current_indent: " + current_indent);
156   - //println("block_indent: " + block_indent);
  165 + println("reading block map entries");
  166 + println("current_indent: " + current_indent);
  167 + println("block_indent: " + block_indent);
157 168 if current_indent = block_indent then
158   - if _read_node(s, yaml_p_info(parent_indent, current_indent, level), none, []) is
  169 + if _read_node(s, yaml_p_info(block_indent, current_indent, level), none, []) is
159 170 {
160 171 failure then
161 172 success([]),
... ... @@ -174,10 +185,10 @@ define Maybe(List(YAML_Key_Value))
174 185 }
175 186  
176 187 list(yaml_lv) then
177   - failure,
  188 + success(so_far),
178 189  
179 190 value(yaml_value) then
180   - failure
  191 + success(so_far)
181 192 }
182 193 }
183 194 else
... ... @@ -186,10 +197,45 @@ define Maybe(List(YAML_Key_Value))
186 197 define Maybe(List(YAML_Value))
187 198 read_block_list
188 199 (
189   - Stream s,
  200 + Stream s,
  201 + YAML_Parser_Info p_info,
  202 + Int block_indent,
190 203 List(YAML_Value) so_far
191 204 )=
192   - failure.
  205 + since p_info is yaml_p_info(parent_indent, context_indent, level),
  206 + skip_spaces(s);
  207 + skip_to_next_node(s);
  208 + with current_indent = current_column(s),
  209 + //println("current_indent: " + current_indent);
  210 + //println("block_indent: " + block_indent);
  211 + if current_indent = block_indent then
  212 + if _read_node(s, yaml_p_info(block_indent, current_indent, level), none, []) is
  213 + {
  214 + failure then
  215 + success([]),
  216 +
  217 + success(node_type) then
  218 + if node_type is
  219 + {
  220 + map(yaml_kvl) then
  221 + failure,
  222 +
  223 + list(yaml_lv) then
  224 + if yaml_lv is
  225 + {
  226 + [ ] then
  227 + failure,
  228 +
  229 + [h . _] then
  230 + read_block_list(s, p_info, block_indent, [ h . so_far ]),
  231 + }
  232 +
  233 + value(yaml_value) then
  234 + failure
  235 + }
  236 + }
  237 + else
  238 + success(so_far).
193 239  
194 240 define Maybe(YAML_Value)
195 241 read_value
... ... @@ -208,7 +254,7 @@ define Maybe(YAML_Value)
208 254 {
209 255 failure then success(null),
210 256 success(c) then
211   - if eol(c) then // \n
  257 + if eol(s, c) then // \n
212 258 if ctx_info is
213 259 {
214 260 none then
... ... @@ -240,8 +286,8 @@ define Maybe(YAML_Value)
240 286 read_value(s, p_info, line, [c . so_far]),
241 287  
242 288 block_start then
  289 + println("block_start");
243 290 unput_byte(c, s);
244   - //println("block_start");
245 291 skip_to_next_node(s);
246 292 with current_indent = current_column(s),
247 293 if current_indent =< parent_indent then
... ... @@ -253,7 +299,8 @@ define Maybe(YAML_Value)
253 299 unput_byte(c, s);
254 300 //println("block level " + level);
255 301 skip_to_next_node(s);
256   - if _read_node(s, p_info, none, []) is
  302 + with current_indent = current_column(s),
  303 + if _read_node(s, yaml_p_info(current_indent, current_indent, level), none, []) is
257 304 {
258 305 failure then
259 306 success(null),
... ... @@ -262,31 +309,80 @@ define Maybe(YAML_Value)
262 309 if node_type is
263 310 {
264 311 map(yaml_kvl) then
265   - if read_block_maps(s, p_info, context_indent, yaml_kvl) is
  312 + println("map node");
  313 + if read_block_maps(s, p_info, current_indent, yaml_kvl) is
266 314 {
267 315 failure then
268   - success(map(var(yaml_kvl))),
  316 + success(map(var(reverse(yaml_kvl)))),
269 317  
270 318 success(list_kv) then
271 319 success(map(var(reverse(list_kv))))
272 320 }
273 321  
274 322 list(yaml_lv) then
275   - if read_block_list(s, yaml_lv) is
  323 + println("list node");
  324 + if read_block_list(s, p_info, current_indent, yaml_lv) is
276 325 {
277 326 failure then
278 327 success(list(var(yaml_lv))),
279 328  
280 329 success(list_v) then
281   - success(list(var(list_v)))
  330 + success(list(var(reverse(list_v))))
282 331 }
283 332  
284 333 value(yaml_value) then
  334 + //println("value node");
285 335 success(yaml_value)
286 336 }
287 337 }
288 338 }
289 339 }.
  340 +
  341 +define Maybe(List(YAML_Value))
  342 + read_values
  343 + (
  344 + Word8 c,
  345 + Stream s,
  346 + YAML_Parser_Info p_info,
  347 + List(YAML_Value) so_far
  348 + )=
  349 + if c = '-' then
  350 + if read_byte(s) is
  351 + {
  352 + failure then
  353 + success(reverse(so_far)),
  354 +
  355 + success(next_char) then
  356 + if next_char = ' ' then
  357 + skip_spaces(s);
  358 + println("found a list");
  359 + since p_info is yaml_p_info(parent_indent, context_indent, level),
  360 + with current_indent = current_column(s),
  361 + if (current_indent-1) =< parent_indent then
  362 + success(so_far)
  363 + else
  364 + if read_value(s, yaml_p_info((current_indent-1), current_indent, level), block, []) is
  365 + {
  366 + failure then
  367 + success(so_far),
  368 +
  369 + success(v) then
  370 + //skip_to_next_node(s);
  371 + if read_byte(s) is
  372 + {
  373 + failure then
  374 + success(so_far),
  375 +
  376 + success(next_next_char) then
  377 + read_values(next_next_char, s, p_info, [v . so_far])
  378 + }
  379 + }
  380 + else
  381 + unput_byte(next_char, s);
  382 + success(so_far),
  383 + }
  384 + else
  385 + success(so_far).
290 386  
291 387 define Maybe(YAML_Parser_Node)
292 388 _read_node
... ... @@ -296,23 +392,29 @@ define Maybe(YAML_Parser_Node)
296 392 YAML_Parser_Node_Info node_info,
297 393 List(Word8) so_far
298 394 )=
299   - skip_spaces(s);
300 395 if read_byte(s) is
301 396 {
302   - failure then failure,
  397 + failure then failure,
303 398 success(c) then
304 399 if node_info is
305 400 {
306 401 none then
307 402 if c = '-' then
308   - todo//_read_list(s, p_info, in_double_quote, so_far)
  403 + if read_values(c, s, p_info, []) is
  404 + {
  405 + failure then
  406 + failure,
  407 +
  408 + success(lv) then
  409 + success(list(lv))
  410 + }
309 411 else if c = '\"' then
310 412 _read_node(s, p_info, in_double_quote, so_far)
311 413 else if c = '\'' then
312 414 _read_node(s, p_info, in_single_quote, so_far)
313 415 else
314 416 _read_node(s, p_info, unquoted, [ c . so_far ]),
315   -
  417 +
316 418 unquoted then
317 419 if c = ':' then
318 420 if read_byte(s) is
... ... @@ -322,36 +424,35 @@ define Maybe(YAML_Parser_Node)
322 424 success(map([key_value(key_name, var(null))])),
323 425  
324 426 success(next_char) then
325   - if eol(next_char) then // \n
  427 + if eol(s, next_char) then // \n
326 428 unput_byte(next_char, s);
327 429 with key_name = implode(reverse(so_far)),
328 430  
329   - //println("key found [LF] \n name: " + key_name);
  431 + println("key found [LF] \n name: " + key_name);
330 432 skip_spaces(s);
331 433 with value = read_value(s, p_info, none, []),
332 434 if value is
333 435 {
334   - failure then failure,
  436 + failure then failure,
335 437 success(v) then
336 438 success(map([key_value(key_name, var(v))]))
337 439 }
338 440 else if next_char = ' ' then
339 441 with key_name = implode(reverse(so_far)),
340   - //println("key found [ ]\n name: " + key_name);
  442 + println("key found [ ]\n name: " + key_name);
341 443 skip_spaces(s);
342 444 with value = read_value(s, p_info, none, []),
343 445 if value is
344 446 {
345   - failure then failure,
  447 + failure then failure,
346 448 success(v) then
347 449 success(map([key_value(key_name, var(v))]))
348 450 }
349   -
350 451 else
351 452 todo
352 453 //_read_node(s, p_info, unquoted, [next_char . [c . so_far]])
353 454 }
354   - else if eol(c) then
  455 + else if eol(s, c) then
355 456 with str = implode(reverse(so_far)),
356 457 //println("value (str): " + str);
357 458 success(value(get_yaml_value(str)))
... ... @@ -385,7 +486,7 @@ public define Maybe(YAML_Document)
385 486  
386 487 success(c) then
387 488 //check for comment
388   - if eol(c) then
  489 + if eol(s, c) then
389 490 get_yaml_document_content(s, doc)
390 491 else if c = '#' then
391 492 //println("skipping comment ["+line+"]");
... ... @@ -401,8 +502,9 @@ public define Maybe(YAML_Document)
401 502 if mb_node is
402 503 {
403 504 failure then
404   - println("Can't get map entries");
  505 + println("Can't parse YAML node at column " + current_column(s) + " line " + current_line(s));
405 506 get_yaml_document_content(s, doc),
  507 +
406 508 success(node_type) then
407 509 since doc is yaml_document(nodes),
408 510 if node_type is
... ...