Anubis / Xlib

Commit 464ffddd1938589554e75c1a95d70bda896aa8c5

Authored by Julien Verneuil
1 parent 2ac800d6

add support for quoted strings value 

add more boolean values (true/false)
fix parsing bug related to missing "unput_byte" leading to erroneous symbols name
fix parsing bug related to missing items on special case in lists
fix parsing bug related to comments in between list items
fix parsing bug related to comment in a list item
Showing 1 changed file with 140 additions and 52 deletions   Show diff stats
Inline Side-by-side
file_processor_io/yaml/yaml_parser.anubis
  Show/Hide comments   View file @464ffdd
... ... @@ -20,6 +20,12 @@ type YAML_Parser_Node_Info:
20 20 unquoted,
21 21 in_single_quote,
22 22 in_double_quote.
  23 +
  24 +type YAML_Parser_String_State:
  25 + none,
  26 + unquoted,
  27 + in_single_quote,
  28 + in_double_quote.
23 29  
24 30 type YAML_Parser_Context_Info:
25 31 none,
... ... @@ -131,6 +137,60 @@ define String
131 137 )=
132 138 implode(reverse(trim_right(reverse(explode(str)), token))).
133 139  
  140 +define Maybe(YAML_Value)
  141 + extract_yaml_string
  142 + (
  143 + String str,
  144 + Stream s,
  145 + YAML_Parser_String_State ss,
  146 + List(Word8) so_far
  147 + )=
  148 + if read_byte(s) is
  149 + {
  150 + failure then success(string(var(implode(reverse(so_far))))),
  151 + success(c) then
  152 + if ss is
  153 + {
  154 + none then
  155 + if c = '\"' then
  156 + extract_yaml_string(str, s, in_double_quote, so_far)
  157 + else if c = '\'' then
  158 + extract_yaml_string(str, s, in_single_quote, so_far)
  159 + else
  160 + extract_yaml_string(str, s, unquoted, so_far),
  161 +
  162 + unquoted then
  163 + success(string(var(str))),
  164 +
  165 + in_single_quote then
  166 + if read_byte(s) is
  167 + {
  168 + failure then
  169 + if c = '\'' then
  170 + success(string(var(implode(reverse(so_far)))))
  171 + else
  172 + failure,
  173 + success(nc) then
  174 + unput_byte(nc, s);
  175 + extract_yaml_string(str, s, in_single_quote, [c . so_far])
  176 + },
  177 +
  178 + in_double_quote then
  179 + if read_byte(s) is
  180 + {
  181 + failure then
  182 + if c = '\"' then
  183 + success(string(var(implode(reverse(so_far)))))
  184 + else
  185 + failure,
  186 + success(nc) then
  187 + unput_byte(nc, s);
  188 + extract_yaml_string(str, s, in_double_quote, [c . so_far])
  189 + },
  190 + }
  191 + }.
  192 +
  193 +// should be of type Maybe(YAML_Value)
134 194 define YAML_Value
135 195 get_yaml_value
136 196 (
... ... @@ -141,11 +201,19 @@ define YAML_Value
141 201 bool(var(true))
142 202 else if str = "no" then
143 203 bool(var(false))
  204 + else if str = "true" then
  205 + bool(var(true))
  206 + else if str = "false" then
  207 + bool(var(false))
144 208 else
145 209 if decimal_scan(str) is
146 210 {
147 211 failure then
148   - string(var(str)),
  212 + if extract_yaml_string(str, make_stream(str), none, []) is
  213 + {
  214 + failure then null,
  215 + success(v) then v
  216 + }
149 217 success(integer) then
150 218 int(var(integer))
151 219 }.
... ... @@ -161,10 +229,11 @@ define Maybe(List(YAML_Key_Value))
161 229 since p_info is yaml_p_info(parent_indent, context_indent, level),
162 230 skip_spaces(s);
163 231 skip_to_next_node(s);
  232 +
164 233 with current_indent = current_column(s),
165   - println("reading block map entries");
166   - println("current_indent: " + current_indent);
167   - println("block_indent: " + block_indent);
  234 + //println("reading block map entries");
  235 + //println("current_indent: " + current_indent);
  236 + //println("block_indent: " + block_indent);
168 237 if current_indent = block_indent then
169 238 if _read_node(s, yaml_p_info(block_indent, current_indent, level), none, []) is
170 239 {
... ... @@ -263,7 +332,6 @@ define Maybe(YAML_Value)
263 332 line then
264 333 with value_str = implode(reverse(so_far)),
265 334  
266   - //println(" value: " + value_str);
267 335 success(get_yaml_value(value_str)),
268 336  
269 337 block_start then
... ... @@ -272,9 +340,6 @@ define Maybe(YAML_Value)
272 340 block then
273 341 read_value(s, p_info, block, so_far)
274 342 }
275   - else if c = '#' then
276   - skip_to_next_line(s);
277   - read_value(s, p_info, ctx_info, so_far)
278 343 else
279 344 if ctx_info is
280 345 {
... ... @@ -283,18 +348,23 @@ define Maybe(YAML_Value)
283 348  
284 349 line then
285 350 //println(implode(reverse([c . so_far])));
286   - read_value(s, p_info, line, [c . so_far]),
  351 + if c = '#' then
  352 + skip_to_next_line(s);
  353 + success(get_yaml_value(implode(reverse(so_far))))
  354 + else
  355 + read_value(s, p_info, line, [c . so_far]),
287 356  
288 357 block_start then
289   - println("block_start");
  358 +// println("block_start");
290 359 unput_byte(c, s);
291 360 skip_to_next_node(s);
292 361 with current_indent = current_column(s),
  362 + // TODO : this return "null" if list character "- " is on the same column as the key first character (it should not)
293 363 if current_indent =< parent_indent then
294 364 success(null)
295 365 else
296 366 read_value(s, yaml_p_info(parent_indent, current_indent, level + 1), block, so_far),
297   -
  367 +
298 368 block then
299 369 unput_byte(c, s);
300 370 //println("block level " + level);
... ... @@ -309,7 +379,7 @@ define Maybe(YAML_Value)
309 379 if node_type is
310 380 {
311 381 map(yaml_kvl) then
312   - println("map node");
  382 + //println("map node");
313 383 if read_block_maps(s, p_info, current_indent, yaml_kvl) is
314 384 {
315 385 failure then
... ... @@ -320,7 +390,7 @@ define Maybe(YAML_Value)
320 390 }
321 391  
322 392 list(yaml_lv) then
323   - println("list node");
  393 + //println("list node");
324 394 if read_block_list(s, p_info, current_indent, yaml_lv) is
325 395 {
326 396 failure then
... ... @@ -355,7 +425,7 @@ define Maybe(List(YAML_Value))
355 425 success(next_char) then
356 426 if next_char = ' ' then
357 427 skip_spaces(s);
358   - println("found a list");
  428 + //println("found a list");
359 429 since p_info is yaml_p_info(parent_indent, context_indent, level),
360 430 with current_indent = current_column(s),
361 431 if (current_indent-1) =< parent_indent then
... ... @@ -367,23 +437,67 @@ define Maybe(List(YAML_Value))
367 437 success(so_far),
368 438  
369 439 success(v) then
370   - //skip_to_next_node(s);
371 440 if read_byte(s) is
372 441 {
373 442 failure then
374   - success(so_far),
  443 + success([v . so_far]),
375 444  
376 445 success(next_next_char) then
377   - read_values(next_next_char, s, p_info, [v . so_far])
  446 + read_values(next_next_char, s, p_info, [v . so_far])
378 447 }
379 448 }
380 449 else
381 450 unput_byte(next_char, s);
  451 + unput_byte(c, s);
382 452 success(so_far),
383 453 }
384 454 else
  455 + unput_byte(c, s);
385 456 success(so_far).
  457 +
  458 +define Maybe(YAML_Parser_Node)
  459 + get_key_node
  460 + (
  461 + Stream s,
  462 + YAML_Parser_Info p_info,
  463 + List(Word8) so_far
  464 + )=
  465 + if read_byte(s) is
  466 + {
  467 + failure then
  468 + with key_name = implode(reverse(so_far)),
  469 + success(map([key_value(key_name, var(null))])),
386 470  
  471 + success(next_char) then
  472 + if eol(s, next_char) then // \n
  473 + unput_byte(next_char, s);
  474 + with key_name = implode(reverse(so_far)),
  475 +
  476 + //println("key found [LF] \n name: " + key_name);
  477 + skip_spaces(s);
  478 + with value = read_value(s, p_info, none, []),
  479 + if value is
  480 + {
  481 + failure then failure,
  482 + success(v) then
  483 + success(map([key_value(key_name, var(v))]))
  484 + }
  485 + else if next_char = ' ' then
  486 + with key_name = implode(reverse(so_far)),
  487 + //println("key found [ ]\n name: " + key_name);
  488 + skip_spaces(s);
  489 + with value = read_value(s, p_info, none, []),
  490 + if value is
  491 + {
  492 + failure then failure,
  493 + success(v) then
  494 + success(map([key_value(key_name, var(v))]))
  495 + }
  496 + else
  497 + failure
  498 + //_read_node(s, p_info, unquoted, [next_char . [c . so_far]])
  499 + }.
  500 +
387 501 define Maybe(YAML_Parser_Node)
388 502 _read_node
389 503 (
... ... @@ -417,46 +531,18 @@ define Maybe(YAML_Parser_Node)
417 531  
418 532 unquoted then
419 533 if c = ':' then
420   - if read_byte(s) is
421   - {
422   - failure then
423   - with key_name = implode(reverse(so_far)),
424   - success(map([key_value(key_name, var(null))])),
425   -
426   - success(next_char) then
427   - if eol(s, next_char) then // \n
428   - unput_byte(next_char, s);
429   - with key_name = implode(reverse(so_far)),
430   -
431   - println("key found [LF] \n name: " + key_name);
432   - skip_spaces(s);
433   - with value = read_value(s, p_info, none, []),
434   - if value is
435   - {
436   - failure then failure,
437   - success(v) then
438   - success(map([key_value(key_name, var(v))]))
439   - }
440   - else if next_char = ' ' then
441   - with key_name = implode(reverse(so_far)),
442   - println("key found [ ]\n name: " + key_name);
443   - skip_spaces(s);
444   - with value = read_value(s, p_info, none, []),
445   - if value is
446   - {
447   - failure then failure,
448   - success(v) then
449   - success(map([key_value(key_name, var(v))]))
450   - }
451   - else
452   - todo
453   - //_read_node(s, p_info, unquoted, [next_char . [c . so_far]])
454   - }
  534 + get_key_node(s, p_info, so_far)
455 535 else if eol(s, c) then
456 536 with str = implode(reverse(so_far)),
457 537 //println("value (str): " + str);
458 538 success(value(get_yaml_value(str)))
  539 + else if c = '#' then
  540 + with str = implode(reverse(so_far)),
  541 +
  542 + skip_to_next_line(s);
  543 + success(value(get_yaml_value(str)))
459 544 else
  545 + //println("" + implode(reverse(so_far)));
460 546 _read_node(s, p_info, unquoted, [ c . so_far ]),
461 547  
462 548 in_single_quote then
... ... @@ -464,6 +550,7 @@ define Maybe(YAML_Parser_Node)
464 550  
465 551 in_double_quote then
466 552 todo
  553 + // _read_node(s, p_info, in_double_quote, [ c . so_far ])
467 554 }
468 555 }.
469 556  
... ... @@ -518,6 +605,7 @@ public define Maybe(YAML_Document)
518 605 get_yaml_document_content(s, yaml_document(nodes))
519 606  
520 607 value(yaml_value) then
  608 + //println("Value found on root");
521 609 failure
522 610 }
523 611 }
... ...