From 464ffddd1938589554e75c1a95d70bda896aa8c5 Mon Sep 17 00:00:00 2001 From: Julien Verneuil Date: Mon, 27 Jun 2016 17:07:42 +0200 Subject: [PATCH] add support for quoted strings value add more boolean values (true/false) fix parsing bug related to missing "unput_byte" leading to erroneous symbols name fix parsing bug related to missing items on special case in lists fix parsing bug related to comments in between list items fix parsing bug related to comment in a list item --- file_processor_io/yaml/yaml_parser.anubis | 192 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++---------------------------------------------------- 1 file changed, 140 insertions(+), 52 deletions(-) diff --git a/file_processor_io/yaml/yaml_parser.anubis b/file_processor_io/yaml/yaml_parser.anubis index 90a48ec..cd10c9d 100644 --- a/file_processor_io/yaml/yaml_parser.anubis +++ b/file_processor_io/yaml/yaml_parser.anubis @@ -20,6 +20,12 @@ type YAML_Parser_Node_Info: unquoted, in_single_quote, in_double_quote. + +type YAML_Parser_String_State: + none, + unquoted, + in_single_quote, + in_double_quote. type YAML_Parser_Context_Info: none, @@ -131,6 +137,60 @@ define String )= implode(reverse(trim_right(reverse(explode(str)), token))). +define Maybe(YAML_Value) + extract_yaml_string + ( + String str, + Stream s, + YAML_Parser_String_State ss, + List(Word8) so_far + )= + if read_byte(s) is + { + failure then success(string(var(implode(reverse(so_far))))), + success(c) then + if ss is + { + none then + if c = '\"' then + extract_yaml_string(str, s, in_double_quote, so_far) + else if c = '\'' then + extract_yaml_string(str, s, in_single_quote, so_far) + else + extract_yaml_string(str, s, unquoted, so_far), + + unquoted then + success(string(var(str))), + + in_single_quote then + if read_byte(s) is + { + failure then + if c = '\'' then + success(string(var(implode(reverse(so_far))))) + else + failure, + success(nc) then + unput_byte(nc, s); + extract_yaml_string(str, s, in_single_quote, [c . so_far]) + }, + + in_double_quote then + if read_byte(s) is + { + failure then + if c = '\"' then + success(string(var(implode(reverse(so_far))))) + else + failure, + success(nc) then + unput_byte(nc, s); + extract_yaml_string(str, s, in_double_quote, [c . so_far]) + }, + } + }. + +// should be of type Maybe(YAML_Value) define YAML_Value get_yaml_value ( @@ -141,11 +201,19 @@ define YAML_Value bool(var(true)) else if str = "no" then bool(var(false)) + else if str = "true" then + bool(var(true)) + else if str = "false" then + bool(var(false)) else if decimal_scan(str) is { failure then - string(var(str)), + if extract_yaml_string(str, make_stream(str), none, []) is + { + failure then null, + success(v) then v + } success(integer) then int(var(integer)) }. @@ -161,10 +229,11 @@ define Maybe(List(YAML_Key_Value)) since p_info is yaml_p_info(parent_indent, context_indent, level), skip_spaces(s); skip_to_next_node(s); + with current_indent = current_column(s), - println("reading block map entries"); - println("current_indent: " + current_indent); - println("block_indent: " + block_indent); + //println("reading block map entries"); + //println("current_indent: " + current_indent); + //println("block_indent: " + block_indent); if current_indent = block_indent then if _read_node(s, yaml_p_info(block_indent, current_indent, level), none, []) is { @@ -263,7 +332,6 @@ define Maybe(YAML_Value) line then with value_str = implode(reverse(so_far)), - //println(" value: " + value_str); success(get_yaml_value(value_str)), block_start then @@ -272,9 +340,6 @@ define Maybe(YAML_Value) block then read_value(s, p_info, block, so_far) } - else if c = '#' then - skip_to_next_line(s); - read_value(s, p_info, ctx_info, so_far) else if ctx_info is { @@ -283,18 +348,23 @@ define Maybe(YAML_Value) line then //println(implode(reverse([c . so_far]))); - read_value(s, p_info, line, [c . so_far]), + if c = '#' then + skip_to_next_line(s); + success(get_yaml_value(implode(reverse(so_far)))) + else + read_value(s, p_info, line, [c . so_far]), block_start then - println("block_start"); +// println("block_start"); unput_byte(c, s); skip_to_next_node(s); with current_indent = current_column(s), + // TODO : this return "null" if list character "- " is on the same column as the key first character (it should not) if current_indent =< parent_indent then success(null) else read_value(s, yaml_p_info(parent_indent, current_indent, level + 1), block, so_far), - + block then unput_byte(c, s); //println("block level " + level); @@ -309,7 +379,7 @@ define Maybe(YAML_Value) if node_type is { map(yaml_kvl) then - println("map node"); + //println("map node"); if read_block_maps(s, p_info, current_indent, yaml_kvl) is { failure then @@ -320,7 +390,7 @@ define Maybe(YAML_Value) } list(yaml_lv) then - println("list node"); + //println("list node"); if read_block_list(s, p_info, current_indent, yaml_lv) is { failure then @@ -355,7 +425,7 @@ define Maybe(List(YAML_Value)) success(next_char) then if next_char = ' ' then skip_spaces(s); - println("found a list"); + //println("found a list"); since p_info is yaml_p_info(parent_indent, context_indent, level), with current_indent = current_column(s), if (current_indent-1) =< parent_indent then @@ -367,23 +437,67 @@ define Maybe(List(YAML_Value)) success(so_far), success(v) then - //skip_to_next_node(s); if read_byte(s) is { failure then - success(so_far), + success([v . so_far]), success(next_next_char) then - read_values(next_next_char, s, p_info, [v . so_far]) + read_values(next_next_char, s, p_info, [v . so_far]) } } else unput_byte(next_char, s); + unput_byte(c, s); success(so_far), } else + unput_byte(c, s); success(so_far). + +define Maybe(YAML_Parser_Node) + get_key_node + ( + Stream s, + YAML_Parser_Info p_info, + List(Word8) so_far + )= + if read_byte(s) is + { + failure then + with key_name = implode(reverse(so_far)), + success(map([key_value(key_name, var(null))])), + success(next_char) then + if eol(s, next_char) then // \n + unput_byte(next_char, s); + with key_name = implode(reverse(so_far)), + + //println("key found [LF] \n name: " + key_name); + skip_spaces(s); + with value = read_value(s, p_info, none, []), + if value is + { + failure then failure, + success(v) then + success(map([key_value(key_name, var(v))])) + } + else if next_char = ' ' then + with key_name = implode(reverse(so_far)), + //println("key found [ ]\n name: " + key_name); + skip_spaces(s); + with value = read_value(s, p_info, none, []), + if value is + { + failure then failure, + success(v) then + success(map([key_value(key_name, var(v))])) + } + else + failure + //_read_node(s, p_info, unquoted, [next_char . [c . so_far]]) + }. + define Maybe(YAML_Parser_Node) _read_node ( @@ -417,46 +531,18 @@ define Maybe(YAML_Parser_Node) unquoted then if c = ':' then - if read_byte(s) is - { - failure then - with key_name = implode(reverse(so_far)), - success(map([key_value(key_name, var(null))])), - - success(next_char) then - if eol(s, next_char) then // \n - unput_byte(next_char, s); - with key_name = implode(reverse(so_far)), - - println("key found [LF] \n name: " + key_name); - skip_spaces(s); - with value = read_value(s, p_info, none, []), - if value is - { - failure then failure, - success(v) then - success(map([key_value(key_name, var(v))])) - } - else if next_char = ' ' then - with key_name = implode(reverse(so_far)), - println("key found [ ]\n name: " + key_name); - skip_spaces(s); - with value = read_value(s, p_info, none, []), - if value is - { - failure then failure, - success(v) then - success(map([key_value(key_name, var(v))])) - } - else - todo - //_read_node(s, p_info, unquoted, [next_char . [c . so_far]]) - } + get_key_node(s, p_info, so_far) else if eol(s, c) then with str = implode(reverse(so_far)), //println("value (str): " + str); success(value(get_yaml_value(str))) + else if c = '#' then + with str = implode(reverse(so_far)), + + skip_to_next_line(s); + success(value(get_yaml_value(str))) else + //println("" + implode(reverse(so_far))); _read_node(s, p_info, unquoted, [ c . so_far ]), in_single_quote then @@ -464,6 +550,7 @@ define Maybe(YAML_Parser_Node) in_double_quote then todo + // _read_node(s, p_info, in_double_quote, [ c . so_far ]) } }. @@ -518,6 +605,7 @@ public define Maybe(YAML_Document) get_yaml_document_content(s, yaml_document(nodes)) value(yaml_value) then + //println("Value found on root"); failure } } -- libgit2 0.21.4