From d98711a4f40f03c52d0510d060ab870eddae936e Mon Sep 17 00:00:00 2001 From: Julien Verneuil Date: Thu, 23 Jun 2016 17:16:09 +0200 Subject: [PATCH] add support for parsing keys/values (WIP for lists and non block map/list) add "print_yaml_data" function to print a human readable representation of the parser result add "null" alternative to YAML_Value type --- file_processor_io/yaml/yaml.anubis | 103 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-- file_processor_io/yaml/yaml_parser.anubis | 478 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-------------------------------------------------------------------------------------------------- file_processor_io/yaml/yaml_type.anubis | 5 +++-- 3 files changed, 484 insertions(+), 102 deletions(-) diff --git a/file_processor_io/yaml/yaml.anubis b/file_processor_io/yaml/yaml.anubis index 9485e92..9466d83 100644 --- a/file_processor_io/yaml/yaml.anubis +++ b/file_processor_io/yaml/yaml.anubis @@ -6,11 +6,111 @@ * © Calexium */ +read tools/basis.anubis read tools/streams.anubis transmit yaml_type.anubis read yaml_parser.anubis +define One + print_yaml_value + ( + String indent_str, + String type, + String value + )= + println(indent_str + " - Type: " + type); + if value = "" then unique else println(indent_str + " - Value: " + value). + +define One + print_yaml_map + ( + String indent_str, + List(YAML_Key_Value) so_far + )= + if so_far is + { + [ ] then unique, + [map . t] then + since map is key_value(key, yaml_value), + println(indent_str + " - Name: " + key); + if *yaml_value is + { + null then + print_yaml_value(indent_str, "null", ""), + + string(str) then + print_yaml_value(indent_str, "String", *str), + + int(integer) then + print_yaml_value(indent_str, "Integer", "" + *integer), + + bool(boolean) then + with value_str = + if *boolean then + "true" + else + "false", + print_yaml_value(indent_str, "Boolean", "" + value_str), + + map(key_values) then + print_yaml_map(indent_str + " ", *key_values), + + list(yaml_values) then + println("list content (todo)") + }; + + (if t is [] then unique else println("")); + print_yaml_map(indent_str, t) + }. + +define One + print_yaml_nodes + ( + List(YAML_Node) so_far + )= + if so_far is + { + [ ] then unique, + [node . t] then + print(" YAML Node ("); + if node is + { + map(key_values) then + println("map)"); + print_yaml_map("", reverse(*key_values)), + + list(values) then + println("list (todo))") + }; + + print_yaml_nodes(t) + }. + +define One + print_yaml_documents + ( + List(YAML_Document) so_far, + Int n + )= + if so_far is + { + [ ] then unique, + [doc . t] then + since doc is yaml_document(nodes), + println("YAML Document " + n + ":"); + print_yaml_nodes(reverse(*nodes)); + print_yaml_documents(t, n + 1) + }. + +public define One + print_yaml_data + ( + YAML yaml + )= + since yaml is yaml(documents), + print_yaml_documents(*documents, 1). + public define Maybe(YAML_Value) get_value ( @@ -144,6 +244,5 @@ global define One ( List(String) dummy )= - forget(load(my_anubis_directory+"/IDPS/suricata/suricata.yaml")) - . + print_yaml_data(load(my_anubis_directory+"/IDPS/suricata/test.yaml")). diff --git a/file_processor_io/yaml/yaml_parser.anubis b/file_processor_io/yaml/yaml_parser.anubis index 89c3878..d3c75c0 100644 --- a/file_processor_io/yaml/yaml_parser.anubis +++ b/file_processor_io/yaml/yaml_parser.anubis @@ -13,131 +13,413 @@ read tools/string.anubis read system/string.anubis type YAML_Parser_Info: - yaml_p_info(Int indent_size, Int level). + yaml_p_info(Int parent_indent, Int context_indent, Int level). + +type YAML_Parser_Node_Info: + none, + unquoted, + in_single_quote, + in_double_quote. -define Maybe(String) - _read_key +type YAML_Parser_Context_Info: + none, + line, + block_start, + block. + +type YAML_Parser_Node: + map(List(YAML_Key_Value) kvl), + list(List(YAML_Value) vl), + value(YAML_Value v). + +define Maybe(YAML_Parser_Node) + _read_node + ( + Stream s, + YAML_Parser_Info p_info, + YAML_Parser_Node_Info node_info, + List(Word8) so_far + ). + +define Bool eol(Word8 c) = + if c = 10 then true // \n + else false. + +define One + skip_to_next_line ( - Stream current_line, - List(Word8) so_far + Stream s )= - failure. + if read_byte(s) is + { + failure then unique, + success(c) then + if eol(c) then + unique + else + skip_to_next_line(s) + }. - if read_byte(current_line) is +define One + skip_spaces + ( + Stream s + )= + if read_byte(s) is { - failure then failure, - success(char) then - if in_word then - - else if in_single_quote then - else if - if char - . + failure then unique, + success(c) then + if c = ' ' then skip_spaces(s) + else + unput_byte(c, s); + unique + }. -define Maybe(String) - read_key +define One + skip_to_next_node ( - Stream current_line + Stream s )= - _read_key(current_line, []). + if read_byte(s) is + { + failure then unique, + success(c) then + if eol(c) then + skip_to_next_node(s) + else if c = ' ' then + skip_to_next_node(s) + else if c = '#' then + //skip_spaces(s); + skip_to_next_line(s); + skip_to_next_node(s) + else + unput_byte(c, s); + unique + }. -define Maybe(YAML_Key_Value) - read_key_value +define List(Word8) + trim_right ( - Stream s, - YAML_Parser_Info p_info, - Stream current_line, + List(Word8) char_list, + Word8 token )= - if read_key(current_line) is + if char_list is { - failure then failure, - success(key) then failure -// if read_value(s, p_info, current_line) is -// { -// failure then failure, -// success(value) then -// success(key_value(key, var(value))) -// } - } - . + [ ] then [ ], + [h . t] then + if h = token + then trim_right(t, token) + else char_list + }. + +define String + trim_right + ( + String str, + Word8 token + )= + implode(reverse(trim_right(reverse(explode(str)), token))). + +define YAML_Value + get_yaml_value + ( + String str + )= + with str = trim_token(str, ' '), + if str = "yes" then + bool(var(true)) + else if str = "no" then + bool(var(false)) + else + if decimal_scan(str) is + { + failure then + string(var(str)), + success(integer) then + int(var(integer)) + }. + +define Maybe(List(YAML_Key_Value)) + read_block_maps + ( + Stream s, + YAML_Parser_Info p_info, + Int block_indent, + List(YAML_Key_Value) so_far + )= + since p_info is yaml_p_info(parent_indent, context_indent, level), + skip_spaces(s); + skip_to_next_node(s); + with current_indent = current_column(s), + //println("current_indent: " + current_indent); + //println("block_indent: " + block_indent); + if current_indent = block_indent then + if _read_node(s, yaml_p_info(parent_indent, current_indent, level), none, []) is + { + failure then + success([]), + + success(node_type) then + if node_type is + { + map(yaml_kvl) then + if yaml_kvl is + { + [ ] then + failure, + + [h . _] then + read_block_maps(s, p_info, block_indent, [ h . so_far ]), + } + + list(yaml_lv) then + failure, + + value(yaml_value) then + failure + } + } + else + success(so_far). -public define List(YAML_Key_Value) - get_yaml_map +define Maybe(List(YAML_Value)) + read_block_list ( - Stream s, - YAML_Parser_Info p_info, - Stream current_line, - List(YAML_Key_Value) so_far + Stream s, + List(YAML_Value) so_far )= - if read_key_value(s, p_info, current_line) is + failure. + +define Maybe(YAML_Value) + read_value + ( + Stream s, + YAML_Parser_Info p_info, + YAML_Parser_Context_Info ctx_info, + List(Word8) so_far + )= + since p_info is yaml_p_info(parent_indent, context_indent, level), + //skip_spaces(s); + + with current_indent = current_column(s), + + if read_byte(s) is + { + failure then success(null), + success(c) then + if eol(c) then // \n + if ctx_info is + { + none then + read_value(s, p_info, block_start, so_far), + + line then + with value_str = implode(reverse(so_far)), + + //println(" value: " + value_str); + success(get_yaml_value(value_str)), + + block_start then + read_value(s, p_info, block_start, so_far), + + block then + read_value(s, p_info, block, so_far) + } + else if c = '#' then + skip_to_next_line(s); + read_value(s, p_info, ctx_info, so_far) + else + if ctx_info is + { + none then + read_value(s, p_info, line, [c . so_far]), + + line then + //println(implode(reverse([c . so_far]))); + read_value(s, p_info, line, [c . so_far]), + + block_start then + unput_byte(c, s); + //println("block_start"); + skip_to_next_node(s); + with current_indent = current_column(s), + if current_indent =< parent_indent then + success(null) + else + read_value(s, yaml_p_info(parent_indent, current_indent, level + 1), block, so_far), + + block then + unput_byte(c, s); + //println("block level " + level); + skip_to_next_node(s); + if _read_node(s, p_info, none, []) is + { + failure then + success(null), + + success(node_type) then + if node_type is + { + map(yaml_kvl) then + if read_block_maps(s, p_info, context_indent, yaml_kvl) is + { + failure then + success(map(var(yaml_kvl))), + + success(list_kv) then + success(map(var(reverse(list_kv)))) + } + + list(yaml_lv) then + if read_block_list(s, yaml_lv) is + { + failure then + success(list(var(yaml_lv))), + + success(list_v) then + success(list(var(list_v))) + } + + value(yaml_value) then + success(yaml_value) + } + } + } + }. + +define Maybe(YAML_Parser_Node) + _read_node + ( + Stream s, + YAML_Parser_Info p_info, + YAML_Parser_Node_Info node_info, + List(Word8) so_far + )= + skip_spaces(s); + if read_byte(s) is { - failure then - println("can't read key value"); - reverse(so_far), - success(key_value) then - if read_line(s) is + failure then failure, + success(c) then + if node_info is { - failure then reverse(so_far), - success(line) then - with line_s = make_stream(line), - //skip all blanks of begin of line - skip_blanks(line_s); - with indent = current_column(line_s), - get_yaml_map(s, p_info, line_s, [key_value . so_far]) + none then + if c = '-' then + todo//_read_list(s, p_info, in_double_quote, so_far) + else if c = '\"' then + _read_node(s, p_info, in_double_quote, so_far) + else if c = '\'' then + _read_node(s, p_info, in_single_quote, so_far) + else + _read_node(s, p_info, unquoted, [ c . so_far ]), + + unquoted then + if c = ':' then + if read_byte(s) is + { + failure then + with key_name = implode(reverse(so_far)), + success(map([key_value(key_name, var(null))])), + + success(next_char) then + if eol(next_char) then // \n + unput_byte(next_char, s); + with key_name = implode(reverse(so_far)), + + //println("key found [LF] \n name: " + key_name); + skip_spaces(s); + with value = read_value(s, p_info, none, []), + if value is + { + failure then failure, + success(v) then + success(map([key_value(key_name, var(v))])) + } + else if next_char = ' ' then + with key_name = implode(reverse(so_far)), + //println("key found [ ]\n name: " + key_name); + skip_spaces(s); + with value = read_value(s, p_info, none, []), + if value is + { + failure then failure, + success(v) then + success(map([key_value(key_name, var(v))])) + } + + else + todo + //_read_node(s, p_info, unquoted, [next_char . [c . so_far]]) + } + else if eol(c) then + with str = implode(reverse(so_far)), + //println("value (str): " + str); + success(value(get_yaml_value(str))) + else + _read_node(s, p_info, unquoted, [ c . so_far ]), + + in_single_quote then + todo, + + in_double_quote then + todo } }. - + public define Maybe(YAML_Document) get_yaml_document_content ( Stream s, YAML_Document doc )= - if read_line(s) is - { - failure then success(doc) - success(line) then - with line_s = make_stream(line), - - //skip all blanks of begin of line - skip_blanks(line_s); - with indent = current_column(line_s), - - if read_byte(line_s) is - { - failure then - println("skipping blank line"); - get_yaml_document_content(s, doc), //blank line - - success(c) then - //check for comment - if c = '#' then - //println("skipping comment ["+line+"]"); - get_yaml_document_content(s, doc) + //skip all blanks of begin of line + skip_spaces(s); + with indent = current_column(s), + + if read_byte(s) is + { + failure then + //println("skipping blank line"); + //get_yaml_document_content(s, doc), //blank line + success(doc) - //check for item of the list - else if c = '-' then - println("list item indent["+indent+"]["+line+"]"); - get_yaml_document_content(s, doc) + success(c) then + //check for comment + if eol(c) then + get_yaml_document_content(s, doc) + else if c = '#' then + //println("skipping comment ["+line+"]"); + skip_to_next_line(s); + get_yaml_document_content(s, doc) + + //check for entry of the map + else + unput_byte(c, s); + //println("map entry indent["+indent+"]");//["+line+"]"); + with mb_node = _read_node(s, yaml_p_info(indent, indent, 0), none, /*make_stream(line)*/[]), - //check for entry of the map - else - println("map entry indent["+indent+"]["+line+"]"); - with list_key_val = get_yaml_map(s, yaml_p_info(indent, 0), line_s, []), - - if list_key_val is - { - [] then - println("Can't get map entries"); - get_yaml_document_content(s, doc), - [_ . _] then - since doc is yaml_document(nodes), - nodes <- [map(var(list_key_val)) . *nodes]; - get_yaml_document_content(s, yaml_document(nodes)) - } - - } //read_byte - }. //read_line + if mb_node is + { + failure then + println("Can't get map entries"); + get_yaml_document_content(s, doc), + success(node_type) then + since doc is yaml_document(nodes), + if node_type is + { + map(yaml_kv) then + nodes <- [map(var(yaml_kv)) . *nodes]; + get_yaml_document_content(s, yaml_document(nodes)) + + list(yaml_lv) then + nodes <- [list(var(yaml_lv)) . *nodes]; + get_yaml_document_content(s, yaml_document(nodes)) + + value(yaml_value) then + failure + } + } + + }. //read_byte public define Maybe(YAML_Document) get_yaml_document diff --git a/file_processor_io/yaml/yaml_type.anubis b/file_processor_io/yaml/yaml_type.anubis index 2bc25c5..c607cc8 100644 --- a/file_processor_io/yaml/yaml_type.anubis +++ b/file_processor_io/yaml/yaml_type.anubis @@ -18,6 +18,7 @@ public type YAML_Key_Value:... public type YAML_Value: //comment(String value), //blank, + null, string(Var(String) s_value), int (Var(Int) i_value), //float (Float value), @@ -31,8 +32,8 @@ public type YAML_Key_Value: key_value(String key, Var(YAML_Value) y_value). public type YAML_Node: - map(Var(List(YAML_Key_Value)) key_values), - list(Var(List(YAML_Value)) values). + map(Var(List(YAML_Key_Value)) key_values), + list(Var(List(YAML_Value)) values). public type YAML_Document: yaml_document(Var(List(YAML_Node)) nodes). -- libgit2 0.21.4