improve "eol" function

improve "print_yaml_data" function to print lists add list parsing (WIP, fail in some conditions) fix parsing stuff

improve "eol" function
improve "print_yaml_data" function to print lists add list parsing (WIP, fail in some conditions) fix parsing stuff
Julien Verneuil
1 parent bc99be1d
Showing 2 changed files with 185 additions and 34 deletions Show diff stats
file_processor_io/yaml/yaml.anubis
file_processor_io/yaml/yaml_parser.anubis
@@ -12,6 +12,8 @@ read tools/streams.anubis
 transmit yaml_type.anubis
 read yaml_parser.anubis
+define One print_yaml_map (String indent_str, List(YAML_Key_Value) so_far).
+  
 define One
   print_yaml_value
   (
@@ -23,6 +25,49 @@ define One
   if value = "" then unique else println(indent_str + "   - Value: " + value).
 define One
+  print_yaml_list
+  (
+    String           indent_str,
+    List(YAML_Value) so_far
+  )=
+  if so_far is
+  {
+    [ ]     then
+      unique,
+      
+    [l . t] then
+      if l is
+      {
+        null then
+          print_yaml_value(indent_str + "  ", "null", ""),
+        
+        string(str) then
+          print_yaml_value(indent_str + "  ", "String", *str),
+          
+        int(integer) then
+          print_yaml_value(indent_str + "  ", "Integer", "" + *integer),
+          
+        bool(boolean) then
+          with value_str = 
+            if *boolean then
+              "true"
+            else
+              "false",
+           print_yaml_value(indent_str + "  ", "Boolean", "" + value_str),
+        
+        map(key_values) then
+          print_yaml_map(indent_str + "  ", *key_values),
+          
+        list(yaml_values) then
+          println(indent_str + "   - List: [");
+          print_yaml_list(indent_str + " ", *yaml_values);
+          println(indent_str + "     ]"),
+      };
+      (if t is [] then unique else println(""));
+      print_yaml_list(indent_str, t)
+  }.
+
+define One
   print_yaml_map
   (
     String               indent_str,
@@ -57,7 +102,9 @@ define One
           print_yaml_map(indent_str + "  ", *key_values),
         list(yaml_values) then
-          println("list content (todo)")
+          println(indent_str + "   - List: [");
+          print_yaml_list(indent_str + "  ", *yaml_values);
+          println(indent_str + "           ]"),
       };
       (if t is [] then unique else println(""));
@@ -81,7 +128,8 @@ define One
           print_yaml_map("", reverse(*key_values)),
         list(values) then
-          println("list (todo))")
+          println("list)");
+          print_yaml_list("", *values)
       };
       print_yaml_nodes(t)
@@ -244,5 +292,6 @@ global define One
   (
     List(String) dummy
   )=
-  print_yaml_data(load(my_anubis_directory+"/IDPS/suricata/test.yaml")).
+  with yaml_parsing_result = load(my_anubis_directory+"/IDPS/suricata/test.yaml"),
+  print_yaml_data(yaml_parsing_result).
@@ -41,9 +41,19 @@ define Maybe(YAML_Parser_Node)
     List(Word8)           so_far
   ).
-define Bool eol(Word8 c) =
+define Bool eol(Stream s, Word8 c) =
   if      c = 10 then true // LF
-  else if c = 13 then true // CR
+  else if c = 13 then // CR
+    if read_byte(s) is // for CR LF
+    {
+      failure then true,
+      success(next_char) then
+        if next_char = 10 then
+          true
+        else
+          unput_byte(next_char, s);
+          true
+    }
   else false.
 define One
@@ -55,7 +65,7 @@ define One
   {
     failure    then unique,
     success(c) then
-      if eol(c) then
+      if eol(s, c) then
         unique
       else
         skip_to_next_line(s)
@@ -85,7 +95,7 @@ define One
   {
     failure    then unique,
     success(c) then
-      if eol(c) then
+      if eol(s, c) then
         skip_to_next_node(s)
       else if c = ' ' then
         skip_to_next_node(s)
@@ -152,10 +162,11 @@ define Maybe(List(YAML_Key_Value))
   skip_spaces(s);
   skip_to_next_node(s);
   with current_indent = current_column(s),
-  //println("current_indent: " + current_indent);
-  //println("block_indent: " + block_indent);
+  println("reading block map entries");
+  println("current_indent: " + current_indent);
+  println("block_indent: " + block_indent);
   if current_indent = block_indent then
-    if _read_node(s, yaml_p_info(parent_indent, current_indent, level), none, []) is
+    if _read_node(s, yaml_p_info(block_indent, current_indent, level), none, []) is
     {
       failure then
         success([]),
@@ -174,10 +185,10 @@ define Maybe(List(YAML_Key_Value))
             }
           list(yaml_lv) then
-            failure,
+            success(so_far),
           value(yaml_value) then
-            failure
+            success(so_far)
         }
     }
   else
@@ -186,10 +197,45 @@ define Maybe(List(YAML_Key_Value))
 define Maybe(List(YAML_Value))
   read_block_list
   (
-    Stream s,
+    Stream           s,
+    YAML_Parser_Info p_info,
+    Int              block_indent,
     List(YAML_Value) so_far
   )=
-  failure.
+  since p_info is yaml_p_info(parent_indent, context_indent, level),
+  skip_spaces(s);
+  skip_to_next_node(s);
+  with current_indent = current_column(s),
+  //println("current_indent: " + current_indent);
+  //println("block_indent: " + block_indent);
+  if current_indent = block_indent then
+    if _read_node(s, yaml_p_info(block_indent, current_indent, level), none, []) is
+    {
+      failure then
+        success([]),
+        
+      success(node_type) then
+        if node_type is
+        {
+          map(yaml_kvl) then
+            failure,
+            
+          list(yaml_lv) then
+            if yaml_lv is
+            {
+              [ ] then
+                failure,
+                
+              [h . _] then
+                read_block_list(s, p_info, block_indent, [ h . so_far ]),
+            }
+            
+          value(yaml_value) then
+            failure
+        }
+    }
+  else
+    success(so_far).
 define Maybe(YAML_Value)
   read_value
@@ -208,7 +254,7 @@ define Maybe(YAML_Value)
   {
     failure    then success(null),
     success(c) then
-      if eol(c) then // \n
+      if eol(s, c) then // \n
         if ctx_info is
         {
           none then
@@ -240,8 +286,8 @@ define Maybe(YAML_Value)
             read_value(s, p_info, line, [c . so_far]),
           block_start then
+            println("block_start");
             unput_byte(c, s);
-            //println("block_start");
             skip_to_next_node(s);
             with current_indent = current_column(s),
             if current_indent =< parent_indent then
@@ -253,7 +299,8 @@ define Maybe(YAML_Value)
             unput_byte(c, s);
             //println("block level " + level);
             skip_to_next_node(s);
-            if _read_node(s, p_info, none, []) is
+            with current_indent = current_column(s),
+            if _read_node(s, yaml_p_info(current_indent, current_indent, level), none, []) is
             {
               failure then
                 success(null),
@@ -262,31 +309,80 @@ define Maybe(YAML_Value)
                 if node_type is
                 {
                   map(yaml_kvl) then
-                    if read_block_maps(s, p_info, context_indent, yaml_kvl) is
+                    println("map node");
+                    if read_block_maps(s, p_info, current_indent, yaml_kvl) is
                     {
                       failure then
-                        success(map(var(yaml_kvl))),
+                        success(map(var(reverse(yaml_kvl)))),
                       success(list_kv) then
                         success(map(var(reverse(list_kv))))
                     }
                   list(yaml_lv) then
-                    if read_block_list(s, yaml_lv) is
+                    println("list node");
+                    if read_block_list(s, p_info, current_indent, yaml_lv) is
                     {
                       failure then
                         success(list(var(yaml_lv))),
                       success(list_v) then
-                        success(list(var(list_v)))
+                        success(list(var(reverse(list_v))))
                     }
                   value(yaml_value) then
+                    //println("value node");
                     success(yaml_value)
                 }
             }
         }
   }.
+  
+define Maybe(List(YAML_Value))
+  read_values
+  (
+    Word8            c,
+    Stream           s,
+    YAML_Parser_Info p_info,
+    List(YAML_Value) so_far
+  )=
+  if c = '-' then
+    if read_byte(s) is
+    {
+      failure then
+        success(reverse(so_far)),
+      
+      success(next_char) then
+        if next_char = ' ' then
+          skip_spaces(s);
+            println("found a list");
+          since p_info is yaml_p_info(parent_indent, context_indent, level),
+          with current_indent = current_column(s),
+          if (current_indent-1) =< parent_indent then
+            success(so_far)
+          else
+            if read_value(s, yaml_p_info((current_indent-1), current_indent, level), block, []) is
+            {
+              failure then
+                success(so_far),
+                
+              success(v) then
+                //skip_to_next_node(s);
+                if read_byte(s) is
+                {
+                  failure then
+                    success(so_far),
+                  
+                  success(next_next_char) then
+                    read_values(next_next_char, s, p_info, [v . so_far])
+                }
+            }
+        else
+          unput_byte(next_char, s);
+          success(so_far),
+    }
+  else
+    success(so_far).
 define Maybe(YAML_Parser_Node)
   _read_node
@@ -296,23 +392,29 @@ define Maybe(YAML_Parser_Node)
     YAML_Parser_Node_Info node_info,
     List(Word8)           so_far
   )=
-  skip_spaces(s);
   if read_byte(s) is
   {
-    failure       then failure,
+    failure    then failure,
     success(c) then
       if node_info is
       {
         none then
           if c = '-' then
-            todo//_read_list(s, p_info, in_double_quote, so_far)
+            if read_values(c, s, p_info, []) is
+            {
+              failure then
+                failure,
+                
+              success(lv) then
+                success(list(lv))
+            }
           else if c = '\"' then
             _read_node(s, p_info, in_double_quote, so_far)
           else if c = '\'' then
             _read_node(s, p_info, in_single_quote, so_far)
           else
             _read_node(s, p_info, unquoted, [ c . so_far ]),
-          
+             
         unquoted then
           if c = ':' then
             if read_byte(s) is
@@ -322,36 +424,35 @@ define Maybe(YAML_Parser_Node)
                 success(map([key_value(key_name, var(null))])),
               success(next_char) then
-                if eol(next_char) then // \n
+                if eol(s, next_char) then // \n
                   unput_byte(next_char, s);
                   with key_name = implode(reverse(so_far)),
-                  //println("key found [LF] \n name: " + key_name);
+                  println("key found [LF] \n name: " + key_name);
                   skip_spaces(s);
                   with value = read_value(s, p_info, none, []),
                   if value is
                   {
-                    failure then failure,
+                    failure    then failure,
                     success(v) then
                       success(map([key_value(key_name, var(v))]))
                   }
                 else if next_char = ' ' then
                   with key_name = implode(reverse(so_far)),
-                  //println("key found [ ]\n name: " + key_name);
+                  println("key found [ ]\n name: " + key_name);
                   skip_spaces(s);
                   with value = read_value(s, p_info, none, []),
                   if value is
                   {
-                    failure then failure,
+                    failure    then failure,
                     success(v) then
                       success(map([key_value(key_name, var(v))]))
                   }
-                  
                 else
                   todo
                   //_read_node(s, p_info, unquoted, [next_char . [c . so_far]])
             }
-          else if eol(c) then
+          else if eol(s, c) then
             with str = implode(reverse(so_far)),
             //println("value (str): " + str);
             success(value(get_yaml_value(str)))
@@ -385,7 +486,7 @@ public define Maybe(YAML_Document)
       success(c)  then
         //check for comment
-        if eol(c) then
+        if eol(s, c) then
           get_yaml_document_content(s, doc)
         else if c = '#' then
         //println("skipping comment ["+line+"]");
@@ -401,8 +502,9 @@ public define Maybe(YAML_Document)
           if mb_node is 
           {
             failure then  
-              println("Can't get map entries");
+              println("Can't parse YAML node at column " + current_column(s) + " line " + current_line(s));
               get_yaml_document_content(s, doc),
+              
             success(node_type) then
               since doc is yaml_document(nodes),
               if node_type is