Creating MF_1_6 branch

Cédric RICARD
1 parent 22453804
Showing 6 changed files with 154 additions and 138 deletions Show diff stats
database/db_utils.anubis
database/migration.anubis
database/migration_common.anubis
database/settings.anubis
mail/smtp_client.anubis
web/CXM_making_a_web_site.anubis
@@ -10,7 +10,7 @@
 read tools/basis.anubis
 read system/logger.anubis
 read system/string.anubis
-read data_base/db_tools.anubis
+read data_base/sqlite.anubis
 read calexium_lib/net_services_protocols/logger_service.anubis
@@ -33,10 +33,10 @@ define One
   else
     unique.
-public define Result(DbError, One)
+public define Result(SQLite3Error, One)
   sql_transaction
   (
-    Database db, 
+    SQLite3DataBase db, 
     String          sql_command,
     String          message
   ) =
@@ -45,15 +45,14 @@ public define Result(DbError, One)
   if db_do_transaction(
         db,
         (One _) |->
-          if db_query(db, sql_command) is
+          if sql_query(db, sql_command) is
           {
             error(err)  then logError("DB", db_error(err,message));failure  //error in the SQL request
-            ok(_,cursor,_)  then success(unique)
+            ok(cursor)  then success(unique)
           }, 
-        success( (DbError err) |-> logError("DB", db_error(err,message))),
+        success( (SQLite3Error err) |-> logError("DB", db_error(err,message))),
         60000,   // max 60s
-        100,      // retry every 100 ms 
-        (DbError _) |-> false
+        100      // retry every 100 ms 
       ) is
   {
     error(err_and_result) then 
@@ -66,19 +65,19 @@ public define Result(DbError, One)
       ok(unique)
   }.
+ 
-
-public define DbQueryResult
+public define SQLite3QueryResult
   sql_query_timeout
   (
-    Database      db,         //database handle
-    String        sql_query,  //sql query itself
-    List(DbBind)  initial_bindings,
-    String        msg         //message to be shown if an error occure
+    SQLite3DataBase   db,         //database handle
+    String            sql_query,  //sql query itself
+    List(SQLite3Bind) initial_bindings,
+    String            msg         //message to be shown if an error occure
   ) =
   //we try with 30 sec of timeout
   with t0 = unow,
-  if sql_query_timeout(db, sql_query, initial_bindings, 60, 100, (DbError _) |-> false) is 
+  if sql_query_timeout(db, sql_query, initial_bindings, 60, 100) is 
   {
     error(sql_error)  then  logError("DB", db_error(sql_error,msg)); 
       __logLongQueries(t0, sql_query);
@@ -89,16 +88,16 @@ public define DbQueryResult
   }.
 // deprecated. Use one of the previous ones.
-public define Maybe(SQLite3HeadersOrRow -> DbRow)
+public define Maybe(SQLite3HeadersOrRow -> SQLite3Row)
   sql_query_timeout
   (
-    Database db,         //database handle
+    SQLite3DataBase db,         //database handle
     String          sql_query,  //sql query itself
     String          msg         //message to be shown if an error occure
   ) =
   //we try with 30 sec of timeout
   with t0 = unow,
-  if sql_query_timeout(db, sql_query, [], 60, 100, (DbError _) |-> false) is 
+  if sql_query_timeout(db, sql_query, [], 60, 100) is 
   {
     error(sql_error)  then  logError("DB", db_error(sql_error,msg));
       __logLongQueries(t0, sql_query);
@@ -108,7 +107,7 @@ public define Maybe(SQLite3HeadersOrRow -&gt; DbRow)
       success((SQLite3HeadersOrRow h_or_r) |-> if h_or_r is
                                                {
                                                   headers then 
-                                                    with cols = map((String c) |-> (DbDatum)db_text(c), headers(unique)),
+                                                    with cols = map((String c) |-> (SQLite3Datum)text(c), headers(unique)),
                                                     row((Int n) |-> force(nth(n, cols), no_such_column)),
                                                   next_row then cursor(unique)
                                                })
@@ -119,8 +118,8 @@ public define Maybe(SQLite3HeadersOrRow -&gt; DbRow)
 public define List(String) 
   db_get_string_list
   ( 
-    One -> DbRow    table_cursor,
-    List(String)    so_far
+    One -> SQLite3Row table_cursor,
+    List(String)      so_far
   ) =
   if table_cursor(unique) is
   {
@@ -135,7 +134,7 @@ public define List(String)
 public define List(String) 
   db_get_string_list
   ( 
-    SQLite3HeadersOrRow -> DbRow      table_cursor,
+    SQLite3HeadersOrRow -> SQLite3Row table_cursor,
     List(String)                      so_far
   ) =
   if table_cursor(next_row) is
@@ -147,11 +146,10 @@ public define List(String)
       db_get_string_list(table_cursor, [s . so_far])
  }.
-// Old sqlite3 API
 public define List(Int) 
   db_get_integer_list
   ( 
-    SQLite3HeadersOrRow -> DbRow table_cursor,
+    SQLite3HeadersOrRow -> SQLite3Row table_cursor,
     List(Int)                       so_far
   ) =
   if table_cursor(next_row) is
@@ -167,7 +165,7 @@ public define List(Int)
 public define List(Int) 
   db_get_integer_list
   ( 
-    One -> DbRow   table_cursor,
+    One -> SQLite3Row   table_cursor,
     List(Int)           so_far
   ) =
   if table_cursor(unique) is
@@ -183,37 +181,4 @@ public define List(Int)
-// -- Migration HELPERS ---------------
-  
-  public define DbBind
-    bind_String
-    (
-      String name,
-      String value
-    ) =
-    db_bind(force(sub_string(name, 1, length(name) - 1), name), db_text(value)).
-    
-  public define DbBind
-    bind_ByteArray
-    (
-      String name,
-      ByteArray value
-    ) =
-    db_bind(force(sub_string(name, 1, length(name) - 1), name), db_blob(value)).
-    
-  public define DbBind
-    bind_Int
-    (
-      String name,
-      Int value
-    ) =
-    db_bind(force(sub_string(name, 1, length(name) - 1), name), db_integer(value)).
-    
-  public define DbBind
-    bind_NULL
-    (
-      String name,
-    ) =
-    db_bind(force(sub_string(name, 1, length(name) - 1), name), null).
-    
-          
+
@@ -10,15 +10,15 @@ read tools/basis.anubis
 read system/files.anubis
 read system/logger.anubis
 read system/string.anubis
-read data_base/db_tools.anubis
+read data_base/alter_table.anubis
+read data_base/sqlite.anubis
 read calexium_lib/database/db_utils.anubis
-read calexium_lib/database/alter_table.anubis
 read calexium_lib/net_services_protocols/logger_service.anubis
  read mf_constants.anubis
- read settings.anubis
+read settings.anubis
 read migration_common.anubis
  read tools/mf_loggers.anubis
@@ -60,27 +60,44 @@ public define Int
                   },
   compare(split(version1, '.'), split(version2, '.')).
+define String
+  get_version_settings
+  (
+    SQLite3DataBase db
+  )= 
+  if is_table_exists(db, "settings") then
+     select_settings(db, "MF_version", "0.0.0.0")
+   else "0.0.0.0".
+
+public define One
+  update_version_settings
+  (
+    SQLite3DataBase db,
+    String          version
+  )= 
+  update_settings(db, "MF_version", version).
+  
 public define Maybe(One)
   backup_database
   (
     Logger        logger,
-//    String        current_version,
+    String        current_version,
     String        db_path,
     String        main_db_name,
     List(String)  other_db_names,
   )
   =
-//  with should_copy = if file_exists(db_path + main_db_name) then
-//                      if sqlite3_open(db_path + main_db_name) is
-//                      {
-//                        error(sql_error)        then false, // maybe not created yet
-//                        ok(Database db)  then 
-//                          with ver = get_version_settings(db),
-//                          ver != current_version
-//                      }
-//                     else false,
+  with should_copy = if file_exists(db_path + main_db_name) then
+                      if sqlite3_open(db_path + main_db_name) is
+                      {
+                        error(sql_error)        then false, // maybe not created yet
+                        ok(SQLite3DataBase db)  then 
+                          with ver = get_version_settings(db),
+                          ver != current_version
+                      }
+                     else false,
-  //if should_copy then
+  if should_copy then
     with  dir_save = db_path + dir_save_database, 
     if (Maybe(String))make_directory(dir_save) is
     {
@@ -111,7 +128,7 @@ public define Maybe(One)
                           },
         copy_db([main_db_name . other_db_names])
     }
-  //else success(unique)
+  else success(unique)
   .
@@ -121,9 +138,9 @@ public define Maybe(One)
 public type Migration:
   migration(String version,
-            (Database, Logger) -> Maybe(One)   migrate_to,
-            (Database, Logger) -> Maybe(One)   create_tables,
-            (Database, Logger) -> Maybe(One)   create_indexes,
+            (SQLite3DataBase, Logger) -> Maybe(One)   migrate_to,
+            (SQLite3DataBase, Logger) -> Maybe(One)   create_tables,
+            (SQLite3DataBase, Logger) -> Maybe(One)   create_indexes,
            ).
@@ -131,13 +148,12 @@ public type Migration:
 define Maybe(One)  
   do_migration_loop
   (
-    Database   db,
+    SQLite3DataBase   db,
     Logger            logger,
     String            current_version,
     String            db_ver,
     List(Migration)   migrations,
-    Maybe(Migration)  need_to_create_table,          // success in case of new DB or when no migration have been done
-    (Database, String) -> One  update_db_version,
+    Maybe(Migration)  need_to_create_table          // success in case of new DB or when no migration have been done
   ) =
   if migrations is
   {
@@ -145,13 +161,13 @@ define Maybe(One)
       if need_to_create_table is
       {
         failure then
-          update_db_version(db, current_version);
+          update_version_settings(db, current_version);
           success(unique),
         success(last) then
           if last is migration(_, _, create_tables, create_indexes) then
           if create_tables(db, logger) is success(_) then
             if create_indexes(db, logger) is success(_) then
-              update_db_version(db, current_version);
+              update_version_settings(db, current_version);
               success(unique)
             else
               failure
@@ -165,9 +181,9 @@ define Maybe(One)
         if migrate_to(db, logger) is failure then 
           failure
         else
-          do_migration_loop(db, logger, current_version, version, t, failure, update_db_version)
+          do_migration_loop(db, logger, current_version, version, t, failure)
       else 
-        do_migration_loop(db, logger, current_version, db_ver, t, success(h), update_db_version),
+        do_migration_loop(db, logger, current_version, db_ver, t, success(h)),
   }.
@@ -175,26 +191,24 @@ define Maybe(One)
 public define Maybe(One)  
   do_migration
   (
-    Database                    db,
-    Logger                      logger,
-    String                      current_version,
-    (Database) -> String       get_db_version,
-    (Database, String) -> One  update_db_version,
-    List(Migration)             all_migrations,
+    SQLite3DataBase         db,
+    Logger                  logger,
+    String                  current_version,
+    List(Migration)         all_migrations,
   )=
   if is_empty_database(db) then  // new empty database
     logInfo(logger, "No database found. Creating a new one...");
-    do_migration_loop(db, logger, current_version, "", [], last(all_migrations), update_db_version)
+    do_migration_loop(db, logger, current_version, "", [], last(all_migrations))
   else
   (
-    with db_version = get_db_version(db),
-    do_migration_loop(db, logger, current_version, db_version, all_migrations, failure, update_db_version)
+    with db_version = get_version_settings(db),
+    do_migration_loop(db, logger, current_version, db_version, all_migrations, failure)
   ).
 public define Maybe(One)  
   do_create_indexes
   (
-    Database         db,
+    SQLite3DataBase         db,
     Logger                  logger,
     String                  current_version,
     List(Migration)         all_migrations,
@@ -8,9 +8,9 @@
 read tools/basis.anubis
 read system/logger.anubis
-read data_base/db_tools.anubis
-read alter_table.anubis
-read sqlite_foreign_key.anubis
+read data_base/sqlite.anubis
+read data_base/alter_table.anubis
+read data_base/sqlite_foreign_key.anubis
 read calexium_lib/database/db_utils.anubis
 read calexium_lib/net_services_protocols/logger_service.anubis
@@ -27,11 +27,11 @@ public type DbTable:
 public define Bool
   is_table_exists
   (
-    Database db,
+    SQLite3DataBase db,
     String          dbName,
     String          table_name
   )=
-  if sql_query_timeout(db, "SELECT * FROM " + dbName + ".sqlite_master WHERE type ='table' AND name = :table_name", [db_bind("table_name", db_text(table_name))], "is_table_exists("+table_name+")") is
+  if sql_query_timeout(db, "SELECT * FROM " + dbName + ".sqlite_master WHERE type ='table' AND name = @table_name", [bind_String("@table_name", table_name)], "is_table_exists("+table_name+")") is
   {
     error(_)         then false,  //error in the SQL request
     ok(_, cursor, _) then
@@ -46,7 +46,7 @@ public define Bool
 public define Bool
   is_table_exists
   (
-    Database db,
+    SQLite3DataBase db,
     String          table_name
   )=
   is_table_exists(db, "main", table_name).
@@ -54,7 +54,7 @@ public define Bool
 public define Bool
   is_empty_database
   (
-    Database db,
+    SQLite3DataBase db,
     String          dbName,
   )=
   if sql_query_timeout(db, "SELECT * FROM " + dbName + ".sqlite_master", [], "is_empty_database("+dbName+")") is
@@ -72,21 +72,21 @@ public define Bool
 public define Bool
   is_empty_database
   (
-    Database db,
+    SQLite3DataBase db,
   )=
   is_empty_database(db, "main").
 public define Maybe(One)
   drop_all_triggers
   (
-    Database db,
+    SQLite3DataBase db,
     Logger          log
   ) =
-  if db_query(db, "SELECT name FROM sqlite_master WHERE type = 'trigger'", []) is
+  if sqlite3_query(db, "SELECT name FROM sqlite_master WHERE type = 'trigger'", []) is
   {
     error(err) then logError(log, db_error(err, "drop_all_triggers")); failure,
     ok(_, cursor, _) then 
-      with drop_trigger = (String name) |->   if db_query(db, "DROP TRIGGER " + name, []) is
+      with drop_trigger = (String name) |->   if sqlite3_query(db, "DROP TRIGGER " + name, []) is
                                               {
                                                 error(err2) then logError(log, db_error(err2, "Failed to drop trigger '" + name + "'")),
                                                 ok(_, cursor, _) then unique
@@ -98,7 +98,7 @@ public define Maybe(One)
 public define Maybe(One)
   create_tables
   (
-    Database       db,
+    SQLite3DataBase       db,
     Logger                logger,
     List(DbTable)         tables,
     String                version_string
@@ -134,7 +134,7 @@ public define Maybe(One)
 public define Maybe(One)
   create_indexes
   (
-    Database db,
+    SQLite3DataBase db,
     Logger          logger,
     List(String)    indexes,
     String          version_string
@@ -155,7 +155,7 @@ public define Maybe(One)
 public define One
   make_foreign_key
   (
-    Database db,
+    SQLite3DataBase db,
     String table_name,
     String field_name,
     String foreign_table_name,
@@ -175,7 +175,7 @@ public define One
 public define Maybe(One)
   make_foreign_keys
   (
-    Database db,
+    SQLite3DataBase db,
     Logger          logger,
     List((String, String, String, String, FK_Null, FK_Cascade)) relations,
     String          version_string
@@ -7,6 +7,7 @@
 read tools/basis.anubis
 read data_base/db_tools.anubis
+read data_base/sqlite.anubis
 read calexium_lib/database/db_utils.anubis
@@ -77,17 +78,47 @@ public define Bool
         success(v) then v!=0
       }
   }.
+
+public define One
+  update_settings
+  (
+    SQLite3DataBase db,
+    String          var,
+    Bool            value
+  )= 
+  with binds = (List(SQLite3Bind))[bind_String("@name", var), bind_Int("@value", if value then 1 else 0)],
+  if select_settings(db, var) is success(_) then 
+    forget(sql_query_timeout(db, "UPDATE settings SET var_value = @value WHERE var_name = @name;", binds, "update_settings"))
+  else
+    forget(sql_query_timeout(db, "INSERT INTO settings (var_name, var_value) VALUES (@name, @value)", binds, "update_settings"))
+  .
 public define One
   update_settings
   (
     Database db,
     String          var,
+    Int             value
+  )= 
+  with binds = (List(SQLite3Bind))[bind_String("@name", var), bind_Int("@value", value)],
+  if select_settings(db, var) is success(_) then 
+    forget(sql_query_timeout(db, "UPDATE settings SET var_value = @value WHERE var_name = @name;", binds, "update_settings"))
+  else
+    forget(sql_query_timeout(db, "INSERT INTO settings (var_name, var_value) VALUES (@name, @value)", binds, "update_settings"))
+  .
+
+       
+public define One
+  update_settings
+  (
+    SQLite3DataBase db,
+    String          var,
     String          value
   )= 
+  with binds = (List(SQLite3Bind))[bind_String("@name", var), bind_String("@value", value)],
   if select_settings(db, var) is success(_) then 
-    forget(sql_query_timeout(db, "UPDATE settings SET var_value = @value WHERE var_name = @name;", [bind_String("@name", var), bind_String("@value", value)], "update_settings"))
+    forget(sql_query_timeout(db, "UPDATE settings SET var_value = @value WHERE var_name = @name;", binds, "update_settings"))
   else
-    forget(sql_query_timeout(db, "INSERT INTO settings (var_name, var_value) VALUES (@name, @value)", [bind_String("@name", var), bind_String("@value", value)], "update_settings"))
+    forget(sql_query_timeout(db, "INSERT INTO settings (var_name, var_value) VALUES (@name, @value)", binds, "update_settings"))
   .
@@ -406,9 +406,9 @@ define Maybe(One)
   {
     failure           then failure,
     success(nb_write) then
-      if now - buffer_start_time > 120 then
+      if now - buffer_start_time > 3600 then
         // security to avoid queue blocking
-        logger(logError, "sm_flush: TIMEOUT sending a " + length(buffer) + " byte-length buffer (taking more than 2 minutes). SendMail canceled.");
+        logger(logError, "sm_flush: TIMEOUT sending a 64kb buffer (taking more than 1 hour). SendMail canceled.");
         failure
       else
         with buffer_size = length(buffer),      
@@ -427,38 +427,44 @@ define SendContentResult
     SmtpClientSession   session,
     Int       start_time,
     Int       so_far,
-    Int       last_reply_check,
     (Int) -> One  progress_report,
     (LogLevel, String) -> One logger
   ) =
-  if last_reply_check + 60 < now then
-    if receive_reply(weaken(target), [], session.enhanced_status, 0, logger) is
-    {
-      error then copy_error(so_far),
-      timeout then
-        sm_copy_Data_IO_to_Stream(source, target, session, start_time, so_far, now, progress_report, logger)
-      no_auth_method then copy_error(so_far), // impossible
-      bad_reply then copy_error(so_far),
-      reply(code, status, lines) then
-        smtp_reply(reply(code, status, lines))
-    }
-  else
-      if read_line(source, 1024) is
+  if receive_reply(weaken(target), [], session.enhanced_status, 0, logger) is
+  {
+    error then copy_error(so_far),
+    timeout then
+      if read_bytes(source, 65536) is
       {
-        error           then logger(logError, "send_content: failed to read input data_io"); copy_error(so_far),
-        timeout         then logger(logError, "send_content: timeout reading input data_io"); copy_error(so_far), 
-        eof             then copy_ok(so_far),
-        ok(line)        then 
-          with buffer = to_byte_array((if nth(0, line) is success(char) then if char = '.' then "." + line else line else line)),
+        failure         then logger(logError, "send_content: failed to read input data_io"); copy_error(so_far),
+        time_out        then logger(logError, "send_content: timeout reading input data_io"); copy_error(so_far), 
+        success(buffer) then 
           if sm_flush( buffer, weaken(target), now, logger ) is
           {
             failure     then copy_error(so_far),
             success(_)  then 
-              with len = length(line) + 2, // 2 is for the CRLF
-              progress_report(so_far + len);
-              sm_copy_Data_IO_to_Stream(source, target, session, start_time, so_far + len, last_reply_check, progress_report, logger)
+              progress_report(so_far + 65536);
+              sm_copy_Data_IO_to_Stream(source, target, session, start_time, so_far + 65536, progress_report, logger)
           },
-      }.
+          
+        truncated(buffer)       then
+          with len = length(buffer),
+          if len = 0 then
+            copy_ok(so_far)
+          else 
+            if sm_flush( buffer, weaken(target), now, logger ) is
+            {
+              failure     then copy_error(so_far),
+              success(_)  then 
+                progress_report(so_far + len);
+                copy_ok(so_far + len)
+            }
+      },
+    no_auth_method then copy_error(so_far), // impossible
+    bad_reply then copy_error(so_far),
+    reply(code, status, lines) then
+      smtp_reply(reply(code, status, lines))
+  }.
 define SendContentResult
   sm_copy_Data_IO_List_to_Stream
@@ -476,7 +482,7 @@ define SendContentResult
     []        then copy_ok(so_far),
     [ h . t ]  then
       if rewind(h)(unique) then
-        with result = sm_copy_Data_IO_to_Stream(h, target, session, start_time, 0, now, progress_report, logger),
+        with result = sm_copy_Data_IO_to_Stream(h, target, session, start_time, 0, progress_report, logger),
         if result is copy_ok(written) then 
           sm_copy_Data_IO_List_to_Stream(t, target, session, start_time, so_far + written, progress_report, logger)
         else
@@ -2267,7 +2267,7 @@ public define Web_Site
       //
       // make required directories (if needed)
       //
-      with //web_sites_directory = (String) make_directory(my_anubis_directory+"/web_sites"),  
+      with web_sites_directory = (String) make_directory(my_anubis_directory+"/web_sites"),  
                 base_directory = (String) make_directory(site_directory),
               // state_directory = make_directory(site_directory+"/states"), 
           forget((String)make_directory(site_directory+"/public"));