From b2882af8a0cef378fc65d5f41b891fb3724871c1 Mon Sep 17 00:00:00 2001 From: Cedric RICARD Date: Tue, 21 Sep 2010 21:23:20 +0000 Subject: [PATCH] Creating MF_1_6 branch --- database/db_utils.anubis | 83 ++++++++++++++++++++++++----------------------------------------------------------- database/migration.anubis | 84 +++++++++++++++++++++++++++++++++++++++++++++++++----------------------------------- database/migration_common.anubis | 30 +++++++++++++++--------------- database/settings.anubis | 35 +++++++++++++++++++++++++++++++++-- mail/smtp_client.anubis | 58 ++++++++++++++++++++++++++++++++-------------------------- web/CXM_making_a_web_site.anubis | 2 +- 6 files changed, 154 insertions(+), 138 deletions(-) diff --git a/database/db_utils.anubis b/database/db_utils.anubis index 72ed529..b1f04ae 100644 --- a/database/db_utils.anubis +++ b/database/db_utils.anubis @@ -10,7 +10,7 @@ read tools/basis.anubis read system/logger.anubis read system/string.anubis -read data_base/db_tools.anubis +read data_base/sqlite.anubis read calexium_lib/net_services_protocols/logger_service.anubis @@ -33,10 +33,10 @@ define One else unique. -public define Result(DbError, One) +public define Result(SQLite3Error, One) sql_transaction ( - Database db, + SQLite3DataBase db, String sql_command, String message ) = @@ -45,15 +45,14 @@ public define Result(DbError, One) if db_do_transaction( db, (One _) |-> - if db_query(db, sql_command) is + if sql_query(db, sql_command) is { error(err) then logError("DB", db_error(err,message));failure //error in the SQL request - ok(_,cursor,_) then success(unique) + ok(cursor) then success(unique) }, - success( (DbError err) |-> logError("DB", db_error(err,message))), + success( (SQLite3Error err) |-> logError("DB", db_error(err,message))), 60000, // max 60s - 100, // retry every 100 ms - (DbError _) |-> false + 100 // retry every 100 ms ) is { error(err_and_result) then @@ -66,19 +65,19 @@ public define Result(DbError, One) ok(unique) }. + - -public define DbQueryResult +public define SQLite3QueryResult sql_query_timeout ( - Database db, //database handle - String sql_query, //sql query itself - List(DbBind) initial_bindings, - String msg //message to be shown if an error occure + SQLite3DataBase db, //database handle + String sql_query, //sql query itself + List(SQLite3Bind) initial_bindings, + String msg //message to be shown if an error occure ) = //we try with 30 sec of timeout with t0 = unow, - if sql_query_timeout(db, sql_query, initial_bindings, 60, 100, (DbError _) |-> false) is + if sql_query_timeout(db, sql_query, initial_bindings, 60, 100) is { error(sql_error) then logError("DB", db_error(sql_error,msg)); __logLongQueries(t0, sql_query); @@ -89,16 +88,16 @@ public define DbQueryResult }. // deprecated. Use one of the previous ones. -public define Maybe(SQLite3HeadersOrRow -> DbRow) +public define Maybe(SQLite3HeadersOrRow -> SQLite3Row) sql_query_timeout ( - Database db, //database handle + SQLite3DataBase db, //database handle String sql_query, //sql query itself String msg //message to be shown if an error occure ) = //we try with 30 sec of timeout with t0 = unow, - if sql_query_timeout(db, sql_query, [], 60, 100, (DbError _) |-> false) is + if sql_query_timeout(db, sql_query, [], 60, 100) is { error(sql_error) then logError("DB", db_error(sql_error,msg)); __logLongQueries(t0, sql_query); @@ -108,7 +107,7 @@ public define Maybe(SQLite3HeadersOrRow -> DbRow) success((SQLite3HeadersOrRow h_or_r) |-> if h_or_r is { headers then - with cols = map((String c) |-> (DbDatum)db_text(c), headers(unique)), + with cols = map((String c) |-> (SQLite3Datum)text(c), headers(unique)), row((Int n) |-> force(nth(n, cols), no_such_column)), next_row then cursor(unique) }) @@ -119,8 +118,8 @@ public define Maybe(SQLite3HeadersOrRow -> DbRow) public define List(String) db_get_string_list ( - One -> DbRow table_cursor, - List(String) so_far + One -> SQLite3Row table_cursor, + List(String) so_far ) = if table_cursor(unique) is { @@ -135,7 +134,7 @@ public define List(String) public define List(String) db_get_string_list ( - SQLite3HeadersOrRow -> DbRow table_cursor, + SQLite3HeadersOrRow -> SQLite3Row table_cursor, List(String) so_far ) = if table_cursor(next_row) is @@ -147,11 +146,10 @@ public define List(String) db_get_string_list(table_cursor, [s . so_far]) }. -// Old sqlite3 API public define List(Int) db_get_integer_list ( - SQLite3HeadersOrRow -> DbRow table_cursor, + SQLite3HeadersOrRow -> SQLite3Row table_cursor, List(Int) so_far ) = if table_cursor(next_row) is @@ -167,7 +165,7 @@ public define List(Int) public define List(Int) db_get_integer_list ( - One -> DbRow table_cursor, + One -> SQLite3Row table_cursor, List(Int) so_far ) = if table_cursor(unique) is @@ -183,37 +181,4 @@ public define List(Int) -// -- Migration HELPERS --------------- - - public define DbBind - bind_String - ( - String name, - String value - ) = - db_bind(force(sub_string(name, 1, length(name) - 1), name), db_text(value)). - - public define DbBind - bind_ByteArray - ( - String name, - ByteArray value - ) = - db_bind(force(sub_string(name, 1, length(name) - 1), name), db_blob(value)). - - public define DbBind - bind_Int - ( - String name, - Int value - ) = - db_bind(force(sub_string(name, 1, length(name) - 1), name), db_integer(value)). - - public define DbBind - bind_NULL - ( - String name, - ) = - db_bind(force(sub_string(name, 1, length(name) - 1), name), null). - - + diff --git a/database/migration.anubis b/database/migration.anubis index f770208..597f9a8 100644 --- a/database/migration.anubis +++ b/database/migration.anubis @@ -10,15 +10,15 @@ read tools/basis.anubis read system/files.anubis read system/logger.anubis read system/string.anubis -read data_base/db_tools.anubis +read data_base/alter_table.anubis +read data_base/sqlite.anubis read calexium_lib/database/db_utils.anubis -read calexium_lib/database/alter_table.anubis read calexium_lib/net_services_protocols/logger_service.anubis read mf_constants.anubis - read settings.anubis +read settings.anubis read migration_common.anubis read tools/mf_loggers.anubis @@ -60,27 +60,44 @@ public define Int }, compare(split(version1, '.'), split(version2, '.')). +define String + get_version_settings + ( + SQLite3DataBase db + )= + if is_table_exists(db, "settings") then + select_settings(db, "MF_version", "0.0.0.0") + else "0.0.0.0". + +public define One + update_version_settings + ( + SQLite3DataBase db, + String version + )= + update_settings(db, "MF_version", version). + public define Maybe(One) backup_database ( Logger logger, -// String current_version, + String current_version, String db_path, String main_db_name, List(String) other_db_names, ) = -// with should_copy = if file_exists(db_path + main_db_name) then -// if sqlite3_open(db_path + main_db_name) is -// { -// error(sql_error) then false, // maybe not created yet -// ok(Database db) then -// with ver = get_version_settings(db), -// ver != current_version -// } -// else false, + with should_copy = if file_exists(db_path + main_db_name) then + if sqlite3_open(db_path + main_db_name) is + { + error(sql_error) then false, // maybe not created yet + ok(SQLite3DataBase db) then + with ver = get_version_settings(db), + ver != current_version + } + else false, - //if should_copy then + if should_copy then with dir_save = db_path + dir_save_database, if (Maybe(String))make_directory(dir_save) is { @@ -111,7 +128,7 @@ public define Maybe(One) }, copy_db([main_db_name . other_db_names]) } - //else success(unique) + else success(unique) . @@ -121,9 +138,9 @@ public define Maybe(One) public type Migration: migration(String version, - (Database, Logger) -> Maybe(One) migrate_to, - (Database, Logger) -> Maybe(One) create_tables, - (Database, Logger) -> Maybe(One) create_indexes, + (SQLite3DataBase, Logger) -> Maybe(One) migrate_to, + (SQLite3DataBase, Logger) -> Maybe(One) create_tables, + (SQLite3DataBase, Logger) -> Maybe(One) create_indexes, ). @@ -131,13 +148,12 @@ public type Migration: define Maybe(One) do_migration_loop ( - Database db, + SQLite3DataBase db, Logger logger, String current_version, String db_ver, List(Migration) migrations, - Maybe(Migration) need_to_create_table, // success in case of new DB or when no migration have been done - (Database, String) -> One update_db_version, + Maybe(Migration) need_to_create_table // success in case of new DB or when no migration have been done ) = if migrations is { @@ -145,13 +161,13 @@ define Maybe(One) if need_to_create_table is { failure then - update_db_version(db, current_version); + update_version_settings(db, current_version); success(unique), success(last) then if last is migration(_, _, create_tables, create_indexes) then if create_tables(db, logger) is success(_) then if create_indexes(db, logger) is success(_) then - update_db_version(db, current_version); + update_version_settings(db, current_version); success(unique) else failure @@ -165,9 +181,9 @@ define Maybe(One) if migrate_to(db, logger) is failure then failure else - do_migration_loop(db, logger, current_version, version, t, failure, update_db_version) + do_migration_loop(db, logger, current_version, version, t, failure) else - do_migration_loop(db, logger, current_version, db_ver, t, success(h), update_db_version), + do_migration_loop(db, logger, current_version, db_ver, t, success(h)), }. @@ -175,26 +191,24 @@ define Maybe(One) public define Maybe(One) do_migration ( - Database db, - Logger logger, - String current_version, - (Database) -> String get_db_version, - (Database, String) -> One update_db_version, - List(Migration) all_migrations, + SQLite3DataBase db, + Logger logger, + String current_version, + List(Migration) all_migrations, )= if is_empty_database(db) then // new empty database logInfo(logger, "No database found. Creating a new one..."); - do_migration_loop(db, logger, current_version, "", [], last(all_migrations), update_db_version) + do_migration_loop(db, logger, current_version, "", [], last(all_migrations)) else ( - with db_version = get_db_version(db), - do_migration_loop(db, logger, current_version, db_version, all_migrations, failure, update_db_version) + with db_version = get_version_settings(db), + do_migration_loop(db, logger, current_version, db_version, all_migrations, failure) ). public define Maybe(One) do_create_indexes ( - Database db, + SQLite3DataBase db, Logger logger, String current_version, List(Migration) all_migrations, diff --git a/database/migration_common.anubis b/database/migration_common.anubis index c9d2d3e..0cbda8b 100644 --- a/database/migration_common.anubis +++ b/database/migration_common.anubis @@ -8,9 +8,9 @@ read tools/basis.anubis read system/logger.anubis -read data_base/db_tools.anubis -read alter_table.anubis -read sqlite_foreign_key.anubis +read data_base/sqlite.anubis +read data_base/alter_table.anubis +read data_base/sqlite_foreign_key.anubis read calexium_lib/database/db_utils.anubis read calexium_lib/net_services_protocols/logger_service.anubis @@ -27,11 +27,11 @@ public type DbTable: public define Bool is_table_exists ( - Database db, + SQLite3DataBase db, String dbName, String table_name )= - if sql_query_timeout(db, "SELECT * FROM " + dbName + ".sqlite_master WHERE type ='table' AND name = :table_name", [db_bind("table_name", db_text(table_name))], "is_table_exists("+table_name+")") is + if sql_query_timeout(db, "SELECT * FROM " + dbName + ".sqlite_master WHERE type ='table' AND name = @table_name", [bind_String("@table_name", table_name)], "is_table_exists("+table_name+")") is { error(_) then false, //error in the SQL request ok(_, cursor, _) then @@ -46,7 +46,7 @@ public define Bool public define Bool is_table_exists ( - Database db, + SQLite3DataBase db, String table_name )= is_table_exists(db, "main", table_name). @@ -54,7 +54,7 @@ public define Bool public define Bool is_empty_database ( - Database db, + SQLite3DataBase db, String dbName, )= if sql_query_timeout(db, "SELECT * FROM " + dbName + ".sqlite_master", [], "is_empty_database("+dbName+")") is @@ -72,21 +72,21 @@ public define Bool public define Bool is_empty_database ( - Database db, + SQLite3DataBase db, )= is_empty_database(db, "main"). public define Maybe(One) drop_all_triggers ( - Database db, + SQLite3DataBase db, Logger log ) = - if db_query(db, "SELECT name FROM sqlite_master WHERE type = 'trigger'", []) is + if sqlite3_query(db, "SELECT name FROM sqlite_master WHERE type = 'trigger'", []) is { error(err) then logError(log, db_error(err, "drop_all_triggers")); failure, ok(_, cursor, _) then - with drop_trigger = (String name) |-> if db_query(db, "DROP TRIGGER " + name, []) is + with drop_trigger = (String name) |-> if sqlite3_query(db, "DROP TRIGGER " + name, []) is { error(err2) then logError(log, db_error(err2, "Failed to drop trigger '" + name + "'")), ok(_, cursor, _) then unique @@ -98,7 +98,7 @@ public define Maybe(One) public define Maybe(One) create_tables ( - Database db, + SQLite3DataBase db, Logger logger, List(DbTable) tables, String version_string @@ -134,7 +134,7 @@ public define Maybe(One) public define Maybe(One) create_indexes ( - Database db, + SQLite3DataBase db, Logger logger, List(String) indexes, String version_string @@ -155,7 +155,7 @@ public define Maybe(One) public define One make_foreign_key ( - Database db, + SQLite3DataBase db, String table_name, String field_name, String foreign_table_name, @@ -175,7 +175,7 @@ public define One public define Maybe(One) make_foreign_keys ( - Database db, + SQLite3DataBase db, Logger logger, List((String, String, String, String, FK_Null, FK_Cascade)) relations, String version_string diff --git a/database/settings.anubis b/database/settings.anubis index 5b8a4f9..41c42b8 100644 --- a/database/settings.anubis +++ b/database/settings.anubis @@ -7,6 +7,7 @@ read tools/basis.anubis read data_base/db_tools.anubis +read data_base/sqlite.anubis read calexium_lib/database/db_utils.anubis @@ -77,17 +78,47 @@ public define Bool success(v) then v!=0 } }. + +public define One + update_settings + ( + SQLite3DataBase db, + String var, + Bool value + )= + with binds = (List(SQLite3Bind))[bind_String("@name", var), bind_Int("@value", if value then 1 else 0)], + if select_settings(db, var) is success(_) then + forget(sql_query_timeout(db, "UPDATE settings SET var_value = @value WHERE var_name = @name;", binds, "update_settings")) + else + forget(sql_query_timeout(db, "INSERT INTO settings (var_name, var_value) VALUES (@name, @value)", binds, "update_settings")) + . public define One update_settings ( Database db, String var, + Int value + )= + with binds = (List(SQLite3Bind))[bind_String("@name", var), bind_Int("@value", value)], + if select_settings(db, var) is success(_) then + forget(sql_query_timeout(db, "UPDATE settings SET var_value = @value WHERE var_name = @name;", binds, "update_settings")) + else + forget(sql_query_timeout(db, "INSERT INTO settings (var_name, var_value) VALUES (@name, @value)", binds, "update_settings")) + . + + +public define One + update_settings + ( + SQLite3DataBase db, + String var, String value )= + with binds = (List(SQLite3Bind))[bind_String("@name", var), bind_String("@value", value)], if select_settings(db, var) is success(_) then - forget(sql_query_timeout(db, "UPDATE settings SET var_value = @value WHERE var_name = @name;", [bind_String("@name", var), bind_String("@value", value)], "update_settings")) + forget(sql_query_timeout(db, "UPDATE settings SET var_value = @value WHERE var_name = @name;", binds, "update_settings")) else - forget(sql_query_timeout(db, "INSERT INTO settings (var_name, var_value) VALUES (@name, @value)", [bind_String("@name", var), bind_String("@value", value)], "update_settings")) + forget(sql_query_timeout(db, "INSERT INTO settings (var_name, var_value) VALUES (@name, @value)", binds, "update_settings")) . diff --git a/mail/smtp_client.anubis b/mail/smtp_client.anubis index 58f6c47..447ddbc 100644 --- a/mail/smtp_client.anubis +++ b/mail/smtp_client.anubis @@ -406,9 +406,9 @@ define Maybe(One) { failure then failure, success(nb_write) then - if now - buffer_start_time > 120 then + if now - buffer_start_time > 3600 then // security to avoid queue blocking - logger(logError, "sm_flush: TIMEOUT sending a " + length(buffer) + " byte-length buffer (taking more than 2 minutes). SendMail canceled."); + logger(logError, "sm_flush: TIMEOUT sending a 64kb buffer (taking more than 1 hour). SendMail canceled."); failure else with buffer_size = length(buffer), @@ -427,38 +427,44 @@ define SendContentResult SmtpClientSession session, Int start_time, Int so_far, - Int last_reply_check, (Int) -> One progress_report, (LogLevel, String) -> One logger ) = - if last_reply_check + 60 < now then - if receive_reply(weaken(target), [], session.enhanced_status, 0, logger) is - { - error then copy_error(so_far), - timeout then - sm_copy_Data_IO_to_Stream(source, target, session, start_time, so_far, now, progress_report, logger) - no_auth_method then copy_error(so_far), // impossible - bad_reply then copy_error(so_far), - reply(code, status, lines) then - smtp_reply(reply(code, status, lines)) - } - else - if read_line(source, 1024) is + if receive_reply(weaken(target), [], session.enhanced_status, 0, logger) is + { + error then copy_error(so_far), + timeout then + if read_bytes(source, 65536) is { - error then logger(logError, "send_content: failed to read input data_io"); copy_error(so_far), - timeout then logger(logError, "send_content: timeout reading input data_io"); copy_error(so_far), - eof then copy_ok(so_far), - ok(line) then - with buffer = to_byte_array((if nth(0, line) is success(char) then if char = '.' then "." + line else line else line)), + failure then logger(logError, "send_content: failed to read input data_io"); copy_error(so_far), + time_out then logger(logError, "send_content: timeout reading input data_io"); copy_error(so_far), + success(buffer) then if sm_flush( buffer, weaken(target), now, logger ) is { failure then copy_error(so_far), success(_) then - with len = length(line) + 2, // 2 is for the CRLF - progress_report(so_far + len); - sm_copy_Data_IO_to_Stream(source, target, session, start_time, so_far + len, last_reply_check, progress_report, logger) + progress_report(so_far + 65536); + sm_copy_Data_IO_to_Stream(source, target, session, start_time, so_far + 65536, progress_report, logger) }, - }. + + truncated(buffer) then + with len = length(buffer), + if len = 0 then + copy_ok(so_far) + else + if sm_flush( buffer, weaken(target), now, logger ) is + { + failure then copy_error(so_far), + success(_) then + progress_report(so_far + len); + copy_ok(so_far + len) + } + }, + no_auth_method then copy_error(so_far), // impossible + bad_reply then copy_error(so_far), + reply(code, status, lines) then + smtp_reply(reply(code, status, lines)) + }. define SendContentResult sm_copy_Data_IO_List_to_Stream @@ -476,7 +482,7 @@ define SendContentResult [] then copy_ok(so_far), [ h . t ] then if rewind(h)(unique) then - with result = sm_copy_Data_IO_to_Stream(h, target, session, start_time, 0, now, progress_report, logger), + with result = sm_copy_Data_IO_to_Stream(h, target, session, start_time, 0, progress_report, logger), if result is copy_ok(written) then sm_copy_Data_IO_List_to_Stream(t, target, session, start_time, so_far + written, progress_report, logger) else diff --git a/web/CXM_making_a_web_site.anubis b/web/CXM_making_a_web_site.anubis index 6007608..7da8f5e 100644 --- a/web/CXM_making_a_web_site.anubis +++ b/web/CXM_making_a_web_site.anubis @@ -2267,7 +2267,7 @@ public define Web_Site // // make required directories (if needed) // - with //web_sites_directory = (String) make_directory(my_anubis_directory+"/web_sites"), + with web_sites_directory = (String) make_directory(my_anubis_directory+"/web_sites"), base_directory = (String) make_directory(site_directory), // state_directory = make_directory(site_directory+"/states"), forget((String)make_directory(site_directory+"/public")); -- libgit2 0.21.4