Anubis / Xlib

/*
 * Created by PyramIDE.
 * User: ricard
 * Date: 22/12/2008
 * Time: 17:49
 * 
 */


read tools/basis.anubis   
read tools/connections.anubis
read tools/findstring.anubis   
read tools/printable_tree.anubis
read data_base/sqlite.anubis
read network/dns.anubis
read network/tools.anubis
read system/files.anubis
read system/string.anubis
read system/data_io.anubis
read system/logger.anubis

read calexium_lib/net_services_protocols/logger_service.anubis

read lexers/enhanced_status.anubis
read smtp_server_extensions.anubis
read authentication.anubis

public define String send_mail_log    = "SendMail".
public define LogMask send_mail_mask  = logMask("send_mail").

define Int        smtp_time_out     = 300.  //5 minutes of timeout
define ByteArray  crlf_dot_crlf     = to_byte_array(implode([13,10,'.',13,10])). 

public type SendMailResult:
  ok,
  error,
  reply(Int code, String enhanced_status, List(String) lines).
  
public type Str_HostName: 
  str_host_name(String str).

public type Str_Sender: 
  str_sender(String str).
  
public type Str_Recipient:
  str_recipient(String str).

public type Str_UID:
  str_UID(String str).
  
public type Send_Auth:
  none,
  login(  String login,
          String password).
      
public type Send_Param:
  send_param( Str_HostName  host_name,
              Str_Sender    sender,
              Str_Recipient recipient,
              List(Data_IO) mail_parts,
              Send_Auth     auth,
              Str_UID       mail_uid,
              Int           last_smtp_code  //0 means it's the first time
              ).
            
public type Send_Timeout:
  send_timeout(Int when).
  
public type Send_Tries:
  send_tries(Int number).
  


  
public type Send_Mail_Session:
  send_mail_session(
    Bool enhanced_status,
    Int size
    ).

type Command_Result:
  failure,
  timeout,
  success(String).
  
define Command_Result
  receive_command
  (
    RStream     conn,
    List(Word8) so_far,
    Word8       previous,
    Int         time_out,
    (LogLevel, String) -> One logger
  ) =
  if read_network_byte(conn, time_out) is 
  {
    failure    then logger(logTrace, "receive_command read_network_byte 0 failure");failure,
    timeout    then logger(logWarning, "[send_mail] receive_command timeout"); timeout,
    success(c) then 
    if c = 10 & previous = 13 then  // <LF>
      with result = implode(reverse(so_far)),
      logger(logTrace, "<-S-"+result); 
      success(result)
    else 
      if previous = 13 then
        receive_command(conn, so_far, c, time_out, logger)
      else
        receive_command(conn, [previous . so_far], c, time_out, logger)
 }.   

define Command_Result
  receive_command
  (
    RStream     conn,
    List(Word8) so_far,
    Int         time_out,
    (LogLevel, String) -> One logger
  ) =
  if read_network_byte(conn, time_out) is 
  {
    failure    then logger(logTrace, "receive_command read_network_byte 0 failure"); failure,
    timeout    then (if time_out = 0 then unique else logger(logWarning, "[send_mail] receive_command timeout")); timeout,
    success(c) then receive_command(conn,[], c, time_out, logger)
  }. 
  
  
type Reply_Result:
  error,
  timeout,
  reply(Int code, List(String) lines).

define SendMailResult
  reply_handling
  (
    Int         code,
    List(String)  lines,
    Bool          enhanced_status
  ) =
    with result = 
    if enhanced_status then
      if extract_enhanced_status(lines) is
      {
        failure then "",
        success(enh) then enh
      }
    else
      "",
    reply(code, result, lines).

  
define Reply_Result
  receive_reply
  (
    RStream      conn,
    List(String)  so_far,
    Int           time_out,
    (LogLevel, String) -> One logger
  ) =
  if receive_command(conn, [], time_out, logger) is
  {
    failure       then logger(logError, "receive_reply: error receiving command"); error,
    timeout       then (if time_out > 0 then logger(logError, "receive_reply: timeout receiving command") else unique); timeout,
    success(line) then  
      //check if we must read another line by presence of hyphen after the reply code
      //220-bla bla bla
      //220 end of bla bla
      if length(line) =< 3 then
        if decimal_scan(line) is
        {
          failure       then logger(logError, "receive_reply: can't extract reply code from '" + line + "'"); error,
          success(code) then reply(code, reverse(so_far))
        }
      else if nth(3, line) is
      {
        failure       then logger(logError, "receive_reply: error getting 4th character from '" + line +"'"); error,
        success(char) then 
          if char = '-' then  
            receive_reply(conn, [line . so_far], time_out, logger)
          else
            //decode the code
            if sub_string(line, 0, 3) is
            {
              failure           then logger(logError, "receive_reply: error extracting reply code from '" + line+"'"); error,  // should never occure
              success(code_str) then 
                if decimal_scan(code_str) is
                {
                  failure       then  logger(logError, "receive_reply: can't extract reply code from '" + code_str + "'"); error, //unreadable code
                  success(code) then reply(code , reverse([line . so_far]))
                }
            }
      }
  }.

define Reply_Result
  receive_reply   
  (
    RWStream conn,
    (LogLevel, String) -> One logger
  ) =
  receive_reply(weaken(conn), [], smtp_time_out, logger).
      
   
   Sending a piece of text (String) from the begining. 
   
define Maybe(One)
  smtp_send_line
  (
    RWStream  conn,
    String    text,
    (LogLevel, String) -> One logger
  ) =
  if reliable_write(tcp(conn),[text + crlf]) is 
  {
    failure     then logger(logError, "smtp_send_line: error writing '" + text + "'"); failure,
    success(_)  then 
      logger(logTrace, "-C->"+text);success(unique)
  } .
  
   Because of attachements, we may have to manipulate very big pieces of text. It would be
   unreasonable  to  concatenate all  pieces  into  a single  String.  Hence,  we use  the
   following type:
   
type StringTree:
  [ ], 
  str_tree(String,StringTree),
  ba_tree(ByteArray,StringTree),
  tree_tree(StringTree,StringTree). 

define StringTree [String     s . StringTree t] = str_tree(s,t). 
define StringTree [ByteArray  s . StringTree t] = ba_tree(s,t). 
define StringTree [StringTree s . StringTree t] = tree_tree(s,t). 
   
   Try to send the 'HELO' command and get the reply. Return 'true' if you cannot. 
   
define Reply_Result
  send_ehlo
  (
    RWStream  conn, 
    String    our_host_name,
    (LogLevel, String) -> One logger
  ) =
  if smtp_send_line(conn,"EHLO "+our_host_name, logger) is 
  {
    failure     then logger(logError, "send_ehlo: error sending EHLO"); error, 
    success(_)  then 
      with rep = if receive_reply(conn, logger) is
                  {
                    error              then logger(logError, "send_ehlo: can't get reply"); error,
                    timeout            then logger(logError, "send_ehlo: timeout getting reply"); timeout,
                    reply(code, lines) then
                      //we manage the 500 error, that mean the remote server is not ESMTP
                      //hence we try we with HELO, the old manner RFC 821
                      if code = 500 | code = 502 then
                        if smtp_send_line(conn,"HELO "+our_host_name, logger) is 
                        {
                          failure     then logger(logError, "send_ehlo: error sending HELO"); error, 
                          success(_)  then receive_reply(conn, logger)
                        }
                      else
                        reply(code, lines)
                  },
      if rep is reply(code, lines) then
        if code = 550 then    // some server simply refuse us because it's temporary overloaded (especially try when sending mailings)
          logger(logDebug, "Converting a 550 error for EHLO to a 432 error as distant server may be temporary overloaded.");
          reply(432, lines)
        else
          rep
      else 
        rep
  }. 
   
   The same one for 'MAIL FROM':

define String
  check_email_syntax
  (
    String email
  ) =
  if nth(0, email) is
  {
    failure then "<>",
    success(b) then 
      if b = '<' then email
                 else "<" + email + ">"
  }.

   
define Reply_Result
  send_mail_from
  (
    RWStream  conn, 
    String    sender,
    (LogLevel, String) -> One logger
  ) =
  with rfc_sender = check_email_syntax(sender),
  if smtp_send_line(conn,"MAIL FROM:"+rfc_sender, logger) is 
  {
    failure     then logger(logError, "error sending 'MAIL FROM:" + rfc_sender + "'"); error, 
    success(_)  then receive_reply(conn, logger)
  }. 
   
   The same one for 'RCPT TO':
   
define Reply_Result
  send_recipient
  (
    RWStream            conn, 
    String              recipient, 
    (LogLevel, String) -> One logger
  ) =
  with rfc_recipient = check_email_syntax(recipient),
  if smtp_send_line(conn,"RCPT TO:"+rfc_recipient, logger) is 
  {
    failure     then logger(logError, "error sending 'RCPT TO:"+rfc_recipient+"'"); error,
    success(_)  then receive_reply(conn, logger)
  }.
   
  
   Try to send 'DATA' and get the reply. Answer 'true' if you cannot. 
   
define Reply_Result
  send_data
  (
    RWStream conn,
    (LogLevel, String) -> One logger
  ) =
  if smtp_send_line(conn,"DATA", logger) is 
  {
    failure     then logger(logError, "error sending DATA"); error, 
    success(_)  then receive_reply(conn, logger)
  }. 

   
   
   The same for the content of the message. We need base64 encoding.
   
read tools/base64.anubis          
type SendContentResult:
  smtp_reply(Reply_Result),
  copy_error(Int written),
  copy_ok(Int written).
  
define Maybe(One)
  sm_flush
  (
    ByteArray buffer,
    WStream   target,
    Int       buffer_start_time,
    (LogLevel, String) -> One logger
  )=
  if write( target , buffer) is
  {
    failure           then failure,
    success(nb_write) then
      if now - buffer_start_time > 3600 then
        // security to avoid queue blocking
        logger(logError, "sm_flush: TIMEOUT sending a 64kb buffer (taking more than 1 hour). SendMail canceled.");
        failure
      else
        with buffer_size = length(buffer),      
        if nb_write = buffer_size then
          success(unique)
        else
          with new_buffer = extract(buffer, nb_write, buffer_size),
          sm_flush(new_buffer, target, buffer_start_time, logger)
  }.
  
public define SendContentResult
  sm_copy_Data_IO_to_Stream
  (
    Data_IO   source, 
    RWStream  target,
    Int       start_time,
    Int       so_far,
    (Int) -> One  progress_report,
    (LogLevel, String) -> One logger
  ) =
  if receive_reply(weaken(target), [], 0, logger) is
  {
    error then copy_error(so_far),
    timeout then
      if read_bytes(source, 65536) is
      {
        failure         then logger(logError, "send_content: failed to read input data_io"); copy_error(so_far),
        time_out        then logger(logError, "send_content: timeout reading input data_io"); copy_error(so_far), 
        success(buffer) then 
          if sm_flush( buffer, weaken(target), now, logger ) is
          {
            failure     then copy_error(so_far),
            success(_)  then 
              progress_report(so_far + 65536);
              sm_copy_Data_IO_to_Stream(source, target, start_time, so_far + 65536, progress_report, logger)
          },
          
        truncated(buffer)       then
          with len = length(buffer),
          if len = 0 then
            copy_ok(so_far)
          else 
            if sm_flush( buffer, weaken(target), now, logger ) is
            {
              failure     then copy_error(so_far),
              success(_)  then 
                progress_report(so_far + len);
                copy_ok(so_far + len)
            }
      },
    reply(code, lines) then
      smtp_reply(reply(code, lines))
  }.

define SendContentResult
  sm_copy_Data_IO_List_to_Stream
  (
    List(Data_IO) io_list,
    RWStream      target,
    Int           start_time,
    Int           so_far,
    (Int) -> One  progress_report,
    (LogLevel, String) -> One logger
  )=
  if io_list is
  {
    []        then copy_ok(so_far),
    [ h . t ]  then
      if rewind(h)(unique) then
        with result = sm_copy_Data_IO_to_Stream(h, target, start_time, 0, progress_report, logger),
        if result is copy_ok(written) then 
          sm_copy_Data_IO_List_to_Stream(t, target, start_time, so_far + written, progress_report, logger)
        else
          result
      else
        copy_error(so_far)
  }.

define Reply_Result
  send_content
  (
    RWStream      conn,
    List(Data_IO)  mail_part,
    (Int) -> One  progress_report,
    (LogLevel, String) -> One logger
  ) =
  with start_time = now,
  if sm_copy_Data_IO_List_to_Stream(mail_part, conn, now, 0, progress_report, logger) is
  {
    smtp_reply(reply)   then reply
    copy_error(written) then 
      with finish_time = now,
      logger(logError, "send_content: error sending data");
      logger(logTrace, "send_content: "+written+" bytes sent in "+finish_time - start_time+" second(s)"); 
      if written > 1024000 then 
        logger(logInfo, "send_content: may be due to over sized mail. Convert it to 452 error.");
        reply(452, ["Error sending big email (more than 1 Mb)"])
      else
        error,
    copy_ok(size)          then
      if reliable_write(tcp(conn), [crlf_dot_crlf]) is
      {
        failure     then logger(logError, "send_content: error writing CRLF.CRLF"); error,
        success(_)  then 
          with finish_time = (Int)now,
          logger(logTrace, "mail sent "+size+" bytes in "+finish_time - start_time+" second(s)"); 
          receive_reply(conn, logger)
      }
  }.

   
   Almost the same for 'QUIT':
   
define One
  send_quit
  (
    RWStream conn,
    (LogLevel, String) -> One logger
  ) =
  if smtp_send_line(conn,"QUIT", logger) is 
    {
      failure     then logger(logWarning, "error sending QUIT"), 
      success(_)  then forget(receive_reply(conn, logger))
    }.

define SendMailResult
  sm_MAIL_FROM
  (                                            
    RWStream                  conn,     
    Send_Param                param,
    Send_Mail_Session         sm_session,
    (Int) -> One              progress_report,
    (LogLevel, String) -> One logger
  ) =
  with enhanced_status = sm_session.enhanced_status,
  //send MAIL FROM
  if send_mail_from(conn, param.sender.str, logger) is
  {
    error               then error, // already logged
    timeout             then error, 
    reply(code, lines)  then
      if code = 250 then
        //send RCPT TO
        if send_recipient(conn,param.recipient.str, logger) is
        {
          error               then error, // already logged
          timeout             then error, 
          reply(code, lines)  then
          if code = 250 then
            //send DATA
            if send_data(conn, logger) is
            {
              error               then  error,  // already logged
              timeout             then error, 
              reply(code, lines)  then
              if code = 354 then 
                if send_content(conn, param.mail_parts, progress_report, logger) is
                {
                  error               then error, // already logged
                  timeout             then error, 
                  reply(code, lines)  then
                    if code = 250 then
                      send_quit(conn, logger);
                      logger(logTrace, "SENT Mail FROM "+param.sender.str+" TO "+param.recipient.str);
                      ok
                    else
                      reply_handling(code, lines, enhanced_status)
                }
              else
                reply_handling(code, lines, enhanced_status)
            }
          else
            reply_handling(code, lines, enhanced_status)
        }
      else
        reply_handling(code, lines, enhanced_status)
  }
.

define List(String)
  get_auth_method
  (
    List(String)  lines,
    List(String)  so_far
  )=
  if lines is
  {
    []    then  so_far,
    [h . t] then  
      if sub_string(h, 4, 4) is
      {
        failure     then get_auth_method(t, so_far),
        success(s)  then 
          with current =  if insensitive_equal(s, "AUTH") then
                            force_Type(list_word(h,9),[])
                          else
                            [],
         get_auth_method(t, current + so_far)
      } 
  }.
  
define SendMailResult
  do_auth_plain
  (
    RWStream  conn,
    String    login,
    String    password,
    Bool      enhanced_status,
    (LogLevel, String) -> One logger
  )=
  
  with plain_str = "AUTH PLAIN "+ to_string(base64_encode(to_byte_array("")+
                                                constant_byte_array(1,0)+
                                                to_byte_array(login)+
                                                constant_byte_array(1,0)+
                                                to_byte_array(password))),
  if smtp_send_line(conn, plain_str, logger) is 
  {
    failure     then error, // already logged
    success(_)  then 
      if receive_reply(conn, logger) is 
      {
        error               then error // already logged
        timeout             then error, 
        reply(code, lines)  then
          if code = 235 then
            ok
          else
            reply_handling(code,lines,enhanced_status)
      }
  }.

define SendMailResult
  do_auth_login
  (
    RWStream  conn,
    String    login,
    String    password,
    Bool      enhanced_status,
    (LogLevel, String) -> One logger
  )=
  if smtp_send_line(conn, "AUTH LOGIN", logger) is 
  {
    failure     then error, // already logged
    success(_)  then 
      if receive_reply(conn, logger) is 
      {
        error               then error // already logged
        timeout             then error, 
        reply(code, lines)  then
          if code = 334 then
            if smtp_send_line(conn, to_string(base64_encode(to_byte_array(login))), logger) is 
            {
              failure     then error, // already logged
              success(_)  then 
                if receive_reply(conn, logger) is 
                {
                  error               then error // already logged
                  timeout             then error, 
                  reply(code_login, lines_login)  then
                    if code = 334 then
                      if smtp_send_line(conn, to_string(base64_encode(to_byte_array(password))), logger) is 
                      {
                        failure     then error, // already logged
                        success(_)  then 
                          if receive_reply(conn, logger) is 
                          {
                            error               then error // already logged
                            timeout             then error, 
                            reply(code_pwd, lines_pwd)  then
                              if code = 235 then
                                ok
                              else
                                reply_handling(code_pwd, lines_pwd, enhanced_status)
                          }
                      }
                    else
                      reply_handling(code_login, lines_login, enhanced_status)
                }
            }
          else
            reply_handling(code, lines, enhanced_status)
      }
  }.
  
define SendMailResult
  do_auth_cram_md5
  (
    RWStream  conn,
    String    login,
    String    password,
    Bool      enhanced_status,
    (LogLevel, String) -> One logger
  )=
  if smtp_send_line(conn, "AUTH CRAM-MD5", logger) is 
  {
    failure     then error, // already logged
    success(_)  then 
      if receive_reply(conn, logger) is 
      {
        error               then error // already logged
        timeout             then error, 
        reply(code, lines)  then
          if code = 334 then
            with challenge = if lines is [h . t] then base64_decode(h) else "",
                 digest = hmac_md5_compute(challenge, password),
            if smtp_send_line(conn, base64_encode(login + " " + digest, false), logger) is 
            {
              failure     then error, // already logged
              success(_)  then 
                if receive_reply(conn, logger) is 
                {
                  error               then error // already logged
                  timeout             then error, 
                  reply(code_login, lines_login)  then
                    if code = 235 then
                      ok
                    else
                      reply_handling(code_login, lines_login, enhanced_status)
                }
            }
          else
            reply_handling(code, lines, enhanced_status)
      }
  }.
  
define SendMailResult
  do_login
  (
    RWStream      conn,   //tcp connection
    List(String)  auth_list,  //list of available authentication method
    String        login,
    String        password,
    Bool          enhanced_status,
    (LogLevel, String) -> One logger
  )=
  if member(auth_list, "CRAM-MD5", insensitive_equal) then
    do_auth_cram_md5(conn, login, password, enhanced_status, logger)
  else if member(auth_list, "PLAIN", insensitive_equal) then
    do_auth_plain(conn, login, password, enhanced_status, logger)
  else if member(auth_list, "LOGIN", insensitive_equal) then
    do_auth_login(conn, login, password, enhanced_status, logger)

  else
    logger(logError, "do_login: none of CRAM-MD5, LOGIN or PLAIN method available"); error.

define SendMailResult
  do_auth
  (
    RWStream      conn,   //tcp connection
    List(String)  lines,  //this is the reply lines provided in response to EHLO command
    Send_Auth     auth,   //authentication method to use for that session
    Bool          enhanced_status,
    (LogLevel, String) -> One logger
  )=
  if auth is
  {
    none                  then ok,  //no need to authenticate, this is a "panties festival"
    login(user, password) then 
      with auth_list = get_auth_method(lines,[]), 
      if auth_list is
      {
        []     then logger(logError, "do_auth error");error,
        [_ . _]  then do_login(conn, auth_list, user, password, enhanced_status, logger)
      }
  }.

  /**
   *   make_sm_session
   */ 
define Send_Mail_Session
  make_sm_session
  (
    List(String)  lines,    //lines given in ehlo stage
    Send_Param    param,
    (LogLevel, String) -> One logger
  )=
  //looking for ENHANCEDSTATUSCODES
  with session_enhanced_status = has_smtp_extension(lines, enhanced_status_codes),
  //looking for SIZE 
  with session_size = if get_smtp_extension_value(lines, size) is
              {
                failure             then  0,
                success(str_value)  then  
                  if decimal_scan(str_value)  is
                  {
                    failure         then 0,
                    success(value)  then
                      logger(logTrace, "smtp server with SIZE "+value);
                      value
                  }
              },
  send_mail_session(session_enhanced_status, session_size).
  
   Apply the whole protocol:
   
   // we can have a list of part file in param type. This can be very useful when we want
   // to send file as mailing. For the mailing, in each mail, only the header is different of the 
   // other mails. The body part is the same. Then, the mailing sender can generate the header in one 
   // file and keep the body in other file wich can be given to send_mail function as last file in the list
public define SendMailResult
  send_mail
  (
    RWStream        conn,     
    Send_Param      param,
    (Send_Mail_Session, Send_Param) -> Result(SendMailResult, Send_Param)   prepare_mail_callback,
    (Int) -> One    progress_report,
    (LogLevel, String) -> One logger
  ) =
   //
   // Our connection to the SMTP server is opened. We just have to apply the protocol.
   //
  if receive_reply(conn, logger) is
  {
    error               then  error,// already logged
    timeout             then  error,
    reply(code, lines)  then
    if code = 220 then
      //send EHLO
      if send_ehlo(conn, param.host_name.str, logger) is   //"mail."+force_Type(get_main_domain(db), "mailfountain.net")) is
      {
        error               then error, // already logged
        timeout             then error, 
        reply(code, lines)  then
          if code = 250 then
            // parse all available options here, and build a smtp context
            with sm_session = make_sm_session(lines, param, logger),
            with auth_result = do_auth(conn, lines, param.auth, sm_session.enhanced_status, logger),
            if auth_result = ok then
              if prepare_mail_callback(sm_session, param) is
              {
                error(result) then result,
                ok(new_param) then sm_MAIL_FROM(conn, new_param, sm_session, progress_report, logger)
              }
            else
              auth_result
          else
            reply_handling(code,lines,true)
      }
      else if code = 550 then
        //some server answer "550 5.7.1 Client host rejected: cannot find your reverse hostname, [88.181.64.17]"
        //before anything, then we try to extract the enhanced status if exists for relaying the mail with the ISP
        logger(logDebug, "Converting a 550 error for connection to a 432 error as distant server may be temporary overloaded.");
        reply_handling(432,lines,true)
      else 
        reply_handling(code,lines,true)
  }.

public define SendMailResult
  send_mail
  (
    RWStream        conn,     
    Send_Param      param,
    (Send_Mail_Session, Send_Param) -> Result(SendMailResult, Send_Param)   prepare_mail_callback,
  ) =
  send_mail(conn, 
            param, 
            prepare_mail_callback,
            (Int _) |-> unique,
            (LogLevel level, String txt) |-> if level is logTrace then logTrace(send_mail_log, send_mail_mask, txt)
                                                                  else log(level, send_mail_log, txt)
            ).

public define SendMailResult
  send_mail
  (
    RWStream        conn,     
    Send_Param      param,
  ) =
  send_mail(conn, 
            param, 
            (Send_Mail_Session sm_session, Send_Param param) |-> ok(param)
            ).

public define SendMailResult
  send_mail
  (
    String          server,
    Word32          port,
    Send_Param      param,
  )=
  if resolve_address(server) is
  {
    failure           then  
      logWarning(send_mail_log,"IP address NOT found for Mail server ["+server+"]");
      error,

    success(ip_addr)  then
      with ip_txt = ip_addr_to_string(ip_addr),
      logTrace(send_mail_log, send_mail_mask, "send_mail: Trying to connect to "+ip_txt);

      if (Result(NetworkConnectError,RWStream))connect(ip_addr, port) is 
      {
        error(e)               then 
          logTrace(send_mail_log,send_mail_mask,"send_mail: Can't connect to "+ip_txt); 
          error,
      
        ok(server_conn)        then 
          //
          //with status = to_Send_Status(send_mail(db, server_conn, send_param(str_sender(sender), str_recipient(recipient), [make_data_io(source)],auth, str_UID(file_uid) ))),
          send_mail(server_conn, param),
      }
  }.