commit 8618b5c7a533f167bff9c25c9653d8d3ab94b68f Merge: 50ce3e8ca 7e97e1fa4 Author: Jeremy Harris Date: Thu Jul 11 19:18:13 2024 +0100 Merge branch '4.next' diff --git a/release-process/scripts/docs_strip_changebars b/release-process/scripts/docs_strip_changebars new file mode 100755 index 000000000..cb6e21827 diff --git a/src/OS/Makefile-Base b/src/OS/Makefile-Base index afa2a7a23..a6354a4c9 100644 --- a/src/OS/Makefile-Base +++ b/src/OS/Makefile-Base @@ -695,6 +695,11 @@ HDRS = blob.h \ hash.h \ hintsdb.h \ hintsdb_structs.h \ + hintsdb/hints_bdb.h \ + hintsdb/hints_gdbm.h \ + hintsdb/hints_ndbm.h \ + hintsdb/hints_sqlite.h \ + hintsdb/hints_tdb.h \ local_scan.h \ macros.h \ mytypes.h \ @@ -709,6 +714,11 @@ PHDRS = ../config.h \ ../globals.h \ ../hintsdb.h \ ../hintsdb_structs.h \ + ../hintsdb/hints_bdb.h \ + ../hintsdb/hints_gdbm.h \ + ../hintsdb/hints_ndbm.h \ + ../hintsdb/hints_sqlite.h \ + ../hintsdb/hints_tdb.h \ ../local_scan.h \ ../macros.h \ ../mytypes.h \ diff --git a/src/scripts/MakeLinks b/src/scripts/MakeLinks index ddb237980..cda5ef65b 100755 --- a/src/scripts/MakeLinks +++ b/src/scripts/MakeLinks @@ -28,22 +28,24 @@ echo ">>> Creating links to source files..." # their own Makefile in their sub-directory. # Firstly the lookups -mkdir lookups -cd lookups +d="lookups" +mkdir $d +cd $d # Makefile is generated for f in README cdb.c dbmdb.c dnsdb.c dsearch.c ibase.c json.c ldap.h ldap.c \ lmdb.c lsearch.c mysql.c nis.c nisplus.c oracle.c passwd.c \ pgsql.c readsock.c redis.c spf.c sqlite.c testdb.c whoson.c \ lf_functions.h lf_check_file.c lf_quote.c lf_sqlperform.c do - ln -s ../../src/lookups/$f $f + ln -s ../../src/$d/$f $f done cd .. # Likewise for the code for the routers -mkdir routers -cd routers +d="routers" +mkdir $d +cd $d for f in README Makefile accept.h accept.c dnslookup.h dnslookup.c \ ipliteral.h ipliteral.c iplookup.h iplookup.c manualroute.h \ manualroute.c queryprogram.h queryprogram.c redirect.h redirect.c \ @@ -51,26 +53,28 @@ for f in README Makefile accept.h accept.c dnslookup.h dnslookup.c \ rf_get_munge_headers.c rf_get_transport.c rf_get_ugid.c rf_queue_add.c \ rf_lookup_hostlist.c rf_self_action.c rf_set_ugid.c do - ln -s ../../src/routers/$f $f + ln -s ../../src/$d/$f $f done cd .. # Likewise for the code for the transports -mkdir transports -cd transports +d="transports" +mkdir $d +cd $d for f in README Makefile appendfile.h appendfile.c autoreply.h \ autoreply.c lmtp.h lmtp.c pipe.h pipe.c queuefile.c queuefile.h \ smtp.h smtp.c smtp_socks.c tf_maildir.c tf_maildir.h do - ln -s ../../src/transports/$f $f + ln -s ../../src/$d/$f $f done cd .. # Likewise for the code for the authentication drivers -mkdir auths -cd auths +d="auths" +mkdir $d +cd $d for f in README Makefile call_pam.c call_pwcheck.c \ call_radius.c check_serv_cond.c cyrus_sasl.c cyrus_sasl.h gsasl_exim.c \ gsasl_exim.h get_data.c get_no64_data.c heimdal_gssapi.c heimdal_gssapi.h \ @@ -78,17 +82,28 @@ for f in README Makefile call_pam.c call_pwcheck.c \ pwcheck.c pwcheck.h auth-spa.c auth-spa.h dovecot.c dovecot.h sha1.c spa.c \ spa.h tls.c tls.h external.c external.h do - ln -s ../../src/auths/$f $f + ln -s ../../src/$d/$f $f +done +cd .. + +# and the hintsdb implementations +d="hintsdb" +mkdir $d +cd $d +for f in hints_bdb.h hints_gdbm.h hints_ndbm.h hints_sqlite.h hints_tdb.h +do + ln -s ../../src/$d/$f $f done cd .. # Likewise for the code for the PDKIM library -mkdir pdkim -cd pdkim +d="pdkim" +mkdir $d +cd $d for f in README Makefile crypt_ver.h pdkim.c \ pdkim.h hash.c hash.h signing.c signing.h blob.h do - ln -s ../../src/pdkim/$f $f + ln -s ../../src/$d/$f $f done cd .. diff --git a/src/src/daemon.c b/src/src/daemon.c index 16137f9f6..f3c49e25e 100644 --- a/src/src/daemon.c +++ b/src/src/daemon.c @@ -324,7 +324,8 @@ if (smtp_accept_max_per_host) tedious per host_address checks. Note that at this stage smtp_accept_count contains the count of *other* connections, not including this one. */ -if (max_for_this_host > 0 && smtp_accept_count >= max_for_this_host) +if ( smtp_slots + && max_for_this_host > 0 && smtp_accept_count >= max_for_this_host) { int host_accept_count = 0; int other_host_count = 0; /* keep a count of non matches to optimise */ @@ -392,9 +393,9 @@ if (pid == 0) arrange to unset the selector in the subprocess. jgh 2023/08/08 :- moved this logging in from the parent process, just - pre-fork. There was a claim back from 2004 that smtp_accept_count could have - become out-of-date by the time the child could log it, and I can't see how - that could happen. */ + pre-fork. There was a claim back from 4.21 (when it was moved from + smtp_start_session()) that smtp_accept_count could have become out-of-date by + the time the child could log it, and I can't see how that could happen. */ if (LOGGING(smtp_connection)) { @@ -404,7 +405,8 @@ if (pid == 0) save_log_selector &= ~L_smtp_connection; else if (LOGGING(connection_id)) log_write(L_smtp_connection, LOG_MAIN, "SMTP connection from %Y " - "Ci=%lu (TCP/IP connection count = %d)", whofrom, connection_id, smtp_accept_count); + "Ci=%lu (TCP/IP connection count = %d)", + whofrom, connection_id, smtp_accept_count); else log_write(L_smtp_connection, LOG_MAIN, "SMTP connection from %Y " "(TCP/IP connection count = %d)", whofrom, smtp_accept_count); @@ -752,7 +754,7 @@ remember the pid for ticking off when the child completes. */ if (pid < 0) never_error(US"daemon: accept process fork failed", US"Fork failed", errno); -else +else if (smtp_slots) { for (int i = 0; i < smtp_accept_max; ++i) if (smtp_slots[i].pid <= 0) @@ -912,12 +914,13 @@ while ((pid = waitpid(-1, &status, WNOHANG)) > 0) if (smtp_slots) { int i; - for (i = 0; i < smtp_accept_max; i++) - if (smtp_slots[i].pid == pid) + smtp_slot * sp; + for (i = 0, sp = smtp_slots; i < smtp_accept_max; i++, sp++) + if (sp->pid == pid) { - if (smtp_slots[i].host_address) - store_free(smtp_slots[i].host_address); - smtp_slots[i] = empty_smtp_slot; + if (sp->host_address) + store_free(sp->host_address); + *sp = empty_smtp_slot; if (--smtp_accept_count < 0) smtp_accept_count = 0; DEBUG(D_any) debug_printf("%d SMTP accept process%s now running\n", smtp_accept_count, smtp_accept_count == 1 ? "" : "es"); diff --git a/src/src/dbfn.c b/src/src/dbfn.c index 2b5ec908b..d1d8c08d4 100644 --- a/src/src/dbfn.c +++ b/src/src/dbfn.c @@ -237,6 +237,78 @@ return dbblock; +/* Only for transaction-capable DB types. Open without locking or +starting a transaction. "lof" and "panic" always true; read/write mode. +*/ + +open_db * +dbfn_open_multi(const uschar * name, int flags, open_db * dbblock) +{ +int rc, save_errno; +flock_t lock_data; +uschar dirname[PATHLEN], filename[PATHLEN]; + +DEBUG(D_hints_lookup) acl_level++; + +dbblock->lockfd = -1; +db_dir_make(TRUE); + +snprintf(CS dirname, sizeof(dirname), "%s/db", spool_directory); +snprintf(CS filename, sizeof(filename), "%s/%s", dirname, name); + +priv_drop_temp(exim_uid, exim_gid); +dbblock->dbptr = exim_dbopen_multi(filename, dirname, flags, EXIMDB_MODE); +if (!dbblock->dbptr && errno == ENOENT && flags == O_RDWR) + { + DEBUG(D_hints_lookup) + debug_printf_indent("%s appears not to exist: trying to create\n", filename); + dbblock->dbptr = exim_dbopen_multi(filename, dirname, O_RDWR|O_CREAT, EXIMDB_MODE); + } +save_errno = errno; +priv_restore(); + +/* If the open has failed, return NULL, leaving errno set. If lof is TRUE, +log the event - also for debugging - but debug only if the file just doesn't +exist. */ + +if (!dbblock->dbptr) + { + errno = save_errno; + if (save_errno != ENOENT) + log_write(0, LOG_MAIN, "%s", string_open_failed("DB file %s", + filename)); + else + DEBUG(D_hints_lookup) + debug_printf_indent("%s\n", CS string_open_failed("DB file %s", + filename)); + errno = save_errno; + DEBUG(D_hints_lookup) acl_level--; + return NULL; + } + +DEBUG(D_hints_lookup) debug_printf_indent( + "opened hints database %s for transactions: NOLOCK flags=O_RDWR\n", filename); + +/* Pass back the block containing the opened database handle */ + +return dbblock; +} + + +BOOL +dbfn_transaction_start(open_db * dbp) +{ +DEBUG(D_hints_lookup) debug_printf_indent("dbfn_transaction_start\n"); +return exim_dbtransaction_start(dbp->dbptr); +} +void +dbfn_transaction_commit(open_db * dbp) +{ +DEBUG(D_hints_lookup) debug_printf_indent("dbfn_transaction_commit\n"); +exim_dbtransaction_commit(dbp->dbptr); +} + + /************************************************* * Unlock and close a database file * @@ -250,11 +322,11 @@ Returns: nothing */ void -dbfn_close(open_db *dbblock) +dbfn_close(open_db * dbp) { -int * fdp = &dbblock->lockfd; +int * fdp = &dbp->lockfd; -exim_dbclose(dbblock->dbptr); +exim_dbclose(dbp->dbptr); if (*fdp >= 0) (void)close(*fdp); DEBUG(D_hints_lookup) { @@ -266,6 +338,18 @@ DEBUG(D_hints_lookup) } +void +dbfn_close_multi(open_db * dbp) +{ +exim_dbclose_multi(dbp->dbptr); +DEBUG(D_hints_lookup) + { + debug_printf_indent("closed hints database\n"); + acl_level--; + } +} + + /************************************************* diff --git a/src/src/dbfunctions.h b/src/src/dbfunctions.h index 2ec37e498..0aa3b777c 100644 --- a/src/src/dbfunctions.h +++ b/src/src/dbfunctions.h @@ -13,12 +13,16 @@ /* Functions for reading/writing exim database files */ void dbfn_close(open_db *); +void dbfn_close_multi(open_db *); int dbfn_delete(open_db *, const uschar *); open_db *dbfn_open(const uschar *, int, open_db *, BOOL, BOOL); +open_db *dbfn_open_multi(const uschar *, int, open_db *); void *dbfn_read_with_length(open_db *, const uschar *, int *); void *dbfn_read_enforce_length(open_db *, const uschar *, size_t); uschar *dbfn_scan(open_db *, BOOL, EXIM_CURSOR **); int dbfn_write(open_db *, const uschar *, void *, int); +BOOL dbfn_transaction_start(open_db *); +void dbfn_transaction_commit(open_db *); /* Macro for the common call to read without wanting to know the length. */ diff --git a/src/src/deliver.c b/src/src/deliver.c index eadc96d22..33d833389 100644 --- a/src/src/deliver.c +++ b/src/src/deliver.c @@ -791,7 +791,7 @@ g = string_append(g, 3, US" [", h->address, US"]"); if (LOGGING(outgoing_port)) g = string_fmt_append(g, ":%d", h->port); -if (continue_sequence > 1) /*XXX this is wrong for a dropped proxyconn. Would have to pass back from transport */ +if (testflag(addr, af_cont_conn)) g = string_catn(g, US"*", 1); #ifdef SUPPORT_SOCKS @@ -1588,12 +1588,6 @@ if (addr->return_file >= 0 && addr->return_filename) (void)close(addr->return_file); } -/* Check if the transport notifed continue-conn status explicitly, and -update our knowlege. */ - -if (testflag(addr, af_new_conn)) continue_sequence = 1; -else if (testflag(addr, af_cont_conn)) continue_sequence++; - /* The success case happens only after delivery by a transport. */ if (result == OK) @@ -2717,8 +2711,7 @@ Returns: Nothing static void do_local_deliveries(void) { -open_db dbblock; -open_db *dbm_file = NULL; +open_db dbblock, * dbm_file = NULL; time_t now = time(NULL); /* Loop until we have exhausted the supply of local deliveries */ @@ -3320,6 +3313,9 @@ int fd = p->fd; uschar *msg = p->msg; BOOL done = p->done; +continue_hostname = NULL; +continue_transport = NULL; + /* Loop through all items, reading from the pipe when necessary. The pipe used to be non-blocking. But I do not see a reason for using non-blocking I/O here, as the preceding poll() tells us, if data is available for reading. @@ -3664,11 +3660,20 @@ while (!done) h->dnssec = *ptr == '2' ? DS_YES : *ptr == '1' ? DS_NO : DS_UNK; - ptr++; addr->host_used = h; } - else ptr++; + ptr++; + continue_flags = 0; +#ifndef DISABLE_TLS + if (testflag(addr, af_cert_verified)) continue_flags |= CTF_CV; +# ifdef SUPPORT_DANE + if (testflag(addr, af_dane_verified)) continue_flags |= CTF_DV; +# endif +# ifndef DISABLE_TLS_RESUME + if (testflag(addr, af_tls_resume)) continue_flags |= CTF_TR; +# endif +#endif /* Finished with this address */ addr = addr->next; @@ -3684,20 +3689,82 @@ while (!done) while (*ptr++) ; break; - /* Z marks the logical end of the data. It is followed by '0' if + /* Z0 marks the logical end of the data. It is followed by '0' if continue_transport was NULL at the end of transporting, otherwise '1'. - We need to know when it becomes NULL during a delivery down a passed SMTP - channel so that we don't try to pass anything more down it. Of course, for - most normal messages it will remain NULL all the time. */ + Those are now for historical reasons only; we always clear the continued + channel info, and then set it explicitly if the transport indicates it + is still open, because it could differ for each transport we are running in + parallel. + + Z1 is a suggested message_id to handle next, used during a + continued-transport sequence. */ case 'Z': - if (*ptr == '0') + switch (*subid) { - continue_transport = NULL; - continue_hostname = NULL; + case '0': /* End marker */ + done = TRUE; + DEBUG(D_deliver) debug_printf("Z0%c item read\n", *ptr); + break; + case '1': /* Suggested continuation message */ + Ustrncpy(continue_next_id, ptr, MESSAGE_ID_LENGTH); + continue_sequence = atoi(CS ptr + MESSAGE_ID_LENGTH + 1); + DEBUG(D_deliver) debug_printf("continue_next_id: %s seq %d\n", + continue_next_id, continue_sequence); + break; + case '2': /* Continued transport, host & addr */ + { + int recvd_fd; + + DEBUG(D_any) if (Ustrcmp(process_purpose, "continued-delivery") != 0) + debug_printf("%s becomes continued-delivery\n", process_purpose); + process_purpose = US"continued-delivery"; + continue_transport = string_copy(ptr); while (*ptr++) ; + continue_hostname = string_copy(ptr); while (*ptr++) ; + continue_host_address = string_copy(ptr); while (*ptr++) ; + continue_sequence = atoi(CS ptr); + + dup2((recvd_fd = recv_fd_from_sock(fd)), 0); + close(recvd_fd); + + DEBUG(D_deliver) + debug_printf("continue: tpt '%s' host '%s' addr '%s' seq %d\n", + continue_transport, continue_hostname, + continue_host_address, continue_sequence); + break; + } + case '3': /* Continued conn info */ + smtp_peer_options = ptr[0]; + f.smtp_authenticated = ptr[1] & 1; + break; +#ifndef DISABLE_TLS + case '4': /* Continued TLS info */ + continue_proxy_cipher = string_copy(ptr); + break; + case '5': /* Continued DANE info */ + case '6': /* Continued TLS info */ +# ifdef SUPPORT_DANE + continue_proxy_dane = *subid == '5'; +# endif + continue_proxy_sni = *ptr ? string_copy(ptr) : NULL; + break; +#endif +#ifndef DISABLE_ESMTP_LIMITS + case '7': /* Continued peer limits */ + sscanf(CS ptr, "%u %u %u", + &continue_limit_mail, &continue_limit_rcpt, + &continue_limit_rcptdom); + break; +#endif +#ifdef SUPPORT_SOCKS + case '8': /* Continued proxy info */ + proxy_local_address = string_copy(ptr); while (*ptr++) ; + proxy_local_port = atoi(CS ptr); while (*ptr++) ; + proxy_external_address = string_copy(ptr); while (*ptr++) ; + proxy_external_port = atoi(CS ptr); + break; +#endif } - done = TRUE; - DEBUG(D_deliver) debug_printf("Z0%c item read\n", *ptr); break; /* Anything else is a disaster. */ @@ -4324,6 +4391,7 @@ So look out for the place it gets used. transport splitting further by max_rcp. So we potentially lose some parallellism. */ + GET_OPTION("max_rcpt"); address_count_max = mua_wrapper || Ustrchr(tp->max_addresses, '$') ? UNLIMITED_ADDRS : expand_max_rcpt(tp->max_addresses); @@ -4371,8 +4439,9 @@ So look out for the place it gets used. && address_count_max < remote_delivery_count/remote_max_parallel ) { - int new_max = remote_delivery_count/remote_max_parallel; - int message_max = tp->connection_max_messages; + int new_max = remote_delivery_count/remote_max_parallel, message_max; + GET_OPTION("connection_max_messages"); + message_max = tp->connection_max_messages; if (connection_max_messages >= 0) message_max = connection_max_messages; message_max -= continue_sequence - 1; if (message_max > 0 && new_max > address_count_max * message_max) @@ -4465,10 +4534,8 @@ nonmatch domains /* Compute the return path, expanding a new one if required. The old one must be set first, as it might be referred to in the expansion. */ - if(addr->prop.errors_address) - return_path = addr->prop.errors_address; - else - return_path = sender_address; + return_path = addr->prop.errors_address + ? addr->prop.errors_address : sender_address; GET_OPTION("return_path"); if (tp->return_path) @@ -4594,19 +4661,30 @@ nonmatch domains continue; } + } - /* Set a flag indicating whether there are further addresses that list - the continued host. This tells the transport to leave the channel open, - but not to pass it to another delivery process. We'd like to do that - for non-continue_transport cases too but the knowlege of which host is - connected to is too hard to manage. Perhaps we need a finer-grain - interface to the transport. */ + /* Once we hit the max number of parallel transports set a flag indicating + whether there are further addresses that list the same host. This tells the + transport to leave the channel open for us. */ +/*XXX maybe we should *count* possible further's, and set continue_more if +parmax * tpt-max is exceeded? */ - for (next = addr_remote; next && !f.continue_more; next = next->next) - for (host_item * h = next->host_list; h; h = h->next) - if (Ustrcmp(h->name, continue_hostname) == 0) - { f.continue_more = TRUE; break; } + if (parcount+1 >= remote_max_parallel) + { + host_item * h1 = addr->host_list; + if (h1) + { + const uschar * name = continue_hostname ? continue_hostname : h1->name; + for (next = addr_remote; next && !f.continue_more; next = next->next) + for (host_item * h = next->host_list; h; h = h->next) + if (Ustrcmp(h->name, name) == 0) + { f.continue_more = TRUE; break; } + } } + else DEBUG(D_deliver) + debug_printf( + "not reached parallelism limit (%d/%d) so not setting continue_more\n", + parcount+1, remote_max_parallel); /* The transports set up the process info themselves as they may connect to more than one remote machine. They also have to set up the filter @@ -4619,13 +4697,16 @@ nonmatch domains fails, it is probably because the value of remote_max_parallel is so large that too many file descriptors for pipes have been created. Arrange to wait for a process to finish, and then try again. If we still can't - create a pipe when all processes have finished, break the retry loop. */ + create a pipe when all processes have finished, break the retry loop. + Use socketpair() rather than pipe() so we can pass an fd back from the + transport process. + */ while (!pipe_done) { - if (pipe(pfd) == 0) pipe_done = TRUE; - else if (parcount > 0) parmax = parcount; - else break; + if (socketpair(AF_UNIX, SOCK_STREAM, 0, pfd) == 0) pipe_done = TRUE; + else if (parcount > 0) parmax = parcount; + else break; /* We need to make the reading end of the pipe non-blocking. There are two different options for this. Exim is cunningly (I hope!) coded so @@ -4683,6 +4764,32 @@ all pipes, so I do not see a reason to use non-blocking IO here search_tidyup(); +/* +A continued-tpt will, in the tpt parent here, call par_reduce for +the one child. But we are hoping to never do continued-transport... +SO.... we may have called par_reduce for a single child, above when we'd +hit the limit on child-count. Possibly multiple times with different +transports and target hosts. Does it matter if several return a suggested +next-id, and we lose all but the last? Hmm. Less parallel working would +happen. Perhaps still do continued-tpt once one has been set? No, that won't +work for all cases. +BAH. +Could take the initial continued-tpt hit, and then do the next-id thing? + +do_remote_deliveries par_reduce par_wait par_read_pipe +*/ + + /*XXX what about firsttime? */ + if (continue_transport && !exim_lockfile_needed()) + if (!continue_wait_db) + { + continue_wait_db = dbfn_open_multi( + string_sprintf("wait-%.200s", continue_transport), + O_RDWR, + (open_db *) store_get(sizeof(open_db), GET_UNTAINTED)); + continue_next_id[0] = '\0'; + } + if ((pid = exim_fork(US"transport")) == 0) { int fd = pfd[pipe_write]; @@ -4775,7 +4882,11 @@ all pipes, so I do not see a reason to use non-blocking IO here is flagged by an identifying byte, and is then in a fixed format (with strings terminated by zeros), and there is a final terminator at the end. The host information and retry information is all attached to - the first address, so that gets sent at the start. */ + the first address, so that gets sent at the start. + + Result item tags: + A C D H I K L P R S T X Z + */ /* Host unusability information: for most success cases this will be null. */ @@ -4784,7 +4895,7 @@ all pipes, so I do not see a reason to use non-blocking IO here { if (!h->address || h->status < hstatus_unusable) continue; sprintf(CS big_buffer, "%c%c%s", h->status, h->why, h->address); - rmt_dlv_checked_write(fd, 'H', '0', big_buffer, Ustrlen(big_buffer+2) + 3); + rmt_dlv_checked_write(fd, 'H','0', big_buffer, Ustrlen(big_buffer+2) + 3); } /* The number of bytes written. This is the same for each address. Even @@ -4798,23 +4909,24 @@ all pipes, so I do not see a reason to use non-blocking IO here /* Information about what happened to each address. Four item types are used: an optional 'X' item first, for TLS information, then an optional "C" item for any client-auth info followed by 'R' items for any retry settings, - and finally an 'A' item for the remaining data. */ + and finally an 'A' item for the remaining data. The actual recipient address + is not sent but is implicit in the address-chain being handled. */ for(; addr; addr = addr->next) { - uschar *ptr; + uschar * ptr; - /* The certificate verification status goes into the flags */ +#ifndef DISABLE_TLS + /* The certificate verification status goes into the flags, in A0 */ if (tls_out.certificate_verified) setflag(addr, af_cert_verified); -#ifdef SUPPORT_DANE +# ifdef SUPPORT_DANE if (tls_out.dane_verified) setflag(addr, af_dane_verified); -#endif +# endif # ifndef DISABLE_TLS_RESUME if (tls_out.resumption & RESUME_USED) setflag(addr, af_tls_resume); # endif /* Use an X item only if there's something to send */ -#ifndef DISABLE_TLS if (addr->cipher) { ptr = big_buffer + sprintf(CS big_buffer, "%.128s", addr->cipher) + 1; @@ -4971,7 +5083,13 @@ all pipes, so I do not see a reason to use non-blocking IO here #endif /* The rest of the information goes in an 'A0' item. */ - +#ifdef notdef + DEBUG(D_deliver) + debug_printf("%s %s for MAIL\n", + addr->special_action == '=' ? "initial RCPT" + : addr->special_action == '-' ? "additional RCPT" : "?", + addr->address); +#endif sprintf(CS big_buffer, "%c%c", addr->transport_return, addr->special_action); ptr = big_buffer + 2; memcpy(ptr, &addr->basic_errno, sizeof(addr->basic_errno)); @@ -5011,14 +5129,76 @@ all pipes, so I do not see a reason to use non-blocking IO here if (LOGGING(incoming_interface) && sending_ip_address) #endif { - uschar * ptr; - ptr = big_buffer + sprintf(CS big_buffer, "%.128s", sending_ip_address) + 1; + uschar * ptr = big_buffer + + sprintf(CS big_buffer, "%.128s", sending_ip_address) + 1; ptr += sprintf(CS ptr, "%d", sending_port) + 1; rmt_dlv_checked_write(fd, 'I', '0', big_buffer, ptr - big_buffer); } + /* Continuation message-id, if a continuation is for that reason, + and the next sequence number (MAIL FROM count) for the connection. */ + + if (*continue_next_id) + rmt_dlv_checked_write(fd, 'Z', '1', big_buffer, + sprintf(CS big_buffer, "%.*s %u", + MESSAGE_ID_LENGTH, continue_next_id, continue_sequence+1) + 1); + + /* Connection details, only on the first suggested continuation for + wait-db ones, but for all continue-more ones (though any after the + delivery proc has the info are pointless). */ + + if (continue_hostname) + { + { + uschar * ptr = big_buffer; + ptr += sprintf(CS ptr, "%.128s", continue_transport) + 1; + ptr += sprintf(CS ptr, "%.128s", continue_hostname) + 1; + ptr += sprintf(CS ptr, "%.128s", continue_host_address) + 1; + ptr += sprintf(CS ptr, "%u", continue_sequence+1) + 1; + rmt_dlv_checked_write(fd, 'Z', '2', big_buffer, ptr - big_buffer); + send_fd_over_socket(fd, continue_fd); + } + + big_buffer[0] = smtp_peer_options; + big_buffer[1] = f.smtp_authenticated ? 1 : 0; + rmt_dlv_checked_write(fd, 'Z', '3', big_buffer, 2); + + if (tls_out.active.sock >= 0 || continue_proxy_cipher) + rmt_dlv_checked_write(fd, 'Z', '4', big_buffer, + sprintf(CS big_buffer, "%.128s", continue_proxy_cipher) + 1); + + if (tls_out.sni) + rmt_dlv_checked_write(fd, 'Z', +#ifdef SUPPORT_DANE + tls_out.dane_verified ? '5' : '6', +#else + '6', +#endif + tls_out.sni, Ustrlen(tls_out.sni)+1); + +#ifndef DISABLE_ESMTP_LIMITS + if (continue_limit_mail || continue_limit_rcpt || continue_limit_rcptdom) + rmt_dlv_checked_write(fd, 'Z', '7', big_buffer, + sprintf(CS big_buffer, "%u %u %u", + continue_limit_mail, continue_limit_rcpt, + continue_limit_rcptdom) + 1); +#endif + +#ifdef SUPPORT_SOCKS + if (proxy_session) + { + uschar * ptr = big_buffer; + ptr += sprintf(CS ptr, "%.128s", proxy_local_address) + 1; + ptr += sprintf(CS ptr, "%u", proxy_local_port) + 1; + ptr += sprintf(CS ptr, "%.128s", proxy_external_address) + 1; + ptr += sprintf(CS ptr, "%u", proxy_external_port) + 1; + rmt_dlv_checked_write(fd, 'Z', '8', big_buffer, ptr - big_buffer); + } +#endif + } + /* Add termination flag, close the pipe, and that's it. The character - after 'Z' indicates whether continue_transport is now NULL or not. + after "Z0" indicates whether continue_transport is now NULL or not. A change from non-NULL to NULL indicates a problem with a continuing connection. */ @@ -5078,14 +5258,20 @@ all pipes, so I do not see a reason to use non-blocking IO here (continue_transport gets set to NULL) before we consider any other addresses in this message. */ - if (continue_transport) par_reduce(0, fallback); + if (continue_transport) + { + par_reduce(0, fallback); + if (!continue_next_id && continue_wait_db) + { dbfn_close_multi(continue_wait_db); continue_wait_db = NULL; } + } /* Otherwise, if we are running in the test harness, wait a bit, to let the newly created process get going before we create another process. This should ensure repeatability in the tests. Wait long enough for most cases to complete the transport. */ - else testharness_pause_ms(600); + else + testharness_pause_ms(600); continue; @@ -5275,6 +5461,7 @@ if (continue_transport) if (Ustrcmp(t->name, continue_transport) == 0) { if (t->info->closedown) (t->info->closedown)(t); + continue_transport = NULL; break; } return DELIVER_NOT_ATTEMPTED; @@ -5501,16 +5688,14 @@ Returns: nothing */ static void -do_duplicate_check(address_item **anchor) +do_duplicate_check(address_item ** anchor) { -address_item *addr; +address_item * addr; while ((addr = *anchor)) { - tree_node *tnode; + tree_node * tnode; if (testflag(addr, af_pfr)) - { - anchor = &(addr->next); - } + anchor = &addr->next; else if ((tnode = tree_search(tree_duplicates, addr->unique))) { DEBUG(D_deliver|D_route) @@ -5523,7 +5708,7 @@ while ((addr = *anchor)) else { tree_add_duplicate(addr->unique, addr); - anchor = &(addr->next); + anchor = &addr->next; } } } @@ -6433,16 +6618,19 @@ Returns: When the global variable mua_wrapper is FALSE: int deliver_message(const uschar * id, BOOL forced, BOOL give_up) { -int i, rc; -int final_yield = DELIVER_ATTEMPTED_NORMAL; -time_t now = time(NULL); -address_item *addr_last = NULL; -uschar *filter_message = NULL; -int process_recipients = RECIP_ACCEPT; -open_db dbblock; -open_db *dbm_file; +int i, rc, final_yield, process_recipients; +time_t now; +address_item * addr_last; +uschar * filter_message, * info; +open_db dbblock, * dbm_file; extern int acl_where; -uschar *info; + +CONTINUED_ID: +final_yield = DELIVER_ATTEMPTED_NORMAL; +now = time(NULL); +addr_last = NULL; +filter_message = NULL; +process_recipients = RECIP_ACCEPT; #ifdef MEASURE_TIMING report_time_since(×tamp_startup, US"delivery start"); /* testcase 0022, 2100 */ @@ -6517,7 +6705,7 @@ opening the data file, message_subdir gets set. */ if ((deliver_datafile = spool_open_datafile(id)) < 0) return continue_closedown(); /* yields DELIVER_NOT_ATTEMPTED */ -/* tHe value of message_size at this point has been set to the data length, +/* The value of message_size at this point has been set to the data length, plus one for the blank line that notionally precedes the data. */ /* Now read the contents of the header file, which will set up the headers in @@ -7007,10 +7195,7 @@ else if (system_filter && process_recipients != RECIP_FAIL_TIMEOUT) transport_instance *tp; for (tp = transports; tp; tp = tp->next) if (Ustrcmp(tp->name, tpname) == 0) - { - p->transport = tp; - break; - } + { p->transport = tp; break; } if (!tp) p->message = string_sprintf("failed to find \"%s\" transport " "for system filter delivery", tpname); @@ -7245,11 +7430,36 @@ while (addr_new) /* Loop until all addresses dealt with */ address_item * addr, * parent; /* Failure to open the retry database is treated the same as if it does - not exist. In both cases, dbm_file is NULL. */ + not exist. In both cases, dbm_file is NULL. For the first stage of a 2-phase + queue run don't bother checking domain- or address-retry info; they will take + effect on the second stage. */ - if (!(dbm_file = dbfn_open(US"retry", O_RDONLY, &dbblock, FALSE, TRUE))) - DEBUG(D_deliver|D_retry|D_route|D_hints_lookup) - debug_printf("no retry data available\n"); + if (f.queue_2stage) + dbm_file = NULL; + else + { + /* If we have transaction-capable hintsdbs, open the retry db without + locking, and leave open for the transport process and for subsequent + deliveries. If the open fails, tag that explicitly for the transport but + retry the open next time around, in case it was created in the interim. */ + + if (continue_retry_db == (open_db *)-1) + continue_retry_db = NULL; + + if (continue_retry_db) + dbm_file = continue_retry_db; + else if (!exim_lockfile_needed() && continue_transport) + { + dbm_file = dbfn_open_multi(US"retry", O_RDONLY, &dbblock); + continue_retry_db = dbm_file ? dbm_file : (open_db *)-1; + } + else + dbm_file = dbfn_open(US"retry", O_RDONLY, &dbblock, FALSE, TRUE); + + if (!dbm_file) + DEBUG(D_deliver|D_retry|D_route|D_hints_lookup) + debug_printf("no retry data available\n"); + } /* Scan the current batch of new addresses, to handle pipes, files and autoreplies, and determine which others are ready for routing. */ @@ -7657,7 +7867,8 @@ while (addr_new) /* Loop until all addresses dealt with */ /* The database is closed while routing is actually happening. Requests to update it are put on a chain and all processed together at the end. */ - if (dbm_file) dbfn_close(dbm_file); + if (dbm_file && !continue_retry_db) + { dbfn_close(dbm_file); dbm_file = NULL; } /* If queue_domains is set, we don't even want to try routing addresses in those domains. During queue runs, queue_domains is forced to be unset. @@ -7826,8 +8037,10 @@ while (addr_new) /* Loop until all addresses dealt with */ } } /* Continue with routing the next address. */ } /* Loop to process any child addresses that the routers created, and - any rerouted addresses that got put back on the new chain. */ + any rerouted addresses that got put back on the new chain. */ +if (dbm_file) /* Can only be continue_retry_db */ + { dbfn_close_multi(continue_retry_db); continue_retry_db = dbm_file = NULL; } /* Debugging: show the results of the routing */ @@ -8626,6 +8839,17 @@ DEBUG(D_deliver) debug_printf("end delivery of %s\n", id); report_time_since(×tamp_startup, US"delivery end"); /* testcase 0005 */ #endif +/* If the transport suggested another message to deliver, go round again. */ + +if (final_yield == DELIVER_ATTEMPTED_NORMAL && *continue_next_id) + { + addr_defer = addr_failed = addr_succeed = NULL; + tree_duplicates = NULL; /* discard dups info from old message */ + id = string_copyn(continue_next_id, MESSAGE_ID_LENGTH); + continue_next_id[0] = '\0'; + goto CONTINUED_ID; + } + /* It is unlikely that there will be any cached resources, since they are released after routing, and in the delivery subprocesses. However, it's possible for an expansion for something afterwards (for example, diff --git a/src/src/dkim.c b/src/src/dkim.c index e0b76c3b1..68f074889 100644 --- a/src/src/dkim.c +++ b/src/src/dkim.c @@ -342,8 +342,8 @@ for (pdkim_signature * sig = dkim_signatures; sig; sig = sig->next) if (sig->domain) g = string_append_listele(g, ':', sig->domain); if (sig->identity) g = string_append_listele(g, ':', sig->identity); } - -if (g) dkim_signers = g->s; +gstring_release_unused(g); +dkim_signers = string_from_gstring(g); out: store_pool = dkim_verify_oldpool; @@ -358,7 +358,8 @@ dkim_acl_call(uschar * id, gstring ** res_ptr, { int rc; DEBUG(D_receive) - debug_printf("calling acl_smtp_dkim for dkim_cur_signer='%s'\n", id); + debug_printf("calling acl_smtp_dkim for identity '%s' domain '%s' sel '%s'\n", + id, dkim_signing_domain, dkim_signing_selector); rc = acl_check(ACL_WHERE_DKIM, NULL, acl_smtp_dkim, user_msgptr, log_msgptr); dkim_exim_verify_log_sig(dkim_cur_sig); @@ -369,6 +370,7 @@ return rc; /* For the given identity, run the DKIM ACL once for each matching signature. +If none match, run it once. Arguments id Identity to look for in dkim signatures @@ -425,7 +427,8 @@ for (pdkim_signature * sig = dkim_signatures; sig; sig = sig->next) dkim_verify_status = dkim_exim_expand_query(DKIM_VERIFY_STATUS); dkim_verify_reason = dkim_exim_expand_query(DKIM_VERIFY_REASON); - if ((rc = dkim_acl_call(id, res_ptr, user_msgptr, log_msgptr)) != OK) + if ( (rc = dkim_acl_call(id, res_ptr, user_msgptr, log_msgptr)) != OK + || dkim_verify_minimal && Ustrcmp(dkim_verify_status, "pass") == 0) return rc; } diff --git a/src/src/exim.c b/src/src/exim.c index 8111a4489..cca02de5e 100644 --- a/src/src/exim.c +++ b/src/src/exim.c @@ -289,7 +289,8 @@ if (US info->si_addr < US 4096) else log_write(0, LOG_MAIN|LOG_PANIC, "SIGSEGV (maybe attempt to write to immutable memory)"); if (process_info_len > 0) - log_write(0, LOG_MAIN|LOG_PANIC, "SIGSEGV (%.*s)", process_info_len, process_info); + log_write(0, LOG_MAIN|LOG_PANIC, "SIGSEGV (%s: %.*s)", + process_purpose, process_info_len, process_info); stackdump(); signal(SIGSEGV, SIG_DFL); kill(getpid(), sig); diff --git a/src/src/expand.c b/src/src/expand.c index e7d089909..8232ed942 100644 --- a/src/src/expand.c +++ b/src/src/expand.c @@ -22,6 +22,7 @@ typedef unsigned esi_flags; #define ESI_BRACE_ENDS BIT(0) /* expansion should stop at } */ #define ESI_HONOR_DOLLAR BIT(1) /* $ is meaningfull */ #define ESI_SKIPPING BIT(2) /* value will not be needed */ +#define ESI_EXISTS_ONLY BIT(3) /* actual value not needed */ #ifdef STAND_ALONE # ifndef SUPPORT_CRYPTEQ @@ -1919,8 +1920,9 @@ chop. Arguments: name the name of the variable being sought - exists_only TRUE if this is a def: test; passed on to find_header() - skipping TRUE => skip any processing evaluation; this is not the same as + flags + exists_only TRUE if this is a def: test; passed on to find_header() + skipping TRUE => skip any processing evaluation; this is not the same as exists_only because def: may test for values that are first evaluated here newsize pointer to an int which is initially zero; if the answer is in @@ -1932,7 +1934,7 @@ Returns: NULL if the variable does not exist, or */ static const uschar * -find_variable(uschar *name, BOOL exists_only, BOOL skipping, int *newsize) +find_variable(uschar * name, esi_flags flags, int * newsize) { var_entry * vp; uschar *s, *domain; @@ -1990,7 +1992,7 @@ if (!(vp = find_var_ent(name))) /* Found an existing variable. If in skipping state, the value isn't needed, and we want to avoid processing (such as looking up the host name). */ -if (skipping) +if (flags & ESI_SKIPPING) return US""; val = vp->value; @@ -2051,11 +2053,13 @@ switch (vp->type) return domain ? domain + 1 : US""; case vtype_msgheaders: - return find_header(NULL, newsize, exists_only ? FH_EXISTS_ONLY : 0, NULL); + return find_header(NULL, newsize, + flags & ESI_EXISTS_ONLY ? FH_EXISTS_ONLY : 0, NULL); case vtype_msgheaders_raw: return find_header(NULL, newsize, - exists_only ? FH_EXISTS_ONLY|FH_WANT_RAW : FH_WANT_RAW, NULL); + flags & ESI_EXISTS_ONLY ? FH_EXISTS_ONLY|FH_WANT_RAW : FH_WANT_RAW, + NULL); case vtype_msgbody: /* Pointer to msgbody string */ case vtype_msgbody_end: /* Ditto, the end of the msg */ @@ -2122,15 +2126,15 @@ switch (vp->type) case vtype_reply: /* Get reply address */ s = find_header(US"reply-to:", newsize, - exists_only ? FH_EXISTS_ONLY|FH_WANT_RAW : FH_WANT_RAW, - headers_charset); + flags & ESI_EXISTS_ONLY ? FH_EXISTS_ONLY|FH_WANT_RAW : FH_WANT_RAW, + headers_charset); if (s) Uskip_whitespace(&s); if (!s || !*s) { *newsize = 0; /* For the *s==0 case */ s = find_header(US"from:", newsize, - exists_only ? FH_EXISTS_ONLY|FH_WANT_RAW : FH_WANT_RAW, - headers_charset); + flags & ESI_EXISTS_ONLY ? FH_EXISTS_ONLY|FH_WANT_RAW : FH_WANT_RAW, + headers_charset); } if (s) { @@ -2690,7 +2694,8 @@ switch(cond_type = identify_operator(&s, &opname)) else { - if (!(t = find_variable(name, TRUE, yield == NULL, NULL))) + if (!(t = find_variable(name, + yield ? ESI_EXISTS_ONLY : ESI_EXISTS_ONLY | ESI_SKIPPING, NULL))) { expand_string_message = name[0] ? string_sprintf("unknown variable \"%s\" after \"def:\"", name) @@ -4736,7 +4741,7 @@ while (*s) /* Variable */ - else if (!(value = find_variable(name, FALSE, !!(flags & ESI_SKIPPING), &newsize))) + else if (!(value = find_variable(name, flags, &newsize))) { expand_string_message = string_sprintf("unknown variable name \"%s\"", name); @@ -8380,7 +8385,7 @@ NOT_ITEM: ; reset_point = store_mark(); g = store_get(sizeof(gstring), GET_UNTAINTED); /* alloc _before_ calling find_variable() */ } - if (!(value = find_variable(name, FALSE, !!(flags & ESI_SKIPPING), &newsize))) + if (!(value = find_variable(name, flags, &newsize))) { expand_string_message = string_sprintf("unknown variable in \"${%s}\"", name); diff --git a/src/src/functions.h b/src/src/functions.h index cdf97f8bd..afb6fd46f 100644 --- a/src/src/functions.h +++ b/src/src/functions.h @@ -444,6 +444,7 @@ extern BOOL receive_check_set_sender(const uschar *); extern BOOL receive_msg(BOOL); extern int_eximarith_t receive_statvfs(BOOL, int *); extern void receive_swallow_smtp(void); +extern int recv_fd_from_sock(int); #ifdef WITH_CONTENT_SCAN extern int regex(const uschar **, BOOL); extern void regex_vars_clear(void); @@ -497,6 +498,7 @@ extern int search_findtype_partial(const uschar *, int *, const uschar **, i int *, const uschar **); extern void *search_open(const uschar *, int, int, uid_t *, gid_t *); extern void search_tidyup(void); +extern BOOL send_fd_over_socket(int, int); extern uschar *sender_helo_verified_boolstr(void); extern void set_process_info(const char *, ...) PRINTF_FUNCTION(1,2); extern void sha1_end(hctx *, const uschar *, int, uschar *); @@ -633,11 +635,6 @@ extern uschar *transport_current_name(void); extern void transport_do_pass_socket(const uschar *, const uschar *, const uschar *, uschar *, int); extern void transport_init(void); -extern BOOL transport_pass_socket(const uschar *, const uschar *, const uschar *, uschar *, int -#ifndef DISABLE_ESMTP_LIMITS - , unsigned, unsigned, unsigned -#endif - ); extern const uschar *transport_rcpt_address(address_item *, BOOL); extern BOOL transport_set_up_command(const uschar ***, const uschar *, unsigned, int, address_item *, const uschar *, uschar **); @@ -1404,6 +1401,19 @@ return poll(&p, 1, tmo_millisec); /******************************************************************************/ /* Client-side smtp log string, for debug */ +static inline void +smtp_debug_cmd_log_init(void) +{ +# ifndef DISABLE_CLIENT_CMD_LOG +int old_pool = store_pool; +store_pool = POOL_PERM; +client_cmd_log = string_get_tainted(56, GET_TAINTED); +*client_cmd_log->s = '\0'; +store_pool = old_pool; +# endif +} + + static inline void smtp_debug_cmd(const uschar * buf, int mode) { @@ -1412,31 +1422,46 @@ HDEBUG(D_transport|D_acl|D_v) debug_printf_indent(" SMTP%c> %s\n", # ifndef DISABLE_CLIENT_CMD_LOG { - int len = Ustrcspn(buf, " \n"); - int old_pool = store_pool; + int len = Ustrcspn(buf, " \n"), old_pool = store_pool; store_pool = POOL_PERM; /* Main pool ACL allocations eg. callouts get released */ client_cmd_log = string_append_listele_n(client_cmd_log, ':', buf, MIN(len, 8)); if (mode == SCMD_BUFFER) - { client_cmd_log = string_catn(client_cmd_log, US"|", 1); - (void) string_from_gstring(client_cmd_log); - } else if (mode == SCMD_MORE) - { client_cmd_log = string_catn(client_cmd_log, US"+", 1); - (void) string_from_gstring(client_cmd_log); - } store_pool = old_pool; } # endif } +/* This might be called both due to callout and then from delivery. +Use memory that will not be released between those phases. +*/ +static inline void +smtp_debug_resp(const uschar * buf) +{ +# ifndef DISABLE_CLIENT_CMD_LOG +int old_pool = store_pool; +store_pool = POOL_PERM; +client_cmd_log = string_append_listele_n(client_cmd_log, ':', buf, + buf[3] == '-' ? 4 : 3); +store_pool = old_pool; +# endif +} + + static inline void smtp_debug_cmd_report(void) { # ifndef DISABLE_CLIENT_CMD_LOG -debug_printf("cmdlog: '%s'\n", client_cmd_log ? client_cmd_log->s : US"(unset)"); +if (client_cmd_log && *client_cmd_log->s) + { + debug_printf("cmdlog: '%Y'\n", client_cmd_log); + gstring_reset(client_cmd_log); + } +else + debug_printf("cmdlog: (unset)\n"); # endif } diff --git a/src/src/globals.c b/src/src/globals.c index c50b7a42e..a4b142bc0 100644 --- a/src/src/globals.c +++ b/src/src/globals.c @@ -741,17 +741,24 @@ uid_t config_uid = 0; uint64_t connection_id = 0L; int connection_max_messages= -1; +unsigned continue_flags = 0; +#ifndef DISABLE_ESMTP_LIMITS +unsigned continue_limit_mail = 0; +unsigned continue_limit_rcpt = 0; +unsigned continue_limit_rcptdom= 0; +int continue_fd = -1; uschar *continue_proxy_cipher = NULL; BOOL continue_proxy_dane = FALSE; uschar *continue_proxy_sni = NULL; -uschar *continue_hostname = NULL; -uschar *continue_host_address = NULL; +const uschar *continue_hostname = NULL; +const uschar *continue_host_address = NULL; +uschar continue_next_id[MESSAGE_ID_LENGTH +1] = {[0]='\0'}; int continue_sequence = 1; uschar *continue_transport = NULL; -#ifndef DISABLE_ESMTP_LIMITS -unsigned continue_limit_mail = 0; -unsigned continue_limit_rcpt = 0; -unsigned continue_limit_rcptdom= 0; +#ifndef COMPILE_UTILITY +open_db *continue_retry_db = NULL; +open_db *continue_wait_db = NULL; +#endif #endif uschar *csa_status = NULL; diff --git a/src/src/globals.h b/src/src/globals.h index dc9d384db..05c39109e 100644 --- a/src/src/globals.h +++ b/src/src/globals.h @@ -445,17 +445,24 @@ extern const uschar *config_main_filelist; /* List of possible config files */ extern uschar *config_main_filename; /* File name actually used */ extern uschar *config_main_directory; /* Directory where the main config file was found */ extern uid_t config_uid; /* Additional owner */ +extern unsigned continue_flags; /* TLS-related info for connection */ +#ifndef DISABLE_ESMTP_LIMITS +extern unsigned continue_limit_mail; /* Peer advertised limit */ +extern unsigned continue_limit_rcpt; +extern unsigned continue_limit_rcptdom; +#endif +extern int continue_fd; /* Connection for continuation */ extern uschar *continue_proxy_cipher; /* TLS cipher for proxied continued delivery */ extern BOOL continue_proxy_dane; /* proxied conn is DANE */ extern uschar *continue_proxy_sni; /* proxied conn SNI */ -extern uschar *continue_hostname; /* Host for continued delivery */ -extern uschar *continue_host_address; /* IP address for ditto */ +extern const uschar *continue_hostname; /* Host for continued delivery */ +extern const uschar *continue_host_address; /* IP address for ditto */ +extern uschar continue_next_id[]; /* Next message_id from hintsdb */ extern int continue_sequence; /* Sequence num for continued delivery */ extern uschar *continue_transport; /* Transport for continued delivery */ -#ifndef DISABLE_ESMTP_LIMITS -extern unsigned continue_limit_mail; /* Peer advertised limit */ -extern unsigned continue_limit_rcpt; -extern unsigned continue_limit_rcptdom; +#ifndef COMPILE_UTILITY +extern open_db *continue_retry_db; /* Hintsdb for retries */ +extern open_db *continue_wait_db; /* Hintsdb for wait-transport */ #endif diff --git a/src/src/hintsdb.h b/src/src/hintsdb.h index a6555ed25..ba50ae1b9 100644 --- a/src/src/hintsdb.h +++ b/src/src/hintsdb.h @@ -60,7 +60,7 @@ Future: consider re-architecting to support caching of the open-handle for hintsdb uses (the dbmdb use gets that already). This would need APIs for transaction locks. Perhaps merge the implementation with the lookups layer, in some way, for the open-handle caching (since that manages closes -required by Exim's process transisitions)? +required by Exim's process transitions)? */ #ifndef HINTSDB_H @@ -71,1078 +71,32 @@ required by Exim's process transisitions)? # if defined(USE_DB) || defined(USE_GDBM) || defined(USE_TDB) # error USE_SQLITE conflict with alternate definition # endif - -/* ********************* sqlite3 interface ************************ */ - -# include - -/* Basic DB type */ -# define EXIM_DB sqlite3 - -# define EXIM_CURSOR int - -# /* The datum type used for queries */ -# define EXIM_DATUM blob - -/* Some text for messages */ -# define EXIM_DBTYPE "sqlite3" - -# /* Access functions */ - -static inline BOOL -exim_lockfile_needed(void) -{ -return FALSE; /* We do transaction; no extra locking needed */ -} - -/* EXIM_DBOPEN - return pointer to an EXIM_DB, NULL if failed */ -static inline EXIM_DB * -exim_dbopen__(const uschar * name, const uschar * dirname, int flags, - unsigned mode) -{ -EXIM_DB * dbp; -int ret, sflags = flags & O_RDWR ? SQLITE_OPEN_READWRITE : SQLITE_OPEN_READONLY; -if (flags & O_CREAT) sflags |= SQLITE_OPEN_CREATE; -if ((ret = sqlite3_open_v2(CCS name, &dbp, sflags, NULL)) == SQLITE_OK) - { - sqlite3_busy_timeout(dbp, 5000); - ret = sqlite3_exec(dbp, "BEGIN TRANSACTION;", NULL, NULL, NULL); - if (ret == SQLITE_OK && flags & O_CREAT) - ret = sqlite3_exec(dbp, - "CREATE TABLE IF NOT EXISTS tbl (ky TEXT PRIMARY KEY, dat BLOB);", - NULL, NULL, NULL); - if (ret != SQLITE_OK) - sqlite3_close(dbp); - } -//else -// fprintf(stderr, "sqlite3_open_v2: %s\n", sqlite3_errmsg(dbp)); -return ret == SQLITE_OK ? dbp : NULL; -} - -/* EXIM_DBGET - returns TRUE if successful, FALSE otherwise */ -/* note we alloc'n'copy - the caller need not do so */ -/* result has a NUL appended, but the length is as per the DB */ - -static inline BOOL -exim_dbget__(EXIM_DB * dbp, const uschar * s, EXIM_DATUM * res) -{ -sqlite3_stmt * statement; -int ret; - -res->len = (size_t) -1; -/* fprintf(stderr, "exim_dbget__(%s)\n", s); */ -if ((ret = sqlite3_prepare_v2(dbp, CCS s, -1, &statement, NULL)) != SQLITE_OK) - { -/* fprintf(stderr, "prepare fail: %s\n", sqlite3_errmsg(dbp)); */ - return FALSE; - } -if (sqlite3_step(statement) != SQLITE_ROW) - { -/* fprintf(stderr, "step fail: %s\n", sqlite3_errmsg(dbp)); */ - sqlite3_finalize(statement); - return FALSE; - } - -res->len = sqlite3_column_bytes(statement, 0); -# ifdef COMPILE_UTILITY -if (!(res->data = malloc(res->len +1))) - { sqlite3_finalize(statement); return FALSE; } -# else -res->data = store_get(res->len +1, GET_TAINTED); -# endif -memcpy(res->data, sqlite3_column_blob(statement, 0), res->len); -res->data[res->len] = '\0'; -/* fprintf(stderr, "res %d bytes: '%.*s'\n", (int)res->len, (int)res->len, res->data); */ -sqlite3_finalize(statement); -return TRUE; -} - -static inline BOOL -exim_dbget(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * res) -{ -# define FMT "SELECT dat FROM tbl WHERE ky = '%.*s';" -uschar * qry; -int i; -BOOL ret; - -# ifdef COMPILE_UTILITY -/* fprintf(stderr, "exim_dbget(k len %d '%.*s')\n", (int)key->len, (int)key->len, key->data); */ -i = snprintf(NULL, 0, FMT, (int) key->len, key->data)+1; -if (!(qry = malloc(i))) - return FALSE; -snprintf(CS qry, i, FMT, (int) key->len, key->data); -ret = exim_dbget__(dbp, qry, res); -free(qry); -# else -/* fprintf(stderr, "exim_dbget(k len %d '%.*s')\n", (int)key->len, (int)key->len, key->data); */ -qry = string_sprintf(FMT, (int) key->len, key->data); -ret = exim_dbget__(dbp, qry, res); -# endif - -return ret; -# undef FMT -} - -/**/ -# define EXIM_DBPUTB_OK 0 -# define EXIM_DBPUTB_DUP (-1) - -static inline int -exim_s_dbp(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * data, const uschar * alt) -{ -int hlen = data->len * 2, off = 0, res; -# define FMT "INSERT OR %s INTO tbl (ky,dat) VALUES ('%.*s', X'%.*s');" -uschar * qry; -# ifdef COMPILE_UTILITY -uschar * hex = malloc(hlen+1); -if (!hex) return EXIM_DBPUTB_DUP; /* best we can do */ -# else -uschar * hex = store_get(hlen+1, data->data); -# endif - -for (const uschar * s = data->data, * t = s + data->len; s < t; s++, off += 2) - sprintf(CS hex + off, "%02X", *s); - -# ifdef COMPILE_UTILITY -res = snprintf(CS hex, 0, FMT, alt, (int) key->len, key->data, hlen, hex) +1; -if (!(qry = malloc(res))) return EXIM_DBPUTB_DUP; -snprintf(CS qry, res, FMT, alt, (int) key->len, key->data, hlen, hex); -/* fprintf(stderr, "exim_s_dbp(%s)\n", qry); */ -res = sqlite3_exec(dbp, CS qry, NULL, NULL, NULL); -free(qry); -free(hex); -# else -qry = string_sprintf(FMT, alt, (int) key->len, key->data, hlen, hex); -/* fprintf(stderr, "exim_s_dbp(%s)\n", qry); */ -res = sqlite3_exec(dbp, CS qry, NULL, NULL, NULL); -/* fprintf(stderr, "exim_s_dbp res %d\n", res); */ -# endif - -if (res != SQLITE_OK) - fprintf(stderr, "sqlite3_exec: %s\n", sqlite3_errmsg(dbp)); - -return res == SQLITE_OK ? EXIM_DBPUTB_OK : EXIM_DBPUTB_DUP; -# undef FMT -} - -/* EXIM_DBPUT - returns nothing useful, assumes replace mode */ - -static inline int -exim_dbput(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * data) -{ -/* fprintf(stderr, "exim_dbput()\n"); */ -(void) exim_s_dbp(dbp, key, data, US"REPLACE"); -return 0; -} - -/* EXIM_DBPUTB - non-overwriting for use by dbmbuild */ - -/* Returns from EXIM_DBPUTB */ - -static inline int -exim_dbputb(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * data) -{ -return exim_s_dbp(dbp, key, data, US"ABORT"); -} - -/* EXIM_DBDEL */ -static inline int -exim_dbdel(EXIM_DB * dbp, EXIM_DATUM * key) -{ -# define FMT "DELETE FROM tbl WHERE ky = '%.*s';" -uschar * qry; -int res; - -# ifdef COMPILE_UTILITY -res = snprintf(NULL, 0, FMT, (int) key->len, key->data) +1; /* res includes nul */ -if (!(qry = malloc(res))) return SQLITE_NOMEM; -snprintf(CS qry, res, FMT, (int) key->len, key->data); -res = sqlite3_exec(dbp, CS qry, NULL, NULL, NULL); -free(qry); -# else -qry = string_sprintf(FMT, (int) key->len, key->data); -res = sqlite3_exec(dbp, CS qry, NULL, NULL, NULL); -# endif - -return res; -# undef FMT -} - - -/* EXIM_DBCREATE_CURSOR - initialize for scanning operation */ -/* Cursors are inefficiently emulated by repeating searches */ - -static inline EXIM_CURSOR * -exim_dbcreate_cursor(EXIM_DB * dbp) -{ -# ifdef COMPILE_UTILITY -EXIM_CURSOR * c = malloc(sizeof(int)); -if (!c) return NULL; -# else -EXIM_CURSOR * c = store_malloc(sizeof(int)); -# endif -*c = 0; -return c; -} - -/* EXIM_DBSCAN */ -/* Note that we return the (next) key, not the record value */ -static inline BOOL -exim_dbscan(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * res, BOOL first, - EXIM_CURSOR * cursor) -{ -# define FMT "SELECT ky FROM tbl ORDER BY ky LIMIT 1 OFFSET %d;" -uschar * qry; -int i; -BOOL ret; - -# ifdef COMPILE_UTILITY -i = snprintf(NULL, 0, FMT, *cursor)+1; -if (!(qry = malloc(i))) return FALSE; -snprintf(CS qry, i, FMT, *cursor); -/* fprintf(stderr, "exim_dbscan(%s)\n", qry); */ -ret = exim_dbget__(dbp, qry, key); -free(qry); -/* fprintf(stderr, "exim_dbscan ret %c\n", ret ? 'T':'F'); */ -# else -qry = string_sprintf(FMT, *cursor); -/* fprintf(stderr, "exim_dbscan(%s)\n", qry); */ -ret = exim_dbget__(dbp, qry, key); -/* fprintf(stderr, "exim_dbscan ret %c\n", ret ? 'T':'F'); */ -# endif -if (ret) *cursor = *cursor + 1; -return ret; -# undef FMT -} - -/* EXIM_DBDELETE_CURSOR - terminate scanning operation. */ -static inline void -exim_dbdelete_cursor(EXIM_CURSOR * cursor) -{ -# ifdef COMPILE_UTILITY -free(cursor); -# else -store_free(cursor); -# endif -} - - -/* EXIM_DBCLOSE */ -static void -exim_dbclose__(EXIM_DB * dbp) -{ -(void) sqlite3_exec(dbp, "COMMIT TRANSACTION;", NULL, NULL, NULL); -sqlite3_close(dbp); -} - - -/* Datum access */ - -static uschar * -exim_datum_data_get(EXIM_DATUM * dp) -{ return US dp->data; } -static void -exim_datum_data_set(EXIM_DATUM * dp, void * s) -{ dp->data = s; } - -static unsigned -exim_datum_size_get(EXIM_DATUM * dp) -{ return dp->len; } -static void -exim_datum_size_set(EXIM_DATUM * dp, unsigned n) -{ dp->len = n; } - - - -static inline void -exim_datum_init(EXIM_DATUM * dp) -{ dp->data = NULL; } /* compiler quietening */ - -/* No free needed for a datum */ - -static inline void -exim_datum_free(EXIM_DATUM * dp) -{ } - -/* size limit */ - -# define EXIM_DB_RLIMIT 150 - - - - - - +# include "hintsdb/hints_sqlite.h" #elif defined(USE_TDB) # if defined(USE_DB) || defined(USE_GDBM) || defined(USE_SQLITE) # error USE_TDB conflict with alternate definition # endif - -/* ************************* tdb interface ************************ */ -/*XXX https://manpages.org/tdb/3 mentions concurrent writes. -Could we lose the file lock? */ - -# include - -/* Basic DB type */ -# define EXIM_DB TDB_CONTEXT - -/* Cursor type: tdb uses the previous "key" in _nextkey() (really it wants -tdb_traverse to be called) */ -# define EXIM_CURSOR TDB_DATA - -/* The datum type used for queries */ -# define EXIM_DATUM TDB_DATA - -/* Some text for messages */ -# define EXIM_DBTYPE "tdb" - -/* Access functions */ - -static inline BOOL -exim_lockfile_needed(void) -{ -return TRUE; -} - -/* EXIM_DBOPEN - return pointer to an EXIM_DB, NULL if failed */ -static inline EXIM_DB * -exim_dbopen__(const uschar * name, const uschar * dirname, int flags, - unsigned mode) -{ -return tdb_open(CS name, 0, TDB_DEFAULT, flags, mode); -} - -/* EXIM_DBGET - returns TRUE if successful, FALSE otherwise */ -static inline BOOL -exim_dbget(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * res) -{ -*res = tdb_fetch(dbp, *key); /* A struct arg and return!! */ -return res->dptr != NULL; -} - -/* EXIM_DBPUT - returns nothing useful, assumes replace mode */ -static inline int -exim_dbput(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * data) -{ return tdb_store(dbp, *key, *data, TDB_REPLACE); } - -/* EXIM_DBPUTB - non-overwriting for use by dbmbuild */ -static inline int -exim_dbputb(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * data) -{ return tdb_store(dbp, *key, *data, TDB_INSERT); } - -/* Returns from EXIM_DBPUTB */ - -# define EXIM_DBPUTB_OK 0 -# define EXIM_DBPUTB_DUP (-1) - -/* EXIM_DBDEL */ -static inline int -exim_dbdel(EXIM_DB * dbp, EXIM_DATUM * key) -{ return tdb_delete(dbp, *key); } - -/* EXIM_DBCREATE_CURSOR - initialize for scanning operation */ -static inline EXIM_CURSOR * -exim_dbcreate_cursor(EXIM_DB * dbp) -{ -# ifdef COMPILE_UTILITY -EXIM_CURSOR * c = malloc(sizeof(TDB_DATA)); -# else -EXIM_CURSOR * c = store_malloc(sizeof(TDB_DATA)); -# endif -c->dptr = NULL; -return c; -} - -/* EXIM_DBSCAN - This is complicated because we have to free the last datum -free() must not die when passed NULL */ - -static inline BOOL -exim_dbscan(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * res, BOOL first, - EXIM_CURSOR * cursor) -{ -*key = first ? tdb_firstkey(dbp) : tdb_nextkey(dbp, *cursor); -free(cursor->dptr); -*cursor = *key; -return key->dptr != NULL; -} - -/* EXIM_DBDELETE_CURSOR - terminate scanning operation. */ -static inline void -exim_dbdelete_cursor(EXIM_CURSOR * cursor) -{ store_free(cursor); } - -/* EXIM_DBCLOSE */ -static inline void -exim_dbclose__(EXIM_DB * db) -{ tdb_close(db); } - -/* Datum access */ - -static inline uschar * -exim_datum_data_get(EXIM_DATUM * dp) -{ return US dp->dptr; } -static inline void -exim_datum_data_set(EXIM_DATUM * dp, void * s) -{ dp->dptr = s; } - -static inline unsigned -exim_datum_size_get(EXIM_DATUM * dp) -{ return dp->dsize; } -static inline void -exim_datum_size_set(EXIM_DATUM * dp, unsigned n) -{ dp->dsize = n; } - -/* No initialization is needed. */ - -static inline void -exim_datum_init(EXIM_DATUM * d) -{ } - -/* Free the stuff inside the datum. */ - -static inline void -exim_datum_free(EXIM_DATUM * d) -{ -free(d->dptr); -d->dptr = NULL; -} - -/* size limit */ - -# define EXIM_DB_RLIMIT 150 - - - - - - -/********************* Berkeley db native definitions **********************/ +# include "hintsdb/hints_tdb.h" #elif defined USE_DB - # if defined(USE_TDB) || defined(USE_GDBM) || defined(USE_SQLITE) # error USE_DB conflict with alternate definition # endif - -# include - -/* 1.x did no locking - 2.x had facilities, but exim does it's own - 3.x+ unknown -*/ - -/* We can distinguish between versions 1.x and 2.x/3.x by looking for a -definition of DB_VERSION_STRING, which is present in versions 2.x onwards. */ - -# ifdef DB_VERSION_STRING - -# if DB_VERSION_MAJOR >= 6 -# error Version 6 and later BDB API is not supported -# endif - -/* The API changed (again!) between the 2.x and 3.x versions */ - -# if DB_VERSION_MAJOR >= 3 - -/***************** Berkeley db 3.x/4.x native definitions ******************/ - -/* Basic DB type */ -# if DB_VERSION_MAJOR > 4 || (DB_VERSION_MAJOR == 4 && DB_VERSION_MINOR >= 1) -# define EXIM_DB DB_ENV -/* Cursor type, for scanning */ -# define EXIM_CURSOR DBC - -/* The datum type used for queries */ -# define EXIM_DATUM DBT - -/* Some text for messages */ -# define EXIM_DBTYPE "db (v4.1+)" - -/* Only more-recent versions. 5+ ? */ -# ifndef DB_FORCESYNC -# define DB_FORCESYNC 0 -# endif - -/* Error callback */ -/* For Berkeley DB >= 2, we can define a function to be called in case of DB -errors. This should help with debugging strange DB problems, e.g. getting "File -exists" when you try to open a db file. The API for this function was changed -at DB release 4.3. */ - -static inline void -dbfn_bdb_error_callback(const DB_ENV * dbenv, const char * pfx, const char * msg) -{ -#ifndef MACRO_PREDEF -log_write(0, LOG_MAIN, "Berkeley DB error: %s", msg); -#endif -} - - - -/* Access functions (BDB 4.1+) */ - -static inline BOOL -exim_lockfile_needed(void) -{ -return TRUE; -} - -/* EXIM_DBOPEN - return pointer to an EXIM_DB, NULL if failed */ -/* The API changed for DB 4.1. - and we also starting using the "env" with a -specified working dir, to avoid the DBCONFIG file trap. */ - -# define ENV_TO_DB(env) ((DB *)(((EXIM_DB *)env)->app_private)) - -static inline EXIM_DB * -exim_dbopen__(const uschar * name, const uschar * dirname, int flags, - unsigned mode) -{ -EXIM_DB * dbp; -DB * b; -if ( db_env_create(&dbp, 0) != 0 - || (dbp->set_errcall(dbp, dbfn_bdb_error_callback), 0) - || dbp->open(dbp, CS dirname, DB_CREATE|DB_INIT_MPOOL|DB_PRIVATE, 0) != 0 - ) - return NULL; -if (db_create(&b, dbp, 0) == 0) - { - dbp->app_private = b; - if (b->open(b, NULL, CS name, NULL, - flags & O_CREAT ? DB_HASH : DB_UNKNOWN, - flags & O_CREAT ? DB_CREATE - : flags & (O_WRONLY|O_RDWR) ? 0 : DB_RDONLY, - mode) == 0 - ) - return dbp; - - b->close(b, 0); - } -dbp->close(dbp, 0); -return NULL; -} - -/* EXIM_DBGET - returns TRUE if successful, FALSE otherwise */ -static inline BOOL -exim_dbget(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * res) -{ -DB * b = ENV_TO_DB(dbp); -return b->get(b, NULL, key, res, 0) == 0; -} - -/* EXIM_DBPUT - returns nothing useful, assumes replace mode */ -static inline int -exim_dbput(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * data) -{ -DB * b = ENV_TO_DB(dbp); -return b->put(b, NULL, key, data, 0); -} - -/* EXIM_DBPUTB - non-overwriting for use by dbmbuild */ -static inline int -exim_dbputb(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * data) -{ -DB * b = ENV_TO_DB(dbp); -return b->put(b, NULL, key, data, DB_NOOVERWRITE); -} - -/* Return values from EXIM_DBPUTB */ - -# define EXIM_DBPUTB_OK 0 -# define EXIM_DBPUTB_DUP DB_KEYEXIST - -/* EXIM_DBDEL */ -static inline int -exim_dbdel(EXIM_DB * dbp, EXIM_DATUM * key) -{ -DB * b = ENV_TO_DB(dbp); -return b->del(b, NULL, key, 0); -} - -/* EXIM_DBCREATE_CURSOR - initialize for scanning operation */ - -static inline EXIM_CURSOR * -exim_dbcreate_cursor(EXIM_DB * dbp) -{ -DB * b = ENV_TO_DB(dbp); -EXIM_CURSOR * c; -b->cursor(b, NULL, &c, 0); -return c; -} - -/* EXIM_DBSCAN - returns TRUE if data is returned, FALSE at end */ -static inline BOOL -exim_dbscan(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * data, BOOL first, - EXIM_CURSOR * cursor) -{ -return cursor->c_get(cursor, key, data, first ? DB_FIRST : DB_NEXT) == 0; -} - -/* EXIM_DBDELETE_CURSOR - terminate scanning operation */ -static inline void -exim_dbdelete_cursor(EXIM_CURSOR * cursor) -{ cursor->c_close(cursor); } - -/* EXIM_DBCLOSE */ -static inline void -exim_dbclose__(EXIM_DB * dbp_o) -{ -DB_ENV * dbp = dbp_o; -DB * b = ENV_TO_DB(dbp); -b->close(b, 0); -dbp->close(dbp, DB_FORCESYNC); -} - -/* Datum access */ - -static inline uschar * -exim_datum_data_get(EXIM_DATUM * dp) -{ return dp->data; } -static inline void -exim_datum_data_set(EXIM_DATUM * dp, void * s) -{ dp->data = s; } - -static inline unsigned -exim_datum_size_get(EXIM_DATUM * dp) -{ return dp->size; } -static inline void -exim_datum_size_set(EXIM_DATUM * dp, unsigned n) -{ dp->size = n; } - -/* The whole datum structure contains other fields that must be cleared -before use, but we don't have to free anything after reading data. */ - -static inline void -exim_datum_init(EXIM_DATUM * d) -{ memset(d, 0, sizeof(*d)); } - -static inline void -exim_datum_free(EXIM_DATUM * d) -{ } - -# else /* pre- 4.1 */ - -# define EXIM_DB DB - -/* Cursor type, for scanning */ -# define EXIM_CURSOR DBC - -/* The datum type used for queries */ -# define EXIM_DATUM DBT - -/* Some text for messages */ -# define EXIM_DBTYPE "db (v3/4)" - -/* Access functions (BDB 3/4) */ - -static inline BOOL -exim_lockfile_needed(void) -{ -return TRUE; -} - -/* EXIM_DBOPEN - return pointer to an EXIM_DB, NULL if failed */ -static inline EXIM_DB * -exim_dbopen__(const uschar * name, const uschar * dirname, int flags, - unsigned mode) -{ -EXIM_DB * dbp; -return db_create(&dbp, NULL, 0) == 0 - && ( dbp->set_errcall(dbp, dbfn_bdb_error_callback), - dbp->open(dbp, CS name, NULL, - flags & O_CREAT ? DB_HASH : DB_UNKNOWN, - flags & O_CREAT ? DB_CREATE - : flags & (O_WRONLY|O_RDWR) ? 0 : DB_RDONLY, - mode) - ) == 0 - ? dbp : NULL; -} - -/* EXIM_DBGET - returns TRUE if successful, FALSE otherwise */ -static inline BOOL -exim_dbget(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * res) -{ return dbp->get(dbp, NULL, key, res, 0) == 0; } - -/* EXIM_DBPUT - returns nothing useful, assumes replace mode */ -static inline int -exim_dbput(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * data) -{ return dbp->put(dbp, NULL, key, data, 0); } - -/* EXIM_DBPUTB - non-overwriting for use by dbmbuild */ -static inline int -exim_dbputb(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * data) -{ return dbp->put(dbp, NULL, key, data, DB_NOOVERWRITE); } - -/* Return values from EXIM_DBPUTB */ - -# define EXIM_DBPUTB_OK 0 -# define EXIM_DBPUTB_DUP DB_KEYEXIST - -/* EXIM_DBDEL */ -static inline int -exim_dbdel(EXIM_DB * dbp, EXIM_DATUM * key) -{ return dbp->del(dbp, NULL, key, 0); } - -/* EXIM_DBCREATE_CURSOR - initialize for scanning operation */ - -static inline EXIM_CURSOR * -exim_dbcreate_cursor(EXIM_DB * dbp) -{ -EXIM_CURSOR * c; -dbp->cursor(dbp, NULL, &c, 0); -return c; -} - -/* EXIM_DBSCAN - returns TRUE if data is returned, FALSE at end */ -static inline BOOL -exim_dbscan(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * data, BOOL first, - EXIM_CURSOR * cursor) -{ -return cursor->c_get(cursor, key, data, first ? DB_FIRST : DB_NEXT) == 0; -} - -/* EXIM_DBDELETE_CURSOR - terminate scanning operation */ -static inline void -exim_dbdelete_cursor(EXIM_CURSOR * cursor) -{ cursor->c_close(cursor); } - -/* EXIM_DBCLOSE */ -static inline void -exim_dbclose__(EXIM_DB * dbp) -{ dbp->close(dbp, 0); } - -/* Datum access */ - -static inline uschar * -exim_datum_data_get(EXIM_DATUM * dp) -{ return US dp->dptr; } -static inline void -exim_datum_data_set(EXIM_DATUM * dp, void * s) -{ dp->dptr = s; } - -static inline uschar * -exim_datum_size_get(EXIM_DATUM * dp) -{ return US dp->size; } -static inline void -exim_datum_size_set(EXIM_DATUM * dp, uschar * s) -{ dp->size = CS s; } - -/* The whole datum structure contains other fields that must be cleared -before use, but we don't have to free anything after reading data. */ - -static inline void -exim_datum_init(EXIM_DATUM * d) -{ memset(d, 0, sizeof(*d)); } - -static inline void -exim_datum_free(EXIM_DATUM * d) -{ } - -# endif - - -# else /* DB_VERSION_MAJOR >= 3 */ -# error Berkeley DB versions earlier than 3 are not supported */ -# endif /* DB_VERSION_MAJOR */ -# else -# error Berkeley DB version 1 is no longer supported -# endif /* DB_VERSION_STRING */ - - -/* all BDB versions */ -/* size limit */ - -# define EXIM_DB_RLIMIT 150 - - - - - - -/********************* gdbm interface definitions **********************/ +# include "hintsdb/hints_bdb.h" #elif defined USE_GDBM -/*XXX TODO: exim's lockfile not needed? */ - # if defined(USE_TDB) || defined(USE_DB) || defined(USE_SQLITE) # error USE_GDBM conflict with alternate definition # endif +# include "hintsdb/hints_gdbm.h" -# include - -/* Basic DB type */ -typedef struct { - GDBM_FILE gdbm; /* Database */ - datum lkey; /* Last key, for scans */ -} EXIM_DB; - -/* Cursor type, not used with gdbm: just set up a dummy */ -# define EXIM_CURSOR int - -/* The datum type used for queries */ -# define EXIM_DATUM datum - -/* Some text for messages */ - -# define EXIM_DBTYPE "gdbm" +#else -/* Access functions (gdbm) */ - -static inline BOOL -exim_lockfile_needed(void) -{ -return TRUE; -} - -/* EXIM_DBOPEN - return pointer to an EXIM_DB, NULL if failed */ -static inline EXIM_DB * -exim_dbopen__(const uschar * name, const uschar * dirname, int flags, - unsigned mode) -{ -EXIM_DB * dbp = malloc(sizeof(EXIM_DB)); /*XXX why not exim mem-mgmt? */ -if (dbp) - { - dbp->lkey.dptr = NULL; - dbp->gdbm = gdbm_open(CS name, 0, - flags & O_CREAT ? GDBM_WRCREAT - : flags & (O_RDWR|O_WRONLY) ? GDBM_WRITER : GDBM_READER, - mode, 0); - if (dbp->gdbm) return dbp; - free(dbp); - } -return NULL; -} - -/* EXIM_DBGET - returns TRUE if successful, FALSE otherwise */ -static inline BOOL -exim_dbget(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * res) -{ -*res = gdbm_fetch(dbp->gdbm, *key); /* A struct arg & return! */ -return res->dptr != NULL; -} - -/* EXIM_DBPUT - returns nothing useful, assumes replace mode */ -static inline int -exim_dbput(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * data) -{ return gdbm_store(dbp->gdbm, *key, *data, GDBM_REPLACE); } - -/* EXIM_DBPUTB - non-overwriting for use by dbmbuild */ -static inline int -exim_dbputb(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * data) -{ return gdbm_store(dbp->gdbm, *key, *data, GDBM_INSERT); } - -/* Returns from EXIM_DBPUTB */ - -# define EXIM_DBPUTB_OK 0 -# define EXIM_DBPUTB_DUP 1 - -/* EXIM_DBDEL */ -static inline int -exim_dbdel(EXIM_DB * dbp, EXIM_DATUM * key) -{ return gdbm_delete(dbp->gdbm, *key); } - -/* EXIM_DBCREATE_CURSOR - initialize for scanning operation (null) */ -static inline EXIM_CURSOR * -exim_dbcreate_cursor(EXIM_DB * dbp) -{ return NULL; } - -/* EXIM_DBSCAN */ -static inline BOOL -exim_dbscan(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * data, BOOL first, - EXIM_CURSOR * cursor) -{ -char * s; -*key = first ? gdbm_firstkey(dbp->gdbm) : gdbm_nextkey(dbp->gdbm, dbp->lkey); -if ((s = dbp->lkey.dptr)) free(s); -dbp->lkey = *key; -return key->dptr != NULL; -} - -/* EXIM_DBDELETE_CURSOR - terminate scanning operation (null). */ -static inline void -exim_dbdelete_cursor(EXIM_CURSOR * cursor) -{ } - -/* EXIM_DBCLOSE */ -static inline void -exim_dbclose__(EXIM_DB * dbp) -{ -char * s; -gdbm_close(dbp->gdbm); -if ((s = dbp->lkey.dptr)) free(s); -free(dbp); -} - -/* Datum access types */ - -static inline uschar * -exim_datum_data_get(EXIM_DATUM * dp) -{ return US dp->dptr; } -static inline void -exim_datum_data_set(EXIM_DATUM * dp, void * s) -{ dp->dptr = s; } - -static inline unsigned -exim_datum_size_get(EXIM_DATUM * dp) -{ return dp->dsize; } -static inline void -exim_datum_size_set(EXIM_DATUM * dp, unsigned n) -{ dp->dsize = n; } - -/* There's no clearing required before use, but we have to free the dptr -after reading data. */ - -static inline void -exim_datum_init(EXIM_DATUM * d) -{ } - -static inline void -exim_datum_free(EXIM_DATUM * d) -{ free(d->dptr); } - -/* size limit. GDBM is int-max limited, but we want to be less silly */ - -# define EXIM_DB_RLIMIT 150 - -#else /* USE_GDBM */ - - - - - - -/* If none of USE_DB, USG_GDBM, USE_SQLITE or USE_TDB are set, +/* If none of USE_{DB,GDBM,SQLITE,TDB} are set the default is the NDBM interface (which seems to be a wrapper for GDBM) */ - -/********************* ndbm interface definitions **********************/ - -# include - -/* Basic DB type */ -# define EXIM_DB DBM - -/* Cursor type, not used with ndbm: just set up a dummy */ -# define EXIM_CURSOR int - -/* The datum type used for queries */ -# define EXIM_DATUM datum - -/* Some text for messages */ - -# define EXIM_DBTYPE "ndbm" - -/* Access functions (ndbm) */ - -static inline BOOL -exim_lockfile_needed(void) -{ -return TRUE; -} - -/* EXIM_DBOPEN - returns a EXIM_DB *, NULL if failed */ -/* Check that the name given is not present. This catches -a directory name; otherwise we would create the name.pag and -name.dir files in the directory's parent. */ - -static inline EXIM_DB * -exim_dbopen__(const uschar * name, const uschar * dirname, int flags, - unsigned mode) -{ -struct stat st; -if (!(flags & O_CREAT) || lstat(CCS name, &st) != 0 && errno == ENOENT) - return dbm_open(CS name, flags, mode); -#ifndef COMPILE_UTILITY -debug_printf("%s %d errno %s\n", __FUNCTION__, __LINE__, strerror(errno)); -#endif -errno = (st.st_mode & S_IFMT) == S_IFDIR ? EISDIR : EEXIST; -return NULL; -} - -/* EXIM_DBGET - returns TRUE if successful, FALSE otherwise */ -static inline BOOL -exim_dbget(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * res) -{ -*res = dbm_fetch(dbp, *key); /* A struct arg & return! */ -return res->dptr != NULL; -} - -/* EXIM_DBPUT - returns nothing useful, assumes replace mode */ -static inline int -exim_dbput(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * data) -{ return dbm_store(dbp, *key, *data, DBM_REPLACE); } - -/* EXIM_DBPUTB - non-overwriting for use by dbmbuild */ -static inline int -exim_dbputb(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * data) -{ return dbm_store(dbp, *key, *data, DBM_INSERT); } - -/* Returns from EXIM_DBPUTB */ - -# define EXIM_DBPUTB_OK 0 -# define EXIM_DBPUTB_DUP 1 - -/* EXIM_DBDEL */ -static inline int -exim_dbdel(EXIM_DB * dbp, EXIM_DATUM * key) -{ return dbm_delete(dbp, *key); } - -/* EXIM_DBCREATE_CURSOR - initialize for scanning operation (null) */ -static inline EXIM_CURSOR * -exim_dbcreate_cursor(EXIM_DB * dbp) -{ return NULL; } - -/* EXIM_DBSCAN */ -static inline BOOL -exim_dbscan(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * data, BOOL first, - EXIM_CURSOR * cursor) -{ -*key = first ? dbm_firstkey(dbp) : dbm_nextkey(dbp); -return key->dptr != NULL; -} - -/* EXIM_DBDELETE_CURSOR - terminate scanning operation (null). */ -static inline void -exim_dbdelete_cursor(EXIM_CURSOR * cursor) -{ } - -/* EXIM_DBCLOSE */ -static inline void -exim_dbclose__(EXIM_DB * dbp) -{ dbm_close(dbp); } - -/* Datum access types */ - -static inline uschar * -exim_datum_data_get(EXIM_DATUM * dp) -{ return US dp->dptr; } -static inline void -exim_datum_data_set(EXIM_DATUM * dp, void * s) -{ dp->dptr = s; } - -static inline unsigned -exim_datum_size_get(EXIM_DATUM * dp) -{ return dp->dsize; } -static inline void -exim_datum_size_set(EXIM_DATUM * dp, unsigned n) -{ dp->dsize = n; } - -/* There's no clearing required before use, and we don't have to free anything -after reading data. */ - -static inline void -exim_datum_init(EXIM_DATUM * d) -{ } - -static inline void -exim_datum_free(EXIM_DATUM * d) -{ } - -/* size limit */ - -# define EXIM_DB_RLIMIT 150 - +# include "hintsdb/hints_ndbm.h" #endif /* !USE_GDBM */ @@ -1200,11 +154,10 @@ DEBUG(D_hints_lookup) debug_printf_indent("EXIM_DBCLOSE(%p)\n", dbp); exim_dbclose__(dbp); } -# endif /* defined(COMPILE_UTILITY) || defined(MACRO_PREDEF) */ +#endif /* defined(COMPILE_UTILITY) || defined(MACRO_PREDEF) */ /********************* End of dbm library definitions **********************/ - #endif /* whole file */ /* End of hintsdb.h */ /* vi: aw ai sw=2 diff --git a/src/src/hintsdb/hints_bdb.h b/src/src/hintsdb/hints_bdb.h new file mode 100644 index 000000000..e629cce4e --- /dev/null +++ b/src/src/hintsdb/hints_bdb.h @@ -0,0 +1,357 @@ +/************************************************* +* Exim - an Internet mail transport agent * +*************************************************/ + +/* Copyright (c) The Exim Maintainers 2020 - 2024 */ +/* Copyright (c) University of Cambridge 1995 - 2018 */ +/* See the file NOTICE for conditions of use and distribution. */ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +/********************* Berkeley db native definitions **********************/ + +# include + +/* 1.x did no locking + 2.x had facilities, but exim does it's own + 3.x+ unknown +*/ + +/* We can distinguish between versions 1.x and 2.x/3.x by looking for a +definition of DB_VERSION_STRING, which is present in versions 2.x onwards. */ + +# ifdef DB_VERSION_STRING + +# if DB_VERSION_MAJOR >= 6 +# error Version 6 and later BDB API is not supported +# endif + +/* The API changed (again!) between the 2.x and 3.x versions */ + +# if DB_VERSION_MAJOR >= 3 + +/***************** Berkeley db 3.x/4.x native definitions ******************/ + +/* Basic DB type */ +# if DB_VERSION_MAJOR > 4 || (DB_VERSION_MAJOR == 4 && DB_VERSION_MINOR >= 1) +# define EXIM_DB DB_ENV +/* Cursor type, for scanning */ +# define EXIM_CURSOR DBC + +/* The datum type used for queries */ +# define EXIM_DATUM DBT + +/* Some text for messages */ +# define EXIM_DBTYPE "db (v4.1+)" + +/* Only more-recent versions. 5+ ? */ +# ifndef DB_FORCESYNC +# define DB_FORCESYNC 0 +# endif + +/* Error callback */ +/* For Berkeley DB >= 2, we can define a function to be called in case of DB +errors. This should help with debugging strange DB problems, e.g. getting "File +exists" when you try to open a db file. The API for this function was changed +at DB release 4.3. */ + +static inline void +dbfn_bdb_error_callback(const DB_ENV * dbenv, const char * pfx, const char * msg) +{ +#ifndef MACRO_PREDEF +log_write(0, LOG_MAIN, "Berkeley DB error: %s", msg); +#endif +} + + + +/* Access functions (BDB 4.1+) */ + +static inline BOOL +exim_lockfile_needed(void) +{ +return TRUE; +} + +static inline EXIM_DB * +exim_dbopen_multi(const uschar * name, const uschar * dirname, int flags, + unsigned mode) { return NULL; } +static inline void exim_dbclose_multi(EXIM_DB * dbp) {} +static inline BOOL exim_dbtransaction_start(EXIM_DB * dbp) { return FALSE; } +static inline void exim_dbtransaction_commit(EXIM_DB * dbp) {} + +/* EXIM_DBOPEN - return pointer to an EXIM_DB, NULL if failed */ +/* The API changed for DB 4.1. - and we also starting using the "env" with a +specified working dir, to avoid the DBCONFIG file trap. */ + +# define ENV_TO_DB(env) ((DB *)(((EXIM_DB *)env)->app_private)) + +static inline EXIM_DB * +exim_dbopen__(const uschar * name, const uschar * dirname, int flags, + unsigned mode) +{ +EXIM_DB * dbp; +DB * b; +if ( db_env_create(&dbp, 0) != 0 + || (dbp->set_errcall(dbp, dbfn_bdb_error_callback), 0) + || dbp->open(dbp, CS dirname, DB_CREATE|DB_INIT_MPOOL|DB_PRIVATE, 0) != 0 + ) + return NULL; +if (db_create(&b, dbp, 0) == 0) + { + dbp->app_private = b; + if (b->open(b, NULL, CS name, NULL, + flags & O_CREAT ? DB_HASH : DB_UNKNOWN, + flags & O_CREAT ? DB_CREATE + : flags & (O_WRONLY|O_RDWR) ? 0 : DB_RDONLY, + mode) == 0 + ) + return dbp; + + b->close(b, 0); + } +dbp->close(dbp, 0); +return NULL; +} + +/* EXIM_DBGET - returns TRUE if successful, FALSE otherwise */ +static inline BOOL +exim_dbget(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * res) +{ +DB * b = ENV_TO_DB(dbp); +return b->get(b, NULL, key, res, 0) == 0; +} + +/* EXIM_DBPUT - returns nothing useful, assumes replace mode */ +static inline int +exim_dbput(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * data) +{ +DB * b = ENV_TO_DB(dbp); +return b->put(b, NULL, key, data, 0); +} + +/* EXIM_DBPUTB - non-overwriting for use by dbmbuild */ +static inline int +exim_dbputb(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * data) +{ +DB * b = ENV_TO_DB(dbp); +return b->put(b, NULL, key, data, DB_NOOVERWRITE); +} + +/* Return values from EXIM_DBPUTB */ + +# define EXIM_DBPUTB_OK 0 +# define EXIM_DBPUTB_DUP DB_KEYEXIST + +/* EXIM_DBDEL */ +static inline int +exim_dbdel(EXIM_DB * dbp, EXIM_DATUM * key) +{ +DB * b = ENV_TO_DB(dbp); +return b->del(b, NULL, key, 0); +} + +/* EXIM_DBCREATE_CURSOR - initialize for scanning operation */ + +static inline EXIM_CURSOR * +exim_dbcreate_cursor(EXIM_DB * dbp) +{ +DB * b = ENV_TO_DB(dbp); +EXIM_CURSOR * c; +b->cursor(b, NULL, &c, 0); +return c; +} + +/* EXIM_DBSCAN - returns TRUE if data is returned, FALSE at end */ +static inline BOOL +exim_dbscan(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * data, BOOL first, + EXIM_CURSOR * cursor) +{ +return cursor->c_get(cursor, key, data, first ? DB_FIRST : DB_NEXT) == 0; +} + +/* EXIM_DBDELETE_CURSOR - terminate scanning operation */ +static inline void +exim_dbdelete_cursor(EXIM_CURSOR * cursor) +{ cursor->c_close(cursor); } + +/* EXIM_DBCLOSE */ +static inline void +exim_dbclose__(EXIM_DB * dbp_o) +{ +DB_ENV * dbp = dbp_o; +DB * b = ENV_TO_DB(dbp); +b->close(b, 0); +dbp->close(dbp, DB_FORCESYNC); +} + +/* Datum access */ + +static inline uschar * +exim_datum_data_get(EXIM_DATUM * dp) +{ return dp->data; } +static inline void +exim_datum_data_set(EXIM_DATUM * dp, void * s) +{ dp->data = s; } + +static inline unsigned +exim_datum_size_get(EXIM_DATUM * dp) +{ return dp->size; } +static inline void +exim_datum_size_set(EXIM_DATUM * dp, unsigned n) +{ dp->size = n; } + +/* The whole datum structure contains other fields that must be cleared +before use, but we don't have to free anything after reading data. */ + +static inline void +exim_datum_init(EXIM_DATUM * d) +{ memset(d, 0, sizeof(*d)); } + +static inline void +exim_datum_free(EXIM_DATUM * d) +{ } + +# else /* pre- 4.1 */ + +# define EXIM_DB DB + +/* Cursor type, for scanning */ +# define EXIM_CURSOR DBC + +/* The datum type used for queries */ +# define EXIM_DATUM DBT + +/* Some text for messages */ +# define EXIM_DBTYPE "db (v3/4)" + +/* Access functions (BDB 3/4) */ + +static inline BOOL +exim_lockfile_needed(void) +{ +return TRUE; +} + +static inline EXIM_DB * +exim_dbopen_multi(const uschar * name, const uschar * dirname, int flags, + unsigned mode) { return NULL; } +static inline void exim_dbclose_multi(EXIM_DB * dbp) {} +static inline BOOL exim_dbtransaction_start(EXIM_DB * dbp) { return FALSE; } +static inline void exim_dbtransaction_commit(EXIM_DB * dbp) {} + +/* EXIM_DBOPEN - return pointer to an EXIM_DB, NULL if failed */ +static inline EXIM_DB * +exim_dbopen__(const uschar * name, const uschar * dirname, int flags, + unsigned mode) +{ +EXIM_DB * dbp; +return db_create(&dbp, NULL, 0) == 0 + && ( dbp->set_errcall(dbp, dbfn_bdb_error_callback), + dbp->open(dbp, CS name, NULL, + flags & O_CREAT ? DB_HASH : DB_UNKNOWN, + flags & O_CREAT ? DB_CREATE + : flags & (O_WRONLY|O_RDWR) ? 0 : DB_RDONLY, + mode) + ) == 0 + ? dbp : NULL; +} + +/* EXIM_DBGET - returns TRUE if successful, FALSE otherwise */ +static inline BOOL +exim_dbget(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * res) +{ return dbp->get(dbp, NULL, key, res, 0) == 0; } + +/* EXIM_DBPUT - returns nothing useful, assumes replace mode */ +static inline int +exim_dbput(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * data) +{ return dbp->put(dbp, NULL, key, data, 0); } + +/* EXIM_DBPUTB - non-overwriting for use by dbmbuild */ +static inline int +exim_dbputb(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * data) +{ return dbp->put(dbp, NULL, key, data, DB_NOOVERWRITE); } + +/* Return values from EXIM_DBPUTB */ + +# define EXIM_DBPUTB_OK 0 +# define EXIM_DBPUTB_DUP DB_KEYEXIST + +/* EXIM_DBDEL */ +static inline int +exim_dbdel(EXIM_DB * dbp, EXIM_DATUM * key) +{ return dbp->del(dbp, NULL, key, 0); } + +/* EXIM_DBCREATE_CURSOR - initialize for scanning operation */ + +static inline EXIM_CURSOR * +exim_dbcreate_cursor(EXIM_DB * dbp) +{ +EXIM_CURSOR * c; +dbp->cursor(dbp, NULL, &c, 0); +return c; +} + +/* EXIM_DBSCAN - returns TRUE if data is returned, FALSE at end */ +static inline BOOL +exim_dbscan(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * data, BOOL first, + EXIM_CURSOR * cursor) +{ +return cursor->c_get(cursor, key, data, first ? DB_FIRST : DB_NEXT) == 0; +} + +/* EXIM_DBDELETE_CURSOR - terminate scanning operation */ +static inline void +exim_dbdelete_cursor(EXIM_CURSOR * cursor) +{ cursor->c_close(cursor); } + +/* EXIM_DBCLOSE */ +static inline void +exim_dbclose__(EXIM_DB * dbp) +{ dbp->close(dbp, 0); } + +/* Datum access */ + +static inline uschar * +exim_datum_data_get(EXIM_DATUM * dp) +{ return US dp->dptr; } +static inline void +exim_datum_data_set(EXIM_DATUM * dp, void * s) +{ dp->dptr = s; } + +static inline uschar * +exim_datum_size_get(EXIM_DATUM * dp) +{ return US dp->size; } +static inline void +exim_datum_size_set(EXIM_DATUM * dp, uschar * s) +{ dp->size = CS s; } + +/* The whole datum structure contains other fields that must be cleared +before use, but we don't have to free anything after reading data. */ + +static inline void +exim_datum_init(EXIM_DATUM * d) +{ memset(d, 0, sizeof(*d)); } + +static inline void +exim_datum_free(EXIM_DATUM * d) +{ } + +# endif + + +# else /* DB_VERSION_MAJOR >= 3 */ +# error Berkeley DB versions earlier than 3 are not supported */ +# endif /* DB_VERSION_MAJOR */ +# else +# error Berkeley DB version 1 is no longer supported +# endif /* DB_VERSION_STRING */ + + +/* all BDB versions */ +/* size limit */ + +# define EXIM_DB_RLIMIT 150 + +/* End of hintsdb/hints_bdb.h */ +/* vi: aw ai sw=2 +*/ diff --git a/src/src/hintsdb/hints_gdbm.h b/src/src/hintsdb/hints_gdbm.h new file mode 100644 index 000000000..b406d45e0 --- /dev/null +++ b/src/src/hintsdb/hints_gdbm.h @@ -0,0 +1,162 @@ +/************************************************* +* Exim - an Internet mail transport agent * +*************************************************/ + +/* Copyright (c) The Exim Maintainers 2020 - 2024 */ +/* Copyright (c) University of Cambridge 1995 - 2018 */ +/* See the file NOTICE for conditions of use and distribution. */ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +/* This header file contains macro definitions for one possible hintsdb +backend provider. */ + +/********************* gdbm interface definitions **********************/ + +/*XXX TODO: exim's lockfile not needed? */ + +# include + +/* Basic DB type */ +typedef struct { + GDBM_FILE gdbm; /* Database */ + datum lkey; /* Last key, for scans */ +} EXIM_DB; + +/* Cursor type, not used with gdbm: just set up a dummy */ +# define EXIM_CURSOR int + +/* The datum type used for queries */ +# define EXIM_DATUM datum + +/* Some text for messages */ + +# define EXIM_DBTYPE "gdbm" + +/* Access functions (gdbm) */ + +static inline BOOL +exim_lockfile_needed(void) +{ +return TRUE; +} + +static inline EXIM_DB * +exim_dbopen_multi(const uschar * name, const uschar * dirname, int flags, + unsigned mode) { return NULL; } +static inline void exim_dbclose_multi(EXIM_DB * dbp) {} +static inline BOOL exim_dbtransaction_start(EXIM_DB * dbp) { return FALSE; } +static inline void exim_dbtransaction_commit(EXIM_DB * dbp) {} + +/* EXIM_DBOPEN - return pointer to an EXIM_DB, NULL if failed */ +static inline EXIM_DB * +exim_dbopen__(const uschar * name, const uschar * dirname, int flags, + unsigned mode) +{ +EXIM_DB * dbp = malloc(sizeof(EXIM_DB)); /*XXX why not exim mem-mgmt? */ +if (dbp) + { + dbp->lkey.dptr = NULL; + dbp->gdbm = gdbm_open(CS name, 0, + flags & O_CREAT ? GDBM_WRCREAT + : flags & (O_RDWR|O_WRONLY) ? GDBM_WRITER : GDBM_READER, + mode, 0); + if (dbp->gdbm) return dbp; + free(dbp); + } +return NULL; +} + +/* EXIM_DBGET - returns TRUE if successful, FALSE otherwise */ +static inline BOOL +exim_dbget(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * res) +{ +*res = gdbm_fetch(dbp->gdbm, *key); /* A struct arg & return! */ +return res->dptr != NULL; +} + +/* EXIM_DBPUT - returns nothing useful, assumes replace mode */ +static inline int +exim_dbput(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * data) +{ return gdbm_store(dbp->gdbm, *key, *data, GDBM_REPLACE); } + +/* EXIM_DBPUTB - non-overwriting for use by dbmbuild */ +static inline int +exim_dbputb(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * data) +{ return gdbm_store(dbp->gdbm, *key, *data, GDBM_INSERT); } + +/* Returns from EXIM_DBPUTB */ + +# define EXIM_DBPUTB_OK 0 +# define EXIM_DBPUTB_DUP 1 + +/* EXIM_DBDEL */ +static inline int +exim_dbdel(EXIM_DB * dbp, EXIM_DATUM * key) +{ return gdbm_delete(dbp->gdbm, *key); } + +/* EXIM_DBCREATE_CURSOR - initialize for scanning operation (null) */ +static inline EXIM_CURSOR * +exim_dbcreate_cursor(EXIM_DB * dbp) +{ return NULL; } + +/* EXIM_DBSCAN */ +static inline BOOL +exim_dbscan(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * data, BOOL first, + EXIM_CURSOR * cursor) +{ +char * s; +*key = first ? gdbm_firstkey(dbp->gdbm) : gdbm_nextkey(dbp->gdbm, dbp->lkey); +if ((s = dbp->lkey.dptr)) free(s); +dbp->lkey = *key; +return key->dptr != NULL; +} + +/* EXIM_DBDELETE_CURSOR - terminate scanning operation (null). */ +static inline void +exim_dbdelete_cursor(EXIM_CURSOR * cursor) +{ } + +/* EXIM_DBCLOSE */ +static inline void +exim_dbclose__(EXIM_DB * dbp) +{ +char * s; +gdbm_close(dbp->gdbm); +if ((s = dbp->lkey.dptr)) free(s); +free(dbp); +} + +/* Datum access types */ + +static inline uschar * +exim_datum_data_get(EXIM_DATUM * dp) +{ return US dp->dptr; } +static inline void +exim_datum_data_set(EXIM_DATUM * dp, void * s) +{ dp->dptr = s; } + +static inline unsigned +exim_datum_size_get(EXIM_DATUM * dp) +{ return dp->dsize; } +static inline void +exim_datum_size_set(EXIM_DATUM * dp, unsigned n) +{ dp->dsize = n; } + +/* There's no clearing required before use, but we have to free the dptr +after reading data. */ + +static inline void +exim_datum_init(EXIM_DATUM * d) +{ } + +static inline void +exim_datum_free(EXIM_DATUM * d) +{ free(d->dptr); } + +/* size limit. GDBM is int-max limited, but we want to be less silly */ + +# define EXIM_DB_RLIMIT 150 + +/* End of hintsdb/hints_gdbm.h */ +/* vi: aw ai sw=2 +*/ diff --git a/src/src/hintsdb/hints_ndbm.h b/src/src/hintsdb/hints_ndbm.h new file mode 100644 index 000000000..fb1db57a8 --- /dev/null +++ b/src/src/hintsdb/hints_ndbm.h @@ -0,0 +1,149 @@ +/************************************************* +* Exim - an Internet mail transport agent * +*************************************************/ + +/* Copyright (c) The Exim Maintainers 2020 - 2024 */ +/* Copyright (c) University of Cambridge 1995 - 2018 */ +/* See the file NOTICE for conditions of use and distribution. */ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +/* This header file contains macro definitions for one possible hintsdb +backend provider. */ + +/********************* ndbm interface definitions **********************/ + +# include + +/* Basic DB type */ +# define EXIM_DB DBM + +/* Cursor type, not used with ndbm: just set up a dummy */ +# define EXIM_CURSOR int + +/* The datum type used for queries */ +# define EXIM_DATUM datum + +/* Some text for messages */ + +# define EXIM_DBTYPE "ndbm" + +/* Access functions (ndbm) */ + +static inline BOOL +exim_lockfile_needed(void) +{ +return TRUE; +} + +static inline EXIM_DB * +exim_dbopen_multi(const uschar * name, const uschar * dirname, int flags, + unsigned mode) { return NULL; } +static inline void exim_dbclose_multi(EXIM_DB * dbp) {} +static inline BOOL exim_dbtransaction_start(EXIM_DB * dbp) { return FALSE; } +static inline void exim_dbtransaction_commit(EXIM_DB * dbp) {} + +/* EXIM_DBOPEN - returns a EXIM_DB *, NULL if failed */ +/* Check that the name given is not present. This catches +a directory name; otherwise we would create the name.pag and +name.dir files in the directory's parent. */ + +static inline EXIM_DB * +exim_dbopen__(const uschar * name, const uschar * dirname, int flags, + unsigned mode) +{ +struct stat st; +if (!(flags & O_CREAT) || lstat(CCS name, &st) != 0 && errno == ENOENT) + return dbm_open(CS name, flags, mode); +#ifndef COMPILE_UTILITY +debug_printf("%s %d errno %s\n", __FUNCTION__, __LINE__, strerror(errno)); +#endif +errno = (st.st_mode & S_IFMT) == S_IFDIR ? EISDIR : EEXIST; +return NULL; +} + +/* EXIM_DBGET - returns TRUE if successful, FALSE otherwise */ +static inline BOOL +exim_dbget(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * res) +{ +*res = dbm_fetch(dbp, *key); /* A struct arg & return! */ +return res->dptr != NULL; +} + +/* EXIM_DBPUT - returns nothing useful, assumes replace mode */ +static inline int +exim_dbput(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * data) +{ return dbm_store(dbp, *key, *data, DBM_REPLACE); } + +/* EXIM_DBPUTB - non-overwriting for use by dbmbuild */ +static inline int +exim_dbputb(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * data) +{ return dbm_store(dbp, *key, *data, DBM_INSERT); } + +/* Returns from EXIM_DBPUTB */ + +# define EXIM_DBPUTB_OK 0 +# define EXIM_DBPUTB_DUP 1 + +/* EXIM_DBDEL */ +static inline int +exim_dbdel(EXIM_DB * dbp, EXIM_DATUM * key) +{ return dbm_delete(dbp, *key); } + +/* EXIM_DBCREATE_CURSOR - initialize for scanning operation (null) */ +static inline EXIM_CURSOR * +exim_dbcreate_cursor(EXIM_DB * dbp) +{ return NULL; } + +/* EXIM_DBSCAN */ +static inline BOOL +exim_dbscan(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * data, BOOL first, + EXIM_CURSOR * cursor) +{ +*key = first ? dbm_firstkey(dbp) : dbm_nextkey(dbp); +return key->dptr != NULL; +} + +/* EXIM_DBDELETE_CURSOR - terminate scanning operation (null). */ +static inline void +exim_dbdelete_cursor(EXIM_CURSOR * cursor) +{ } + +/* EXIM_DBCLOSE */ +static inline void +exim_dbclose__(EXIM_DB * dbp) +{ dbm_close(dbp); } + +/* Datum access types */ + +static inline uschar * +exim_datum_data_get(EXIM_DATUM * dp) +{ return US dp->dptr; } +static inline void +exim_datum_data_set(EXIM_DATUM * dp, void * s) +{ dp->dptr = s; } + +static inline unsigned +exim_datum_size_get(EXIM_DATUM * dp) +{ return dp->dsize; } +static inline void +exim_datum_size_set(EXIM_DATUM * dp, unsigned n) +{ dp->dsize = n; } + +/* There's no clearing required before use, and we don't have to free anything +after reading data. */ + +static inline void +exim_datum_init(EXIM_DATUM * d) +{ } + +static inline void +exim_datum_free(EXIM_DATUM * d) +{ } + +/* size limit */ + +# define EXIM_DB_RLIMIT 150 + +/* End of hintsdb/hints_ndbm.h */ +/* vi: aw ai sw=2 +*/ diff --git a/src/src/hintsdb/hints_sqlite.h b/src/src/hintsdb/hints_sqlite.h new file mode 100644 index 000000000..da3bc2bff --- /dev/null +++ b/src/src/hintsdb/hints_sqlite.h @@ -0,0 +1,341 @@ +/************************************************* +* Exim - an Internet mail transport agent * +*************************************************/ + +/* Copyright (c) The Exim Maintainers 2020 - 2024 */ +/* Copyright (c) University of Cambridge 1995 - 2018 */ +/* See the file NOTICE for conditions of use and distribution. */ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +/* This header file contains macro definitions for one possible hintsdb +backend provider. */ + +/* ********************* sqlite3 interface ************************ */ + +# include + +/* Basic DB type */ +# define EXIM_DB sqlite3 + +# define EXIM_CURSOR int + +# /* The datum type used for queries */ +# define EXIM_DATUM blob + +/* Some text for messages */ +# define EXIM_DBTYPE "sqlite3" + +# /* Access functions */ + +static inline BOOL +exim_lockfile_needed(void) +{ +return FALSE; /* We do transaction; no extra locking needed */ +} + +/* EXIM_DBOPEN - return pointer to an EXIM_DB, NULL if failed */ +static inline EXIM_DB * +exim_dbopen_multi(const uschar * name, const uschar * dirname, int flags, + unsigned mode) +{ +EXIM_DB * dbp; +int ret, sflags = flags & O_RDWR ? SQLITE_OPEN_READWRITE : SQLITE_OPEN_READONLY; +if (flags & O_CREAT) sflags |= SQLITE_OPEN_CREATE; +if ((ret = sqlite3_open_v2(CCS name, &dbp, sflags, NULL)) == SQLITE_OK) + { + sqlite3_busy_timeout(dbp, 5000); + if (flags & O_CREAT) + ret = sqlite3_exec(dbp, + "CREATE TABLE IF NOT EXISTS tbl (ky TEXT PRIMARY KEY, dat BLOB);", + NULL, NULL, NULL); + if (ret != SQLITE_OK) + sqlite3_close(dbp); + } +//else +// fprintf(stderr, "sqlite3_open_v2: %s\n", sqlite3_errmsg(dbp)); +return ret == SQLITE_OK ? dbp : NULL; +} + +static inline BOOL +exim_dbtransaction_start(EXIM_DB * dbp) +{ +return sqlite3_exec(dbp, "BEGIN TRANSACTION;", NULL, NULL, NULL) == SQLITE_OK; +} + +static inline EXIM_DB * +exim_dbopen__(const uschar * name, const uschar * dirname, int flags, + unsigned mode) +{ +EXIM_DB * dbp = exim_dbopen_multi(name, dirname, flags, mode); +if (!dbp || exim_dbtransaction_start(dbp)) + return dbp; +sqlite3_close(dbp); +return NULL; +} + +/* EXIM_DBGET - returns TRUE if successful, FALSE otherwise */ +/* note we alloc'n'copy - the caller need not do so */ +/* result has a NUL appended, but the length is as per the DB */ + +static inline BOOL +exim_dbget__(EXIM_DB * dbp, const uschar * s, EXIM_DATUM * res) +{ +sqlite3_stmt * statement; +int ret; + +res->len = (size_t) -1; +/* fprintf(stderr, "exim_dbget__(%s)\n", s); */ +if ((ret = sqlite3_prepare_v2(dbp, CCS s, -1, &statement, NULL)) != SQLITE_OK) + { +/* fprintf(stderr, "prepare fail: %s\n", sqlite3_errmsg(dbp)); */ + return FALSE; + } +if (sqlite3_step(statement) != SQLITE_ROW) + { +/* fprintf(stderr, "step fail: %s\n", sqlite3_errmsg(dbp)); */ + sqlite3_finalize(statement); + return FALSE; + } + +res->len = sqlite3_column_bytes(statement, 0); +# ifdef COMPILE_UTILITY +if (!(res->data = malloc(res->len +1))) + { sqlite3_finalize(statement); return FALSE; } +# else +res->data = store_get(res->len +1, GET_TAINTED); +# endif +memcpy(res->data, sqlite3_column_blob(statement, 0), res->len); +res->data[res->len] = '\0'; +/* fprintf(stderr, "res %d bytes: '%.*s'\n", (int)res->len, (int)res->len, res->data); */ +sqlite3_finalize(statement); +return TRUE; +} + +static inline BOOL +exim_dbget(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * res) +{ +# define FMT "SELECT dat FROM tbl WHERE ky = '%.*s';" +uschar * qry; +int i; +BOOL ret; + +# ifdef COMPILE_UTILITY +/* fprintf(stderr, "exim_dbget(k len %d '%.*s')\n", (int)key->len, (int)key->len, key->data); */ +i = snprintf(NULL, 0, FMT, (int) key->len, key->data)+1; +if (!(qry = malloc(i))) + return FALSE; +snprintf(CS qry, i, FMT, (int) key->len, key->data); +ret = exim_dbget__(dbp, qry, res); +free(qry); +# else +/* fprintf(stderr, "exim_dbget(k len %d '%.*s')\n", (int)key->len, (int)key->len, key->data); */ +qry = string_sprintf(FMT, (int) key->len, key->data); +ret = exim_dbget__(dbp, qry, res); +# endif + +return ret; +# undef FMT +} + +/* Note that we return claiming a duplicate record for any error. +It seem not uncommon to get a "database is locked" error. */ +# define EXIM_DBPUTB_OK 0 +# define EXIM_DBPUTB_DUP (-1) + +static inline int +exim_s_dbp(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * data, const uschar * alt) +{ +int hlen = data->len * 2, off = 0, res; +# define FMT "INSERT OR %s INTO tbl (ky,dat) VALUES ('%.*s', X'%.*s');" +uschar * qry; +# ifdef COMPILE_UTILITY +uschar * hex = malloc(hlen+1); +if (!hex) return EXIM_DBPUTB_DUP; /* best we can do */ +# else +uschar * hex = store_get(hlen+1, data->data); +# endif + +for (const uschar * s = data->data, * t = s + data->len; s < t; s++, off += 2) + sprintf(CS hex + off, "%02X", *s); + +# ifdef COMPILE_UTILITY +res = snprintf(CS hex, 0, FMT, alt, (int) key->len, key->data, hlen, hex) +1; +if (!(qry = malloc(res))) return EXIM_DBPUTB_DUP; +snprintf(CS qry, res, FMT, alt, (int) key->len, key->data, hlen, hex); +/* fprintf(stderr, "exim_s_dbp(%s)\n", qry); */ +res = sqlite3_exec(dbp, CS qry, NULL, NULL, NULL); +free(qry); +free(hex); +# else +qry = string_sprintf(FMT, alt, (int) key->len, key->data, hlen, hex); +/* fprintf(stderr, "exim_s_dbp(%s)\n", qry); */ +res = sqlite3_exec(dbp, CS qry, NULL, NULL, NULL); +/* fprintf(stderr, "exim_s_dbp res %d\n", res); */ +# endif + +# ifdef COMPILE_UTILITY +if (res != SQLITE_OK) + fprintf(stderr, "sqlite3_exec: %s\n", sqlite3_errmsg(dbp)); +# endif + +return res == SQLITE_OK ? EXIM_DBPUTB_OK : EXIM_DBPUTB_DUP; +# undef FMT +} + +/* EXIM_DBPUT - returns nothing useful, assumes replace mode */ + +static inline int +exim_dbput(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * data) +{ +/* fprintf(stderr, "exim_dbput()\n"); */ +(void) exim_s_dbp(dbp, key, data, US"REPLACE"); +return 0; +} + +/* EXIM_DBPUTB - non-overwriting for use by dbmbuild */ + +/* Returns from EXIM_DBPUTB */ + +static inline int +exim_dbputb(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * data) +{ +return exim_s_dbp(dbp, key, data, US"ABORT"); +} + +/* EXIM_DBDEL */ +static inline int +exim_dbdel(EXIM_DB * dbp, EXIM_DATUM * key) +{ +# define FMT "DELETE FROM tbl WHERE ky = '%.*s';" +uschar * qry; +int res; + +# ifdef COMPILE_UTILITY +res = snprintf(NULL, 0, FMT, (int) key->len, key->data) +1; /* res includes nul */ +if (!(qry = malloc(res))) return SQLITE_NOMEM; +snprintf(CS qry, res, FMT, (int) key->len, key->data); +res = sqlite3_exec(dbp, CS qry, NULL, NULL, NULL); +free(qry); +# else +qry = string_sprintf(FMT, (int) key->len, key->data); +res = sqlite3_exec(dbp, CS qry, NULL, NULL, NULL); +# endif + +return res; +# undef FMT +} + + +/* EXIM_DBCREATE_CURSOR - initialize for scanning operation */ +/* Cursors are inefficiently emulated by repeating searches */ + +static inline EXIM_CURSOR * +exim_dbcreate_cursor(EXIM_DB * dbp) +{ +# ifdef COMPILE_UTILITY +EXIM_CURSOR * c = malloc(sizeof(int)); +if (!c) return NULL; +# else +EXIM_CURSOR * c = store_malloc(sizeof(int)); +# endif +*c = 0; +return c; +} + +/* EXIM_DBSCAN */ +/* Note that we return the (next) key, not the record value */ +static inline BOOL +exim_dbscan(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * res, BOOL first, + EXIM_CURSOR * cursor) +{ +# define FMT "SELECT ky FROM tbl ORDER BY ky LIMIT 1 OFFSET %d;" +uschar * qry; +int i; +BOOL ret; + +# ifdef COMPILE_UTILITY +i = snprintf(NULL, 0, FMT, *cursor)+1; +if (!(qry = malloc(i))) return FALSE; +snprintf(CS qry, i, FMT, *cursor); +/* fprintf(stderr, "exim_dbscan(%s)\n", qry); */ +ret = exim_dbget__(dbp, qry, key); +free(qry); +/* fprintf(stderr, "exim_dbscan ret %c\n", ret ? 'T':'F'); */ +# else +qry = string_sprintf(FMT, *cursor); +/* fprintf(stderr, "exim_dbscan(%s)\n", qry); */ +ret = exim_dbget__(dbp, qry, key); +/* fprintf(stderr, "exim_dbscan ret %c\n", ret ? 'T':'F'); */ +# endif +if (ret) *cursor = *cursor + 1; +return ret; +# undef FMT +} + +/* EXIM_DBDELETE_CURSOR - terminate scanning operation. */ +static inline void +exim_dbdelete_cursor(EXIM_CURSOR * cursor) +{ +# ifdef COMPILE_UTILITY +free(cursor); +# else +store_free(cursor); +# endif +} + + +/* EXIM_DBCLOSE */ +static inline void +exim_dbclose_multi(EXIM_DB * dbp) +{ +sqlite3_close(dbp); +} +static inline void +exim_dbtransaction_commit(EXIM_DB * dbp) +{ +(void) sqlite3_exec(dbp, "COMMIT TRANSACTION;", NULL, NULL, NULL); +} +static inline void +exim_dbclose__(EXIM_DB * dbp) +{ +exim_dbtransaction_commit(dbp); +exim_dbclose_multi(dbp); +} + + +/* Datum access */ + +static uschar * +exim_datum_data_get(EXIM_DATUM * dp) +{ return US dp->data; } +static void +exim_datum_data_set(EXIM_DATUM * dp, void * s) +{ dp->data = s; } + +static unsigned +exim_datum_size_get(EXIM_DATUM * dp) +{ return dp->len; } +static void +exim_datum_size_set(EXIM_DATUM * dp, unsigned n) +{ dp->len = n; } + + + +static inline void +exim_datum_init(EXIM_DATUM * dp) +{ dp->data = NULL; } /* compiler quietening */ + +/* No free needed for a datum */ + +static inline void +exim_datum_free(EXIM_DATUM * dp) +{ } + +/* size limit */ + +# define EXIM_DB_RLIMIT 150 + + +/* End of hints_sqlite.h */ +/* vi: aw ai sw=2 +*/ diff --git a/src/src/hintsdb/hints_tdb.h b/src/src/hintsdb/hints_tdb.h new file mode 100644 index 000000000..436597115 --- /dev/null +++ b/src/src/hintsdb/hints_tdb.h @@ -0,0 +1,156 @@ +/************************************************* +* Exim - an Internet mail transport agent * +*************************************************/ + +/* Copyright (c) The Exim Maintainers 2020 - 2024 */ +/* Copyright (c) University of Cambridge 1995 - 2018 */ +/* See the file NOTICE for conditions of use and distribution. */ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +/* This header file contains macro definitions for one possible hintsdb +backend provider. */ + +/* ************************* tdb interface ************************ */ +/*XXX https://manpages.org/tdb/3 mentions concurrent writes. +Could we lose the file lock? */ + +# include + +/* Basic DB type */ +# define EXIM_DB TDB_CONTEXT + +/* Cursor type: tdb uses the previous "key" in _nextkey() (really it wants +tdb_traverse to be called) */ +# define EXIM_CURSOR TDB_DATA + +/* The datum type used for queries */ +# define EXIM_DATUM TDB_DATA + +/* Some text for messages */ +# define EXIM_DBTYPE "tdb" + +/* Access functions */ + +static inline BOOL +exim_lockfile_needed(void) +{ +return TRUE; +} + +static inline EXIM_DB * +exim_dbopen_multi(const uschar * name, const uschar * dirname, int flags, + unsigned mode) { return NULL; } +static inline void exim_dbclose_multi(EXIM_DB * dbp) {} +static inline BOOL exim_dbtransaction_start(EXIM_DB * dbp) { return FALSE; } +static inline void exim_dbtransaction_commit(EXIM_DB * dbp) {} + +/* EXIM_DBOPEN - return pointer to an EXIM_DB, NULL if failed */ +static inline EXIM_DB * +exim_dbopen__(const uschar * name, const uschar * dirname, int flags, + unsigned mode) +{ +return tdb_open(CS name, 0, TDB_DEFAULT, flags, mode); +} + +/* EXIM_DBGET - returns TRUE if successful, FALSE otherwise */ +static inline BOOL +exim_dbget(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * res) +{ +*res = tdb_fetch(dbp, *key); /* A struct arg and return!! */ +return res->dptr != NULL; +} + +/* EXIM_DBPUT - returns nothing useful, assumes replace mode */ +static inline int +exim_dbput(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * data) +{ return tdb_store(dbp, *key, *data, TDB_REPLACE); } + +/* EXIM_DBPUTB - non-overwriting for use by dbmbuild */ +static inline int +exim_dbputb(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * data) +{ return tdb_store(dbp, *key, *data, TDB_INSERT); } + +/* Returns from EXIM_DBPUTB */ + +# define EXIM_DBPUTB_OK 0 +# define EXIM_DBPUTB_DUP (-1) + +/* EXIM_DBDEL */ +static inline int +exim_dbdel(EXIM_DB * dbp, EXIM_DATUM * key) +{ return tdb_delete(dbp, *key); } + +/* EXIM_DBCREATE_CURSOR - initialize for scanning operation */ +static inline EXIM_CURSOR * +exim_dbcreate_cursor(EXIM_DB * dbp) +{ +# ifdef COMPILE_UTILITY +EXIM_CURSOR * c = malloc(sizeof(TDB_DATA)); +# else +EXIM_CURSOR * c = store_malloc(sizeof(TDB_DATA)); +# endif +c->dptr = NULL; +return c; +} + +/* EXIM_DBSCAN - This is complicated because we have to free the last datum +free() must not die when passed NULL */ + +static inline BOOL +exim_dbscan(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * res, BOOL first, + EXIM_CURSOR * cursor) +{ +*key = first ? tdb_firstkey(dbp) : tdb_nextkey(dbp, *cursor); +free(cursor->dptr); +*cursor = *key; +return key->dptr != NULL; +} + +/* EXIM_DBDELETE_CURSOR - terminate scanning operation. */ +static inline void +exim_dbdelete_cursor(EXIM_CURSOR * cursor) +{ store_free(cursor); } + +/* EXIM_DBCLOSE */ +static inline void +exim_dbclose__(EXIM_DB * db) +{ tdb_close(db); } + +/* Datum access */ + +static inline uschar * +exim_datum_data_get(EXIM_DATUM * dp) +{ return US dp->dptr; } +static inline void +exim_datum_data_set(EXIM_DATUM * dp, void * s) +{ dp->dptr = s; } + +static inline unsigned +exim_datum_size_get(EXIM_DATUM * dp) +{ return dp->dsize; } +static inline void +exim_datum_size_set(EXIM_DATUM * dp, unsigned n) +{ dp->dsize = n; } + +/* No initialization is needed. */ + +static inline void +exim_datum_init(EXIM_DATUM * d) +{ } + +/* Free the stuff inside the datum. */ + +static inline void +exim_datum_free(EXIM_DATUM * d) +{ +free(d->dptr); +d->dptr = NULL; +} + +/* size limit */ + +# define EXIM_DB_RLIMIT 150 + +/* End of hints_tdb.h */ +/* vi: aw ai sw=2 +*/ diff --git a/src/src/log.c b/src/src/log.c index f12721cf5..2e97660a0 100644 --- a/src/src/log.c +++ b/src/src/log.c @@ -308,8 +308,8 @@ Send fd over socketpair. Return: true iff good. */ -static BOOL -log_send_fd(const int sock, const int fd) +BOOL +send_fd_over_socket(const int sock, const int fd) { struct msghdr msg; union { @@ -343,8 +343,8 @@ return n == 1; Return fd passed over socketpair, or -1 on error. */ -static int -log_recv_fd(const int sock) +int +recv_fd_from_sock(const int sock) { struct msghdr msg; union { @@ -415,7 +415,7 @@ else if (euid == root_uid) || getgid() != exim_gid || getegid() != exim_gid || (fd = log_open_already_exim(name)) < 0 - || !log_send_fd(sock[1], fd) + || !send_fd_over_socket(sock[1], fd) ) _exit(EXIT_FAILURE); (void)close(sock[1]); _exit(EXIT_SUCCESS); @@ -424,7 +424,7 @@ else if (euid == root_uid) (void)close(sock[1]); if (pid > 0) { - fd = log_recv_fd(sock[0]); + fd = recv_fd_from_sock(sock[0]); while (waitpid(pid, NULL, 0) == -1 && errno == EINTR); } (void)close(sock[0]); diff --git a/src/src/macros.h b/src/src/macros.h index 1a619b951..3ccbe3353 100644 --- a/src/src/macros.h +++ b/src/src/macros.h @@ -1189,6 +1189,11 @@ typedef unsigned mcs_flags; #define SR_FINAL TRUE #define SR_NOT_FINAL FALSE +/* Flags for continued-TLS-connection */ +#define CTF_CV BIT(0) +#define CTF_DV BIT(1) +#define CTF_TR BIT(2) + /* Return codes for smtp_write_mail_and_rcpt_cmds() */ typedef enum { sw_mrc_ok, /* good, rcpt results in addr->transport_return (PENDING_OK, DEFER, FAIL) */ diff --git a/src/src/pdkim/pdkim.c b/src/src/pdkim/pdkim.c index 4fb22a113..b2caa81ab 100644 --- a/src/src/pdkim/pdkim.c +++ b/src/src/pdkim/pdkim.c @@ -468,15 +468,12 @@ return b64encode(CUS b->data, b->len); static pdkim_signature * pdkim_parse_sig_header(pdkim_ctx * ctx, uschar * raw_hdr) { -pdkim_signature * sig; -uschar *q; -gstring * cur_tag = NULL; -gstring * cur_val = NULL; -BOOL past_hname = FALSE; -BOOL in_b_val = FALSE; +pdkim_signature * sig = store_get(sizeof(pdkim_signature), GET_UNTAINTED); +uschar * q; +gstring * cur_tag = NULL, * cur_val = NULL; +BOOL past_hname = FALSE, in_b_val = FALSE; int where = PDKIM_HDR_LIMBO; -sig = store_get(sizeof(pdkim_signature), GET_UNTAINTED); memset(sig, 0, sizeof(pdkim_signature)); sig->bodylength = -1; @@ -1899,11 +1896,17 @@ for (pdkim_signature * sig = ctx->sig; sig; sig = sig->next) { sig->verify_status = PDKIM_VERIFY_PASS; verify_pass = TRUE; - if (dkim_verify_minimal) break; + /*XXX We used to "break" here if dkim_verify_minimal, but that didn't + stop the ACL being called. So move that test. Unfortunately, we + need to eval all the sigs here only to possibly ignore some later, + because we don't know what verify options might say. + Could we change to a later eval of the sig? + Both bits are called from receive_msg(). + Moving the test is also suboptimal for the case of no ACL (or no + signers to check!) so keep it for that case, but after debug output */ } NEXT_VERIFY: - DEBUG(D_acl) { debug_printf("DKIM [%s] %s signature status: %s", @@ -1915,6 +1918,10 @@ NEXT_VERIFY: else debug_printf("\n"); } + + if ( verify_pass && dkim_verify_minimal + && !(acl_smtp_dkim && dkim_verify_signers && *dkim_verify_signers)) + break; } } diff --git a/src/src/queue.c b/src/src/queue.c index a4afdcc98..3073ee780 100644 --- a/src/src/queue.c +++ b/src/src/queue.c @@ -341,8 +341,10 @@ so force the first one. The selecting string can optionally be a regex, or refer to the sender instead of recipients. If queue_2stage is set, the queue is scanned twice. The first time, queue_smtp -is set so that routing is done for all messages. Thus in the second run those -that are routed to the same host should go down the same SMTP connection. +is set so that routing is done for all messages. A call of the transport adds +each message_id in turn to a list for the resulting host. +Then in the second run those that are routed to the same host should all go down +a single SMTP connection. Arguments: q queue-runner descriptor @@ -358,9 +360,9 @@ queue_run(qrunner * q, const uschar * start_id, const uschar * stop_id, BOOL rec { BOOL force_delivery = q->queue_run_force || deliver_selectstring || deliver_selectstring_sender; -const pcre2_code *selectstring_regex = NULL; -const pcre2_code *selectstring_regex_sender = NULL; -uschar *log_detail = NULL; +const pcre2_code * selectstring_regex = NULL; +const pcre2_code * selectstring_regex_sender = NULL; +uschar * log_detail = NULL; int subcount = 0; uschar subdirs[64]; pid_t qpid[4] = {0}; /* Parallelism factor for q2stage 1st phase */ @@ -672,13 +674,15 @@ for (int i = queue_run_in_order ? -1 : 0; name. The return of the process is zero if a delivery was attempted. */ fq->text[Ustrlen(fq->text)-2] = 0; - set_process_info("running queue: %s", fq->text); + set_process_info("running queue%s: %s", + q->queue_2stage ? "(ph 1)" : "", fq->text); #ifdef MEASURE_TIMING report_time_since(×tamp_startup, US"queue msg selected"); #endif single_item_retry: - if ((pid = exim_fork(US"qrun-delivery")) == 0) + if ((pid = exim_fork( + q->queue_2stage ? US"qrun-p1-delivery" : US"qrun-delivery")) == 0) { int rc; (void)close(pfd[pipe_read]); @@ -792,9 +796,10 @@ if (q->queue_2stage) else break; #ifdef MEASURE_TIMING - report_time_since(×tamp_startup, US"queue_run 1st phase done"); + report_time_since(×tamp_startup, US"queue_run phase 1 done"); #endif q->queue_2stage = f.queue_2stage = FALSE; + DEBUG(D_queue_run) debug_printf("queue_run phase 2 start\n"); queue_run(q, start_id, stop_id, TRUE); } diff --git a/src/src/receive.c b/src/src/receive.c index 9fae6ad60..cc64f44f4 100644 --- a/src/src/receive.c +++ b/src/src/receive.c @@ -3518,7 +3518,7 @@ else #ifndef DISABLE_DKIM if (!f.dkim_disable_verify) { - /* Finish verification */ + /* Finish off the body hashes, calculate sigs and do compares */ dkim_exim_verify_finish(); /* Check if we must run the DKIM ACL */ @@ -3527,12 +3527,10 @@ else { uschar * dkim_verify_signers_expanded = expand_string(dkim_verify_signers); - gstring * results = NULL; - int signer_sep = 0; + gstring * results = NULL, * seen_items = NULL; + int signer_sep = 0, old_pool = store_pool; const uschar * ptr; uschar * item; - gstring * seen_items = NULL; - int old_pool = store_pool; store_pool = POOL_PERM; /* Allow created variables to live to data ACL */ @@ -3541,7 +3539,10 @@ else "expansion of dkim_verify_signers option failed: %s", expand_string_message); - /* Default to OK when no items are present */ + /* Loop over signers we want to verify, calling ACL. Default to OK + when no signers are present. Each call from here expands to a n ACL + call per matching sig in the message. */ + rc = OK; while ((item = string_nextinlist(&ptr, &signer_sep, NULL, 0))) { @@ -3586,6 +3587,9 @@ else cancel_cutthrough_connection(TRUE, US"dkim acl not ok"); break; } + else + if (dkim_verify_minimal && Ustrcmp(dkim_verify_status, "pass") == 0) + break; } dkim_verify_status = string_from_gstring(results); store_pool = old_pool; @@ -3606,7 +3610,7 @@ else goto NOT_ACCEPTED; /* Skip to end of function */ } } - else + else /* No acl or no wanted signers */ dkim_exim_verify_log_all(); } #endif /* DISABLE_DKIM */ diff --git a/src/src/retry.c b/src/src/retry.c index 9dfe1d587..fdcb6abea 100644 --- a/src/src/retry.c +++ b/src/src/retry.c @@ -192,7 +192,12 @@ if ((node = tree_search(tree_unusable, host_key))) /* Open the retry database, giving up if there isn't one. Otherwise, search for the retry records, and then close the database again. */ -if (!(dbm_file = dbfn_open(US"retry", O_RDONLY, &dbblock, FALSE, TRUE))) +if (!continue_retry_db) + dbm_file = dbfn_open(US"retry", O_RDONLY, &dbblock, FALSE, TRUE); +else if ((dbm_file = continue_retry_db) == (open_db *)-1) + dbm_file = NULL; + +if (!dbm_file) { DEBUG(D_deliver|D_retry|D_hints_lookup) debug_printf("no retry data available\n"); @@ -200,7 +205,8 @@ if (!(dbm_file = dbfn_open(US"retry", O_RDONLY, &dbblock, FALSE, TRUE))) } host_retry_record = dbfn_read(dbm_file, host_key); message_retry_record = dbfn_read(dbm_file, message_key); -dbfn_close(dbm_file); +if (!continue_retry_db) + dbfn_close(dbm_file); /* Ignore the data if it is too old - too long since it was written */ @@ -545,11 +551,10 @@ void retry_update(address_item ** addr_defer, address_item ** addr_failed, address_item ** addr_succeed) { -open_db dbblock; -open_db *dbm_file = NULL; +open_db dbblock, * dbm_file = NULL; time_t now = time(NULL); -DEBUG(D_retry) debug_printf("Processing retry items\n"); +DEBUG(D_retry) { debug_printf("Processing retry items\n"); acl_level++; } /* Three-times loop to handle succeeded, failed, and deferred addresses. Deferred addresses must be handled after failed ones, because some may be moved @@ -562,7 +567,7 @@ for (int i = 0; i < 3; i++) address_item ** paddr = i==0 ? addr_succeed : i==1 ? addr_failed : addr_defer; address_item ** saved_paddr = NULL; - DEBUG(D_retry) debug_printf("%s addresses:\n", + DEBUG(D_retry) debug_printf_indent("%s addresses:\n", i == 0 ? "Succeeded" : i == 1 ? "Failed" : "Deferred"); /* Loop for each address on the chain. For deferred addresses, the whole @@ -584,7 +589,7 @@ for (int i = 0; i < 3; i++) int update_count = 0; int timedout_count = 0; - DEBUG(D_retry) debug_printf(" %s%s\n", addr->address, + DEBUG(D_retry) debug_printf_indent(" %s%s\n", addr->address, addr->retries ? "" : ": no retry items"); /* Loop for each retry item. */ @@ -609,7 +614,7 @@ for (int i = 0; i < 3; i++) if (!dbm_file) { DEBUG(D_deliver|D_retry|D_hints_lookup) - debug_printf("retry database not available for updating\n"); + debug_printf_indent("retry database not available for updating\n"); return; } @@ -631,7 +636,7 @@ for (int i = 0; i < 3; i++) { (void)dbfn_delete(dbm_file, rti->key); DEBUG(D_retry) - debug_printf("deleted retry information for %s\n", rti->key); + debug_printf_indent("deleted retry information for %s\n", rti->key); continue; } @@ -651,7 +656,7 @@ for (int i = 0; i < 3; i++) rti->flags & rf_host ? addr->domain : NULL, rti->basic_errno, rti->more_errno))) { - DEBUG(D_retry) debug_printf("No configured retry item for %s%s%s\n", + DEBUG(D_retry) debug_printf_indent("No configured retry item for %s%s%s\n", rti->key, rti->flags & rf_host ? US" or " : US"", rti->flags & rf_host ? addr->domain : US""); @@ -661,11 +666,11 @@ for (int i = 0; i < 3; i++) DEBUG(D_retry) if (rti->flags & rf_host) - debug_printf("retry for %s (%s) = %s %d %d\n", rti->key, + debug_printf_indent("retry for %s (%s) = %s %d %d\n", rti->key, addr->domain, retry->pattern, retry->basic_errno, retry->more_errno); else - debug_printf("retry for %s = %s %d %d\n", rti->key, retry->pattern, + debug_printf_indent("retry for %s = %s %d %d\n", rti->key, retry->pattern, retry->basic_errno, retry->more_errno); /* Set up the message for the database retry record. Because DBM @@ -711,7 +716,7 @@ for (int i = 0; i < 3; i++) /* Compute how long this destination has been failing */ failing_interval = now - retry_record->first_failed; - DEBUG(D_retry) debug_printf("failing_interval=%d message_age=%d\n", + DEBUG(D_retry) debug_printf_indent("failing_interval=%d message_age=%d\n", failing_interval, message_age); /* For a non-host error, if the message has been on the queue longer @@ -793,7 +798,7 @@ for (int i = 0; i < 3; i++) ; if (now - received_time.tv_sec > last_rule->timeout) { - DEBUG(D_retry) debug_printf("on queue longer than maximum retry\n"); + DEBUG(D_retry) debug_printf_indent("on queue longer than maximum retry\n"); timedout_count++; rule = NULL; } @@ -862,11 +867,11 @@ for (int i = 0; i < 3; i++) DEBUG(D_retry) { int letter = retry_record->more_errno & 255; - debug_printf("Writing retry data for %s\n", rti->key); - debug_printf(" first failed=%d last try=%d next try=%d expired=%d\n", + debug_printf_indent("Writing retry data for %s\n", rti->key); + debug_printf_indent(" first failed=%d last try=%d next try=%d expired=%d\n", (int)retry_record->first_failed, (int)retry_record->last_try, (int)retry_record->next_try, retry_record->expired); - debug_printf(" errno=%d more_errno=", retry_record->basic_errno); + debug_printf_indent(" errno=%d more_errno=", retry_record->basic_errno); if (letter == 'A' || letter == 'M') debug_printf("%d,%c", (retry_record->more_errno >> 8) & 255, letter); @@ -886,12 +891,12 @@ for (int i = 0; i < 3; i++) if (update_count > 0 && update_count == timedout_count) if (!testflag(endaddr, af_retry_skipped)) { - DEBUG(D_retry) debug_printf("timed out: all retries expired\n"); + DEBUG(D_retry) debug_printf_indent("timed out: all retries expired\n"); timed_out = TRUE; } else DEBUG(D_retry) - debug_printf("timed out but some hosts were skipped\n"); + debug_printf_indent("timed out but some hosts were skipped\n"); } /* Loop for an address and its parents */ /* If this is a deferred address, and retry processing was requested by @@ -958,7 +963,7 @@ for (int i = 0; i < 3; i++) if (dbm_file) dbfn_close(dbm_file); -DEBUG(D_retry) debug_printf("end of retry processing\n"); +DEBUG(D_retry) { acl_level--; debug_printf("end of retry processing\n"); } } /* End of retry.c */ diff --git a/src/src/smtp_in.c b/src/src/smtp_in.c index c52d3f4d6..f8656a6e8 100644 --- a/src/src/smtp_in.c +++ b/src/src/smtp_in.c @@ -1688,7 +1688,6 @@ spf_result_guessed = FALSE; #ifndef DISABLE_DKIM dkim_cur_signer = dkim_signers = dkim_signing_domain = dkim_signing_selector = dkim_signatures = NULL; -dkim_cur_signer = dkim_signers = dkim_signing_domain = dkim_signing_selector = NULL; f.dkim_disable_verify = FALSE; dkim_collect_input = 0; dkim_verify_overall = dkim_verify_status = dkim_verify_reason = NULL; diff --git a/src/src/smtp_out.c b/src/src/smtp_out.c index cfc96c13c..1ff93b9e4 100644 --- a/src/src/smtp_out.c +++ b/src/src/smtp_out.c @@ -553,10 +553,10 @@ Returns: TRUE if OK, FALSE on error, with errno set static BOOL flush_buffer(smtp_outblock * outblock, int mode) { -int rc; -int n = outblock->ptr - outblock->buffer; +int n = outblock->ptr - outblock->buffer, rc; BOOL more = mode == SCMD_MORE; client_conn_ctx * cctx; +const uschar * where; HDEBUG(D_transport|D_acl) debug_printf_indent("cmd buf flush %d bytes%s\n", n, more ? " (more expected)" : ""); @@ -569,6 +569,7 @@ if (!(cctx = outblock->cctx)) } #ifndef DISABLE_TLS +where = US"tls_write"; if (cctx->tls_ctx) /*XXX have seen a null cctx here, rvfy sending QUIT, hence check above */ rc = tls_write(cctx->tls_ctx, outblock->buffer, n, more); else @@ -584,6 +585,7 @@ else requirement: TFO with data can, in rare cases, replay the data to the receiver. */ + where = US"smtp_connect"; if ( (cctx->sock = smtp_connect(outblock->conn_args, &early_data)) < 0) return FALSE; @@ -592,6 +594,7 @@ else } else { + where = US"send"; rc = send(cctx->sock, outblock->buffer, n, #ifdef MSG_MORE more ? MSG_MORE : 0 @@ -606,6 +609,7 @@ else This is despite NODELAY being active. https://bugzilla.redhat.com/show_bug.cgi?id=1803806 */ + where = US"cork"; if (!more) setsockopt(cctx->sock, IPPROTO_TCP, TCP_CORK, &off, sizeof(off)); #endif @@ -614,7 +618,8 @@ else if (rc <= 0) { - HDEBUG(D_transport|D_acl) debug_printf_indent("send failed: %s\n", strerror(errno)); + HDEBUG(D_transport|D_acl) debug_printf_indent("%s (fd %d) failed: %s\n", + where, cctx->sock, strerror(errno)); return FALSE; } @@ -625,22 +630,6 @@ return TRUE; -/* This might be called both due to callout and then from delivery. -Use memory that will not be released between those phases. -*/ -static void -smtp_debug_resp(const uschar * buf) -{ -#ifndef DISABLE_CLIENT_CMD_LOG -int old_pool = store_pool; -store_pool = POOL_PERM; -client_cmd_log = string_append_listele_n(client_cmd_log, ':', buf, - buf[3] == ' ' ? 3 : 4); -store_pool = old_pool; -#endif -} - - /************************************************* * Write SMTP command * *************************************************/ diff --git a/src/src/transport.c b/src/src/transport.c index 658fc6235..327d09cff 100644 --- a/src/src/transport.c +++ b/src/src/transport.c @@ -1499,9 +1499,8 @@ Returns: nothing void transport_update_waiting(host_item * hostlist, uschar * tpname) { -const uschar *prevname = US""; -open_db dbblock; -open_db *dbm_file; +const uschar * prevname = US""; +open_db dbblock, * dbp; if (!is_new_message_id(message_id)) { @@ -1512,10 +1511,13 @@ if (!is_new_message_id(message_id)) DEBUG(D_transport) debug_printf("updating wait-%s database\n", tpname); -/* Open the database for this transport */ +/* Open the database (or transaction) for this transport */ -if (!(dbm_file = dbfn_open(string_sprintf("wait-%.200s", tpname), - O_RDWR, &dbblock, TRUE, TRUE))) +if ( continue_wait_db + ? !dbfn_transaction_start(dbp = continue_wait_db) + : !(dbp = dbfn_open(string_sprintf("wait-%.200s", tpname), + O_RDWR, &dbblock, TRUE, TRUE)) + ) return; /* Scan the list of hosts for which this message is waiting, and ensure @@ -1536,7 +1538,7 @@ for (host_item * host = hostlist; host; host = host->next) /* Look up the host record; if there isn't one, make an empty one. */ - if (!(host_record = dbfn_read(dbm_file, host->name))) + if (!(host_record = dbfn_read(dbp, host->name))) { host_record = store_get(sizeof(dbdata_wait) + MESSAGE_ID_LENGTH, GET_UNTAINTED); host_record->count = host_record->sequence = 0; @@ -1560,9 +1562,9 @@ for (host_item * host = hostlist; host; host = host->next) debug_printf_indent("NOTE: old or corrupt message-id found in wait=%.200s" " hints DB; deleting records for %s\n", tpname, host->name); - (void) dbfn_delete(dbm_file, host->name); + (void) dbfn_delete(dbp, host->name); for (int i = host_record->sequence - 1; i >= 0; i--) - (void) dbfn_delete(dbm_file, + (void) dbfn_delete(dbp, (sprintf(CS buffer, "%.200s:%d", host->name, i), buffer)); host_record->count = host_record->sequence = 0; @@ -1579,7 +1581,7 @@ for (host_item * host = hostlist; host; host = host->next) { dbdata_wait *cont; sprintf(CS buffer, "%.200s:%d", host->name, i); - if ((cont = dbfn_read(dbm_file, buffer))) + if ((cont = dbfn_read(dbp, buffer))) { int clen = cont->count * MESSAGE_ID_LENGTH; for (uschar * s = cont->text; s < cont->text + clen; s += MESSAGE_ID_LENGTH) @@ -1605,7 +1607,7 @@ for (host_item * host = hostlist; host; host = host->next) if (host_record->count >= WAIT_NAME_MAX) { sprintf(CS buffer, "%.200s:%d", host->name, host_record->sequence); - dbfn_write(dbm_file, buffer, host_record, sizeof(dbdata_wait) + host_length); + dbfn_write(dbp, buffer, host_record, sizeof(dbdata_wait) + host_length); #ifndef DISABLE_QUEUE_RAMP if (f.queue_2stage && queue_fast_ramp && !queue_run_in_order) queue_notify_daemon(message_id); @@ -1634,14 +1636,17 @@ for (host_item * host = hostlist; host; host = host->next) /* Update the database */ - dbfn_write(dbm_file, host->name, host_record, sizeof(dbdata_wait) + host_length); + dbfn_write(dbp, host->name, host_record, sizeof(dbdata_wait) + host_length); DEBUG(D_transport) debug_printf("added %.*s to queue for %s\n", MESSAGE_ID_LENGTH, message_id, host->name); } /* All now done */ -dbfn_close(dbm_file); +if (continue_wait_db) + dbfn_transaction_commit(dbp); +else + dbfn_close(dbp); } @@ -1688,8 +1693,7 @@ transport_check_waiting(const uschar * transport_name, const uschar * hostname, { dbdata_wait * host_record; int host_length; -open_db dbblock; -open_db * dbm_file; +open_db dbblock, * dbp; int i; struct stat statbuf; @@ -1715,17 +1719,19 @@ if (local_message_max > 0 && continue_sequence >= local_message_max) /* Open the waiting information database. */ -if (!(dbm_file = dbfn_open(string_sprintf("wait-%.200s", transport_name), - O_RDWR, &dbblock, TRUE, TRUE))) +if ( continue_wait_db + ? !dbfn_transaction_start(dbp = continue_wait_db) + : !(dbp = dbfn_open(string_sprintf("wait-%.200s", transport_name), + O_RDWR, &dbblock, TRUE, TRUE)) + ) goto retfalse; /* See if there is a record for this host; if not, there's nothing to do. */ -if (!(host_record = dbfn_read(dbm_file, hostname))) +if (!(host_record = dbfn_read(dbp, hostname))) { - dbfn_close(dbm_file); DEBUG(D_transport) debug_printf_indent("no messages waiting for %s\n", hostname); - goto retfalse; + goto dbclose_false; } /* If the data in the record looks corrupt, just log something and @@ -1733,10 +1739,9 @@ don't try to use it. */ if (host_record->count > WAIT_NAME_MAX) { - dbfn_close(dbm_file); log_write(0, LOG_MAIN|LOG_PANIC, "smtp-wait database entry for %s has bad " "count=%d (max=%d)", hostname, host_record->count, WAIT_NAME_MAX); - goto retfalse; + goto dbclose_false; } /* Scan the message ids in the record in order @@ -1772,11 +1777,11 @@ while (1) DEBUG(D_hints_lookup) debug_printf_indent("NOTE: old or corrupt message-id found in wait=%.200s" " hints DB; deleting records for %s\n", transport_name, hostname); - (void) dbfn_delete(dbm_file, hostname); + (void) dbfn_delete(dbp, hostname); for (int j = host_record->sequence - 1; j >= 0; j--) - (void) dbfn_delete(dbm_file, + (void) dbfn_delete(dbp, (sprintf(CS buffer, "%.200s:%d", hostname, j), buffer)); - goto retfalse; + goto dbclose_false; } msgq[i].bKeep = TRUE; @@ -1856,20 +1861,20 @@ while (1) for (int i = host_record->sequence - 1; i >= 0 && !newr; i--) { sprintf(CS buffer, "%.200s:%d", hostname, i); - newr = dbfn_read(dbm_file, buffer); + newr = dbfn_read(dbp, buffer); } /* If no continuation, delete the current and break the loop */ if (!newr) { - dbfn_delete(dbm_file, hostname); + dbfn_delete(dbp, hostname); break; } /* Else replace the current with the continuation */ - dbfn_delete(dbm_file, buffer); + dbfn_delete(dbp, buffer); host_record = newr; host_length = host_record->count * MESSAGE_ID_LENGTH; @@ -1885,9 +1890,8 @@ while (1) if (host_length <= 0) { - dbfn_close(dbm_file); DEBUG(D_transport) debug_printf_indent("waiting messages already delivered\n"); - goto retfalse; + goto dbclose_false; } /* we were not able to find an acceptable message, nor was there a @@ -1897,8 +1901,7 @@ while (1) if (!bContinuation) { Ustrcpy(new_message_id, message_id); - dbfn_close(dbm_file); - goto retfalse; + goto dbclose_false; } } /* we need to process a continuation record */ @@ -1910,10 +1913,14 @@ record if required, close the database, and return TRUE. */ if (host_length > 0) { host_record->count = host_length/MESSAGE_ID_LENGTH; - dbfn_write(dbm_file, hostname, host_record, (int)sizeof(dbdata_wait) + host_length); + dbfn_write(dbp, hostname, host_record, (int)sizeof(dbdata_wait) + host_length); } -dbfn_close(dbm_file); +if (continue_wait_db) + dbfn_transaction_commit(dbp); +else + dbfn_close(dbp); + DEBUG(D_transport) { acl_level--; @@ -1921,9 +1928,16 @@ DEBUG(D_transport) } return TRUE; +dbclose_false: + if (continue_wait_db) + dbfn_transaction_commit(dbp); + else + dbfn_close(dbp); + retfalse: -DEBUG(D_transport) {acl_level--; debug_printf("transport_check_waiting: FALSE\n"); } -return FALSE; + DEBUG(D_transport) + {acl_level--; debug_printf("transport_check_waiting: FALSE\n"); } + return FALSE; } /************************************************* @@ -1984,7 +1998,7 @@ if (smtp_peer_options & OPTION_TLS) #endif #ifndef DISABLE_ESMTP_LIMITS -if (continue_limit_rcpt || continue_limit_rcptdom) +if (continue_limit_mail || continue_limit_rcpt || continue_limit_rcptdom) { argv[i++] = US"-MCL"; argv[i++] = string_sprintf("%u", continue_limit_mail); @@ -2038,73 +2052,6 @@ _exit(errno); /* Note: must be _exit(), NOT exit() */ -/* Fork a new exim process to deliver the message, and do a re-exec, both to -get a clean delivery process, and to regain root privilege in cases where it -has been given away. - -Arguments: - transport_name to pass to the new process - hostname ditto - hostaddress ditto - id the new message to process - socket_fd the connected socket - -Returns: FALSE if fork fails; TRUE otherwise -*/ - -BOOL -transport_pass_socket(const uschar *transport_name, const uschar *hostname, - const uschar *hostaddress, uschar *id, int socket_fd -#ifndef DISABLE_ESMTP_LIMITS - , unsigned peer_limit_mail, unsigned peer_limit_rcpt, unsigned peer_limit_rcptdom -#endif - ) -{ -pid_t pid; -int status; - -DEBUG(D_transport) debug_printf("transport_pass_socket entered\n"); - -#ifndef DISABLE_ESMTP_LIMITS -continue_limit_mail = peer_limit_mail; -continue_limit_rcpt = peer_limit_rcpt; -continue_limit_rcptdom = peer_limit_rcptdom; -#endif - -if ((pid = exim_fork(US"continued-transport-interproc")) == 0) - { - /* Disconnect entirely from the parent process. If we are running in the - test harness, wait for a bit to allow the previous process time to finish, - write the log, etc., so that the output is always in the same order for - automatic comparison. */ - - if ((pid = exim_fork(US"continued-transport")) != 0) - _exit(EXIT_SUCCESS); - testharness_pause_ms(1000); - - transport_do_pass_socket(transport_name, hostname, hostaddress, - id, socket_fd); - } - -/* If the process creation succeeded, wait for the first-level child, which -immediately exits, leaving the second level process entirely disconnected from -this one. */ - -if (pid > 0) - { - int rc; - while ((rc = wait(&status)) != pid && (rc >= 0 || errno != ECHILD)); - return TRUE; - } -else - { - DEBUG(D_transport) debug_printf("transport_pass_socket failed to fork: %s\n", - strerror(errno)); - return FALSE; - } -} - - /* Enforce all args untainted, for consistency with a router-sourced pipe command, where (because the whole line is passed as one to the tpt) a diff --git a/src/src/transports/smtp.c b/src/src/transports/smtp.c index 8296814b5..581e94f76 100644 --- a/src/src/transports/smtp.c +++ b/src/src/transports/smtp.c @@ -465,6 +465,10 @@ for (address_item * addr = addrlist; addr; addr = addr->next) if (host) { addr->host_used = host; + if (continue_sequence > 1) + { clearflag(addr, af_new_conn); setflag(addr, af_cont_conn); } + else + { clearflag(addr, af_cont_conn); setflag(addr, af_new_conn); } #ifdef EXPERIMENTAL_DSN_INFO if (smtp_greeting) {uschar * s = Ustrchr(smtp_greeting, '\n'); if (s) *s = '\0';} @@ -823,17 +827,17 @@ if (regex_match(regex_LIMITS, sx->buffer, -1, &match)) if (strncmpic(s, US"MAILMAX=", 8) == 0) { - sx->peer_limit_mail = atoi(CS (s += 8)); + continue_limit_mail = sx->peer_limit_mail = atoi(CS (s += 8)); while (isdigit(*s)) s++; } else if (strncmpic(s, US"RCPTMAX=", 8) == 0) { - sx->peer_limit_rcpt = atoi(CS (s += 8)); + continue_limit_rcpt = sx->peer_limit_rcpt = atoi(CS (s += 8)); while (isdigit(*s)) s++; } else if (strncmpic(s, US"RCPTDOMAINMAX=", 14) == 0) { - sx->peer_limit_rcptdom = atoi(CS (s += 14)); + continue_limit_rcptdom = sx->peer_limit_rcptdom = atoi(CS (s += 14)); while (isdigit(*s)) s++; } else @@ -1277,6 +1281,10 @@ if (sx->pending_MAIL) { while (addr->transport_return != PENDING_DEFER) addr = addr->next; addr->host_used = sx->conn_args.host; + if (continue_sequence > 1) + { clearflag(addr, af_new_conn); setflag(addr, af_cont_conn); } + else + { clearflag(addr, af_cont_conn); setflag(addr, af_new_conn); } addr = addr->next; } return RESP_MAIL_OR_DATA_ERROR; @@ -1297,6 +1305,10 @@ while (count-- > 0) /* The address was accepted */ addr->host_used = sx->conn_args.host; + if (continue_sequence > 1) + { clearflag(addr, af_new_conn); setflag(addr, af_cont_conn); } + else + { clearflag(addr, af_cont_conn); setflag(addr, af_new_conn); } DEBUG(D_transport) debug_printf("%s expect rcpt for %s\n", __FUNCTION__, addr->address); if (smtp_read_response(sx, sx->buffer, sizeof(sx->buffer), @@ -2248,16 +2260,22 @@ if (continue_hostname && continue_proxy_cipher) else { DEBUG(D_transport) - debug_printf("Closing proxied-TLS connection due to SNI mismatch\n"); +# ifdef SUPPORT_DANE + if (continue_proxy_dane != sx->conn_args.dane) + debug_printf( + "Closing proxied-TLS connection due to dane requirement mismatch\n"); + else +# endif + debug_printf("Closing proxied-TLS connection (SNI '%s') " + "due to SNI mismatch (transport requirement '%s')\n", + continue_proxy_sni, sni); smtp_debug_cmd(US"QUIT", 0); write(0, "QUIT\r\n", 6); close(0); continue_hostname = continue_proxy_cipher = NULL; f.continue_more = FALSE; - continue_sequence = 1; /* Unfortunately, this process cannot affect success log - which is done by delivery proc. Would have to pass this - back through reporting pipe. */ + continue_sequence = 1; /* Ensure proper logging of non-cont-conn */ } } #endif /*!DISABLE_TLS*/ @@ -2268,13 +2286,8 @@ specially so they can be identified for retries. */ if (!continue_hostname) { - if (sx->verify) - HDEBUG(D_verify) debug_printf("interface=%s port=%d\n", sx->conn_args.interface, sx->port); - - /* Arrange to report to calling process this is a new connection */ - - clearflag(sx->first_addr, af_cont_conn); - setflag(sx->first_addr, af_new_conn); + if (sx->verify) HDEBUG(D_verify) + debug_printf("interface=%s port=%d\n", sx->conn_args.interface, sx->port); /* Get the actual port the connection will use, into sx->conn_args.host */ @@ -2316,9 +2329,7 @@ if (!continue_hostname) sx->peer_limit_mail = sx->peer_limit_rcpt = sx->peer_limit_rcptdom = #endif sx->avoid_option = sx->peer_offered = smtp_peer_options = 0; -#ifndef DISABLE_CLIENT_CMD_LOG - client_cmd_log = NULL; -#endif + smtp_debug_cmd_log_init(); #ifndef DISABLE_PIPE_CONNECT if ( verify_check_given_host(CUSS &ob->hosts_pipe_connect, @@ -2540,7 +2551,8 @@ goto SEND_QUIT; if ( (ob->hosts_require_auth || ob->hosts_try_auth) && f.smtp_in_early_pipe_no_auth) { - DEBUG(D_transport) debug_printf("may need to auth, so pipeline no further\n"); + DEBUG(D_transport) + debug_printf("may need to auth, so pipeline no further\n"); if (smtp_write_command(sx, SCMD_FLUSH, NULL) < 0) goto SEND_FAILED; if (sync_responses(sx, 2, 0) != RESP_NOERROR) @@ -2631,7 +2643,8 @@ goto SEND_QUIT; if ( (sx->peer_offered & (OPTION_PIPE | OPTION_EARLY_PIPE)) == (OPTION_PIPE | OPTION_EARLY_PIPE)) { - DEBUG(D_transport) debug_printf("PIPECONNECT usable in future for this IP\n"); + DEBUG(D_transport) + debug_printf("PIPECONNECT usable in future for this IP\n"); sx->ehlo_resp.cleartext_auths = study_ehlo_auths(sx); write_ehlo_cache_entry(sx); } @@ -2652,17 +2665,15 @@ goto SEND_QUIT; } } -/* For continuing deliveries down the same channel, having re-exec'd the socket +/* For continuing deliveries down the same channel, the socket is the standard input; for a socket held open from verify it is recorded in the cutthrough context block. Either way we don't need to redo EHLO here (but may need to do so for TLS - see below). -Set up the pointer to where subsequent commands will be left, for -error messages. Note that smtp_peer_options will have been -set from the command line if they were set in the process that passed the -connection on. */ +Set up the pointer "smtp_command" to where subsequent commands will be left, +for error messages. Other stuff was set up for us by the delivery process. */ /*XXX continue case needs to propagate DSN_INFO, prob. in deliver.c -as the continue goes via transport_pass_socket() and doublefork and exec. +as the continue goes via pass-fd to the delivery process. It does not wait. Unclear how we keep separate host's responses separate - we could match up by host ip+port as a bodge. */ @@ -2677,10 +2688,10 @@ else { sx->cctx.sock = 0; /* stdin */ sx->cctx.tls_ctx = NULL; - smtp_port_for_connect(sx->conn_args.host, sx->port); /* Record the port that was used */ + smtp_port_for_connect(sx->conn_args.host, sx->port); /* Record the port that was used */ } - sx->inblock.cctx = sx->outblock.cctx = &sx->cctx; smtp_command = big_buffer; + sx->inblock.cctx = sx->outblock.cctx = &sx->cctx; sx->peer_offered = smtp_peer_options; #ifndef DISABLE_ESMTP_LIMITS /* Limits passed by cmdline over exec. */ @@ -2702,6 +2713,14 @@ else sx->pipelining_used = pipelining_active = !!(smtp_peer_options & OPTION_PIPE); HDEBUG(D_transport) debug_printf("continued connection, %s TLS\n", continue_proxy_cipher ? "proxied" : "verify conn with"); + + tls_out.certificate_verified = !!(continue_flags & CTF_CV); +#ifdef SUPPORT_DANE + tls_out.dane_verified = !!(continue_flags & CTF_DV); +#endif +#ifndef DISABLE_TLS_RESUME + if (continue_flags & CTF_TR) tls_out.resumption |= RESUME_USED; +#endif return OK; } HDEBUG(D_transport) debug_printf("continued connection, no TLS\n"); @@ -3285,7 +3304,6 @@ sx->cctx.sock = -1; (void) event_raise(sx->conn_args.tblock->event_action, US"tcp:close", NULL, NULL); #endif -smtp_debug_cmd_report(); continue_transport = NULL; continue_hostname = NULL; return yield; @@ -3432,7 +3450,7 @@ if (sx->peer_offered & OPTION_DSN && !(addr->dsn_flags & rf_dsnlasthop)) /* Send MAIL FROM and RCPT TO commands. See sw_mrc_t definition for return codes. - */ +*/ sw_mrc_t smtp_write_mail_and_rcpt_cmds(smtp_context * sx, int * yield) @@ -3661,6 +3679,7 @@ struct pollfd p[2] = {{.fd = tls_out.active.sock, .events = POLLIN}, int rc, i; BOOL send_tls_shutdown = TRUE; +acl_level++; close(pfd[1]); if ((rc = exim_fork(US"tls-proxy"))) _exit(rc < 0 ? EXIT_FAILURE : EXIT_SUCCESS); @@ -3711,9 +3730,12 @@ do for (int nbytes = 0; rc - nbytes > 0; nbytes += i) if ((i = write(pfd[0], buf + nbytes, rc - nbytes)) < 0) goto done; - /* Handle outbound data. We cannot combine payload and the TLS-close - due to the limitations of the (pipe) channel feeding us. Maybe use a unix-domain - socket? */ + /* Handle outbound data. We cannot yet combine payload and the TLS-close + due to the limitations of the (pipe) channel feeding us. Could we use + a poll/POLLRDHUP? Would that need an extra poll call after every read + (likely not worth it), or (best case) could we get POLLIN+POLLRDHUP for + the final data blob? */ + if (p[1].revents & POLLIN) if ((rc = read(pfd[0], buf, bsize)) <= 0) { @@ -3802,7 +3824,6 @@ int save_errno; int rc; uschar *message = NULL; -uschar new_message_id[MESSAGE_ID_LENGTH + 1]; smtp_context * sx = store_get(sizeof(*sx), GET_TAINTED); /* tainted, for the data buffers */ BOOL pass_message = FALSE; #ifndef DISABLE_ESMTP_LIMITS @@ -3811,9 +3832,10 @@ BOOL mail_limit = FALSE; #ifdef SUPPORT_DANE BOOL dane_held; #endif -BOOL tcw_done = FALSE, tcw = FALSE; +BOOL tcw_done = FALSE, tcw = FALSE, passback_conn = FALSE; *message_defer = FALSE; +continue_next_id[0] = '\0'; memset(sx, 0, sizeof(*sx)); sx->addrlist = addrlist; @@ -4134,7 +4156,7 @@ else && #endif transport_check_waiting(tblock->name, host->name, - tblock->connection_max_messages, new_message_id, + tblock->connection_max_messages, continue_next_id, (oicf)smtp_are_same_identities, (void*)&t_compare); if (!tcw) { @@ -4374,6 +4396,10 @@ else addr->delivery_time = delivery_time; addr->special_action = flag; addr->message = conf; + if (continue_sequence > 1) + { clearflag(addr, af_new_conn); setflag(addr, af_cont_conn); } + else + { clearflag(addr, af_cont_conn); setflag(addr, af_new_conn); } if (tcp_out_fastopen) { @@ -4534,10 +4560,11 @@ if (!sx->ok) case ERRNO_SMTPCLOSED: /* If the peer closed the TCP connection after end-of-data, but before - we could send QUIT, do TLS close, etc - call it a message error. - Otherwise, if all the recipients have been dealt with, call a close no - error at all; each address_item should have a suitable result already - (2xx: PENDING_OK, 4xx: DEFER, 5xx: FAIL) */ + we could send QUIT, do TLS close, etc - it is a message error. + If not, and all the recipients have been dealt with, call such a close + no error at all; each address_item should have a suitable result already + (2xx: PENDING_OK, 4xx: DEFER, 5xx: FAIL). + Otherwise, it is a non-message error. */ if (!(message_error = Ustrncmp(smtp_command,"end ",4) == 0)) { @@ -4659,9 +4686,9 @@ message (indicated by first_addr being non-NULL) we want to carry on with the rest of them. Also, it is desirable to send more than one message down the SMTP connection if there are several waiting, provided we haven't already sent so many as to hit the configured limit. The function transport_check_waiting looks -for a waiting message and returns its id. Then transport_pass_socket tries to -set up a continued delivery by passing the socket on to another process. The -variable send_rset is FALSE if a message has just been successfully transferred. +for a waiting message and returns its id. We pass it back to the delivery +process via the reporting pipe. The variable send_rset is FALSE if a message has +just been successfully transferred. If we are already sending down a continued channel, there may be further addresses not yet delivered that are aimed at the same host, but which have not @@ -4694,6 +4721,12 @@ if (sx->completed_addr && sx->ok && sx->send_quit) { DEBUG(D_transport) debug_printf("reached limit %u for MAILs per conn\n", sx->max_mail); + /* We will close the smtp session and connection, and clear + continue_hostname. Then if there are further addrs for the message we will + loop to the top of this function and make a fresh connection. Any further + message found in the wait-tpt hintsdb would then do a pass-fd over the + transport reporting pipe to get the connection fd back to the delivery + process. */ } else #endif @@ -4701,9 +4734,9 @@ if (sx->completed_addr && sx->ok && sx->send_quit) smtp_compare_t t_compare = {.tblock = tblock, .current_sender_address = sender_address}; - if ( sx->first_addr /* more addrs for this message */ - || f.continue_more /* more addrs for continued-host */ - || tcw_done && tcw /* more messages for host */ + if ( sx->first_addr /* more addrs for this message */ + || f.continue_more /* more addrs for continued-host */ + || tcw_done && tcw /* more messages for host */ || ( #ifndef DISABLE_TLS ( tls_out.active.sock < 0 && !continue_proxy_cipher @@ -4712,7 +4745,7 @@ if (sx->completed_addr && sx->ok && sx->send_quit) && #endif transport_check_waiting(tblock->name, host->name, - sx->max_mail, new_message_id, + sx->max_mail, continue_next_id, (oicf)smtp_are_same_identities, (void*)&t_compare) ) ) { @@ -4746,8 +4779,7 @@ if (sx->completed_addr && sx->ok && sx->send_quit) #ifndef DISABLE_TLS int pfd[2]; #endif - int socket_fd = sx->cctx.sock; - + continue_fd = sx->cctx.sock; if (sx->first_addr) /* More addresses still to be sent */ { /* for this message */ #ifndef DISABLE_ESMTP_LIMITS @@ -4757,117 +4789,116 @@ if (sx->completed_addr && sx->ok && sx->send_quit) a->transport_return = PENDING_DEFER; #endif continue_sequence++; /* for consistency */ - clearflag(sx->first_addr, af_new_conn); - setflag(sx->first_addr, af_cont_conn); /* Causes * in logging */ pipelining_active = sx->pipelining_used; /* was cleared at DATA */ goto SEND_MESSAGE; } - /* Unless caller said it already has more messages listed for this host, - pass the connection on to a new Exim process (below, the call to - transport_pass_socket). If the caller has more ready, just return with - the connection still open. */ + /* If there is a next-message-id from the wait-transport hintsdb, + pretend caller said it has further message for us. Note that we lose + the TLS session (below), and that our caller will pass back the id to + the delivery process. */ + + if (f.continue_more) + { + passback_conn = TRUE; + continue_next_id[0] = '\0'; + } + else if (*continue_next_id) + passback_conn = f.continue_more = TRUE; #ifndef DISABLE_TLS + /* If we will be returning with the connection still open and have a TLS + endpoint, shut down TLS if we must, or if this is a first-time passback + fork a proxy process with the TLS state. */ + if (tls_out.active.sock >= 0) - if ( f.continue_more - || verify_check_given_host(CUSS &ob->hosts_noproxy_tls, host) == OK) + { + if ( (continue_hostname || passback_conn) + && verify_check_given_host(CUSS &ob->hosts_noproxy_tls, host) == OK + ) { - /* Before passing the socket on, or returning to caller with it still - open, we must shut down TLS. Not all MTAs allow for the continuation - of the SMTP session when TLS is shut down. We test for this by sending - a new EHLO. If we don't get a good response, we don't attempt to pass - the socket on. + /* Not all MTAs allow for the continuation of the SMTP session when + TLS is shut down. We test for this by sending a new EHLO. If we + don't get a good response, we don't attempt to pass the socket on. NB: TLS close is *required* per RFC 9266 when tls-exporter info has been used, which we do under TLSv1.3 for the gsasl SCRAM*PLUS methods. - But we were always doing it anyway. */ - - tls_close(sx->cctx.tls_ctx, - sx->send_tlsclose ? TLS_SHUTDOWN_WAIT : TLS_SHUTDOWN_WONLY); - sx->send_tlsclose = FALSE; - sx->cctx.tls_ctx = NULL; - tls_out.active.sock = -1; - smtp_peer_options = smtp_peer_options_wrap; - sx->ok = !sx->smtps - && smtp_write_command(sx, SCMD_FLUSH, "EHLO %s\r\n", sx->helo_data) - >= 0 - && smtp_read_response(sx, sx->buffer, sizeof(sx->buffer), - '2', ob->command_timeout); - - if (sx->ok && f.continue_more) - goto TIDYUP; /* More addresses for another run */ + XXX TODO */ + + tls_close(sx->cctx.tls_ctx, + sx->send_tlsclose ? TLS_SHUTDOWN_WAIT : TLS_SHUTDOWN_WONLY); + sx->send_tlsclose = FALSE; + sx->cctx.tls_ctx = NULL; + tls_out.active.sock = -1; + smtp_peer_options = smtp_peer_options_wrap; + sx->ok = !sx->smtps + && smtp_write_command(sx, SCMD_FLUSH, "EHLO %s\r\n",sx->helo_data) + >= 0 + && smtp_read_response(sx, sx->buffer, sizeof(sx->buffer), + '2', ob->command_timeout); } - else + else if (passback_conn) { /* Set up a pipe for proxying TLS for the new transport process */ smtp_peer_options |= OPTION_TLS; if ((sx->ok = socketpair(AF_UNIX, SOCK_STREAM, 0, pfd) == 0)) - socket_fd = pfd[1]; - else - set_errno(sx->first_addr, errno, US"internal allocation problem", - DEFER, FALSE, host, -# ifdef EXPERIMENTAL_DSN_INFO - sx->smtp_greeting, sx->helo_response, -# endif - &sx->delivery_start); - } - else -#endif - if (f.continue_more) - goto TIDYUP; /* More addresses for another run */ - - /* If the socket is successfully passed, we mustn't send QUIT (or - indeed anything!) from here. */ - - /*XXX DSN_INFO: assume likely to do new HELO; but for greet we'll want to - propagate it from the initial - */ - if (sx->ok && transport_pass_socket(tblock->name, host->name, - host->address, new_message_id, socket_fd -#ifndef DISABLE_ESMTP_LIMITS - , sx->peer_limit_mail, sx->peer_limit_rcpt, sx->peer_limit_rcptdom -#endif - )) - { - sx->send_quit = FALSE; - - /* We have passed the client socket to a fresh transport process. - If TLS is still active, we need to proxy it for the transport we - just passed the baton to. Fork a child to to do it, and return to - get logging done asap. Which way to place the work makes assumptions - about post-fork prioritisation which may not hold on all platforms. */ -#ifndef DISABLE_TLS - if (tls_out.active.sock >= 0) - { - int pid = exim_fork(US"tls-proxy-interproc"); - if (pid == 0) /* child; fork again to disconnect totally */ { - /* does not return */ - smtp_proxy_tls(sx->cctx.tls_ctx, sx->buffer, sizeof(sx->buffer), pfd, - ob->command_timeout, host->name); - } + int pid = exim_fork(US"tls-proxy-interproc"); + if (pid == 0) /* child; fork again to disconnect totally */ + { + /* does not return */ + smtp_proxy_tls(sx->cctx.tls_ctx, sx->buffer, sizeof(sx->buffer), + pfd, ob->command_timeout, host->name); + } + + if (pid < 0) + log_write(0, LOG_PANIC_DIE, "fork failed"); - if (pid > 0) /* parent */ - { close(pfd[0]); + continue_fd = pfd[1]; /* tidy the inter-proc to disconn the proxy proc */ waitpid(pid, NULL, 0); tls_close(sx->cctx.tls_ctx, TLS_NO_SHUTDOWN); sx->cctx.tls_ctx = NULL; (void)close(sx->cctx.sock); sx->cctx.sock = -1; - continue_transport = NULL; - continue_hostname = NULL; - goto TIDYUP; + + continue_proxy_cipher = tls_out.cipher; + continue_proxy_sni = tls_out.sni; +# ifdef SUPPORT_DANE + continue_proxy_dane = tls_out.sni && tls_out.dane_verified; +# endif } - log_write(0, LOG_PANIC_DIE, "fork failed"); + else + set_errno(sx->first_addr, errno, US"internal allocation problem", + DEFER, FALSE, host, +# ifdef EXPERIMENTAL_DSN_INFO + sx->smtp_greeting, sx->helo_response, +# endif + &sx->delivery_start); } -#endif } +#endif /*DISABLE_TLS*/ + + /* If a connection re-use is possible, arrange to pass back all the info + about it so that further forks of the delivery process see it. */ + + if (passback_conn) + { + continue_transport = transport_name; + continue_hostname = host->name; + continue_host_address = host->address; + } + else + continue_hostname = NULL; + + if (sx->ok && f.continue_more) /* More addresses for another run; */ + goto TIDYUP; /* skip the channel closedown */ } - /* If RSET failed and there are addresses left, they get deferred. */ + /* If RSET failed and there are addresses left, they get deferred. + Do not pass back a next-id or conn info. */ + else set_errno(sx->first_addr, errno, msg, DEFER, FALSE, host, #ifdef EXPERIMENTAL_DSN_INFO @@ -4970,9 +5001,8 @@ if (sx->send_quit || tcw_done && !tcw) HDEBUG(D_transport|D_acl|D_v) debug_printf_indent(" SMTP(close)>>\n"); (void)close(sx->cctx.sock); sx->cctx.sock = -1; -continue_transport = NULL; continue_hostname = NULL; -smtp_debug_cmd_report(); +continue_next_id[0] = '\0'; #ifndef DISABLE_EVENT (void) event_raise(tblock->event_action, US"tcp:close", NULL, NULL); @@ -4992,8 +5022,6 @@ if (dane_held) to get the domain string for SNI */ sx->first_addr = a; - clearflag(a, af_cont_conn); - setflag(a, af_new_conn); /* clear * from logging */ DEBUG(D_transport) debug_printf("DANE: go-around for %s\n", a->domain); } } @@ -5006,25 +5034,25 @@ if (dane_held) if (mail_limit && sx->first_addr) { /* Reset the sequence count since we closed the connection. This is flagged - on the pipe back to the delivery process so that a non-continued-conn delivery - is logged. */ + on the pipe back to the delivery process so that it can reset it's count. + Also set flags on the addr so that a non-continued-conn delivery is logged. */ continue_sequence = 1; /* for consistency */ - clearflag(sx->first_addr, af_cont_conn); - setflag(sx->first_addr, af_new_conn); /* clear * from logging */ - goto REPEAT_CONN; + goto REPEAT_CONN; /* open a fresh connection */ } #endif -return yield; +OUT: + smtp_debug_cmd_report(); + return yield; TIDYUP: #ifdef SUPPORT_DANE -if (dane_held) for (address_item * a = sx->addrlist->next; a; a = a->next) - if (a->transport_return == DANE) - a->transport_return = PENDING_DEFER; + if (dane_held) for (address_item * a = sx->addrlist->next; a; a = a->next) + if (a->transport_return == DANE) + a->transport_return = PENDING_DEFER; #endif -return yield; + goto OUT; } @@ -5510,7 +5538,7 @@ retry_non_continued: result of the lookup. Set expired FALSE, to save the outer loop executing twice. */ - if (continue_hostname) + if (continue_sequence > 1) if ( Ustrcmp(continue_hostname, host->name) != 0 || Ustrcmp(continue_host_address, host->address) != 0 ) @@ -5955,7 +5983,7 @@ retry_non_continued: case when we were trying to deliver down an existing channel and failed. Don't try any other hosts in this case. */ - if (continue_hostname) break; + if (continue_sequence > 1) break; /* If the whole delivery, or some individual addresses, were deferred and there are more hosts that could be tried, do not count this host towards @@ -6006,7 +6034,7 @@ retry_non_continued: for routing that changes from run to run, or big multi-IP sites with round-robin DNS. */ - if (continue_hostname && !continue_host_tried) + if (continue_sequence > 1 && !continue_host_tried) { int fd = cutthrough.cctx.sock >= 0 ? cutthrough.cctx.sock : 0; @@ -6030,6 +6058,7 @@ retry_non_continued: (void) close(fd); cutthrough.cctx.sock = -1; continue_hostname = NULL; + continue_sequence = 1; goto retry_non_continued; } diff --git a/src/src/verify.c b/src/src/verify.c index b1e5d6802..ab56ed374 100644 --- a/src/src/verify.c +++ b/src/src/verify.c @@ -714,6 +714,8 @@ tls_retry_connection: #endif if (yield != OK) { + smtp_debug_cmd_report(); /*XXX we seem to exit without what should + be a common call to this. How? */ errno = addr->basic_errno; /* For certain errors we want specifically to log the transport name, commit 380482840626329785618df79dadf5ab04688c49 Author: Jeremy Harris Date: Fri Jul 12 12:22:11 2024 +0100 Build: drop printf-like annotations diff --git a/src/src/dbfn.c b/src/src/dbfn.c index d1d8c08d4..d64f1927b 100644 --- a/src/src/dbfn.c +++ b/src/src/dbfn.c @@ -145,7 +145,7 @@ open_db * dbfn_open(const uschar * name, int flags, open_db * dbblock, BOOL lof, BOOL panic) { -int rc, save_errno; +int rc, save_errno, dlen, flen; flock_t lock_data; uschar dirname[PATHLEN], filename[PATHLEN]; @@ -164,8 +164,11 @@ make the directory as well, just in case. We won't be doing this many times unnecessarily, because usually the lock file will be there. If the directory exists, there is no error. */ -snprintf(CS dirname, sizeof(dirname), "%s/db", spool_directory); -snprintf(CS filename, sizeof(filename), "%s/%s.lockfile", dirname, name); +dlen = snprintf(CS dirname, sizeof(dirname), "%s/db", spool_directory); +flen = Ustrlen(name); +snprintf(CS filename, sizeof(filename), "%.*s/%.*s.lockfile", + (int)sizeof(filename) - dlen - flen - 11, dirname, + flen, name); dbblock->lockfd = -1; if (!exim_lockfile_needed()) @@ -189,7 +192,7 @@ it easy to pin this down, there are now debug statements on either side of the open call. */ flags &= O_RDONLY | O_RDWR; -snprintf(CS filename, sizeof(filename), "%s/%s", dirname, name); +snprintf(CS filename, sizeof(filename), "%.*s/%s", dlen, dirname, name); priv_drop_temp(exim_uid, exim_gid); dbblock->dbptr = exim_dbopen(filename, dirname, flags, EXIMDB_MODE); @@ -244,7 +247,7 @@ starting a transaction. "lof" and "panic" always true; read/write mode. open_db * dbfn_open_multi(const uschar * name, int flags, open_db * dbblock) { -int rc, save_errno; +int rc, save_errno, dlen; flock_t lock_data; uschar dirname[PATHLEN], filename[PATHLEN]; @@ -253,8 +256,8 @@ DEBUG(D_hints_lookup) acl_level++; dbblock->lockfd = -1; db_dir_make(TRUE); -snprintf(CS dirname, sizeof(dirname), "%s/db", spool_directory); -snprintf(CS filename, sizeof(filename), "%s/%s", dirname, name); +dlen = snprintf(CS dirname, sizeof(dirname), "%s/db", spool_directory); +snprintf(CS filename, sizeof(filename), "%.*s/%s", dlen, dirname, name); priv_drop_temp(exim_uid, exim_gid); dbblock->dbptr = exim_dbopen_multi(filename, dirname, flags, EXIMDB_MODE); diff --git a/src/src/functions.h b/src/src/functions.h index afb6fd46f..21214d15b 100644 --- a/src/src/functions.h +++ b/src/src/functions.h @@ -1256,12 +1256,13 @@ timediff(diff, then); static inline uschar * string_timediff(const struct timeval * diff) { -static uschar buf[sizeof("0.000s")]; +static uschar buf[16]; if (diff->tv_sec >= 5 || !LOGGING(millisec)) return readconf_printtime((int)diff->tv_sec); -snprintf(CS buf, sizeof(buf), "%u.%03us", (uint)diff->tv_sec, (uint)diff->tv_usec/1000); +snprintf(CS buf, sizeof(buf), "%u.%03us", + (uint)diff->tv_sec, (uint)diff->tv_usec/1000); return buf; } diff --git a/src/src/mytypes.h b/src/src/mytypes.h index c39083be8..cac4ee3c9 100644 --- a/src/src/mytypes.h +++ b/src/src/mytypes.h @@ -30,15 +30,13 @@ local_scan.h includes it and exim.h includes them both (to get this earlier). */ #endif -/* If gcc is being used to compile Exim, we can use its facility for checking -the arguments of printf-like functions. This is done by a macro. -OpenBSD has unfortunately taken to objecting to use of %n in printf -so we have to give up on all of the available parameter checking. */ +/* We gave up on trying to get compilers to check on printf-like functions +because they are both whiney about value sizes where they cannot do decent +static analysis, and incapable of handling extensions to printf formats. +The annotation on functions is still in place but does nothing. */ #if defined(__GNUC__) || defined(__clang__) -# ifndef __OpenBSD__ -# define PRINTF_FUNCTION(A,B) __attribute__((format(printf,A,B))) -# endif +/* # define PRINTF_FUNCTION(A,B) __attribute__((format(printf,A,B))) */ # define ARG_UNUSED __attribute__((__unused__)) # define FUNC_MAYBE_UNUSED __attribute__((__unused__)) # define WARN_UNUSED_RESULT __attribute__((__warn_unused_result__)) diff --git a/src/src/tod.c b/src/src/tod.c index 364703d53..560f69bee 100644 --- a/src/src/tod.c +++ b/src/src/tod.c @@ -169,6 +169,7 @@ switch(type) diff_min += (local.tm_yday > gmt->tm_yday) ? 1440 : -1440; diff_hour = diff_min/60; diff_min = abs(diff_min - diff_hour*60); + diff_min &= 63; /* compiler quietening */ } else /* subminute offset, eg. TAI */ { commit 47c4ed10319e4048b75b10103f514290b3a056ee Author: Jeremy Harris Date: Tue Jul 16 12:30:43 2024 +0100 Debug: indents diff --git a/src/src/deliver.c b/src/src/deliver.c index 33d833389..7576682f4 100644 --- a/src/src/deliver.c +++ b/src/src/deliver.c @@ -2921,7 +2921,7 @@ while (addr_local) if (dbm_file) { - dbdata_retry *retry_record = dbfn_read(dbm_file, retry_key); + dbdata_retry * retry_record = dbfn_read(dbm_file, retry_key); /* If there is no retry record, delivery happens. If there is, remember it exists so it can be deleted after a successful delivery. */ @@ -7447,7 +7447,10 @@ while (addr_new) /* Loop until all addresses dealt with */ continue_retry_db = NULL; if (continue_retry_db) + { + DEBUG(D_hints_lookup) debug_printf("using cached retry hintsdb handle\n"); dbm_file = continue_retry_db; + } else if (!exim_lockfile_needed() && continue_transport) { dbm_file = dbfn_open_multi(US"retry", O_RDONLY, &dbblock); @@ -7703,6 +7706,11 @@ while (addr_new) /* Loop until all addresses dealt with */ sender attached, because this form is used by the smtp transport after a 4xx response to RCPT when address_retry_include_sender is true. */ + DEBUG(D_retry) + { + debug_printf_indent("checking router retry status\n"); + acl_level++; + } addr->domain_retry_key = string_sprintf("R:%s", addr->domain); addr->address_retry_key = string_sprintf("R:%s@%s", addr->local_part, addr->domain); @@ -7715,7 +7723,7 @@ while (addr_new) /* Loop until all addresses dealt with */ ) { DEBUG(D_deliver|D_retry) - debug_printf("domain retry record present but expired\n"); + debug_printf_indent("domain retry record present but expired\n"); domain_retry_record = NULL; /* Ignore if too old */ } @@ -7725,7 +7733,7 @@ while (addr_new) /* Loop until all addresses dealt with */ ) { DEBUG(D_deliver|D_retry) - debug_printf("address retry record present but expired\n"); + debug_printf_indent("address retry record present but expired\n"); address_retry_record = NULL; /* Ignore if too old */ } @@ -7738,7 +7746,7 @@ while (addr_new) /* Loop until all addresses dealt with */ && now - address_retry_record->time_stamp > retry_data_expire) { DEBUG(D_deliver|D_retry) - debug_printf("address retry record present but expired\n"); + debug_printf_indent("address retry record present but expired\n"); address_retry_record = NULL; /* Ignore if too old */ } } @@ -7749,18 +7757,19 @@ while (addr_new) /* Loop until all addresses dealt with */ DEBUG(D_deliver|D_retry) { if (!domain_retry_record) - debug_printf("no domain retry record\n"); + debug_printf_indent("no domain retry record\n"); else - debug_printf("have domain retry record; next_try = now%+d\n", + debug_printf_indent("have domain retry record; next_try = now%+d\n", f.running_in_test_harness ? 0 : (int)(domain_retry_record->next_try - now)); if (!address_retry_record) - debug_printf("no address retry record\n"); + debug_printf_indent("no address retry record\n"); else - debug_printf("have address retry record; next_try = now%+d\n", + debug_printf_indent("have address retry record; next_try = now%+d\n", f.running_in_test_harness ? 0 : (int)(address_retry_record->next_try - now)); + acl_level--; } /* If we are sending a message down an existing SMTP connection, we must diff --git a/src/src/retry.c b/src/src/retry.c index fdcb6abea..071c94cd8 100644 --- a/src/src/retry.c +++ b/src/src/retry.c @@ -554,7 +554,7 @@ retry_update(address_item ** addr_defer, address_item ** addr_failed, open_db dbblock, * dbm_file = NULL; time_t now = time(NULL); -DEBUG(D_retry) { debug_printf("Processing retry items\n"); acl_level++; } +DEBUG(D_retry) { debug_printf_indent("Processing retry items\n"); acl_level++; } /* Three-times loop to handle succeeded, failed, and deferred addresses. Deferred addresses must be handled after failed ones, because some may be moved @@ -567,8 +567,12 @@ for (int i = 0; i < 3; i++) address_item ** paddr = i==0 ? addr_succeed : i==1 ? addr_failed : addr_defer; address_item ** saved_paddr = NULL; - DEBUG(D_retry) debug_printf_indent("%s addresses:\n", - i == 0 ? "Succeeded" : i == 1 ? "Failed" : "Deferred"); + DEBUG(D_retry) + { + debug_printf_indent("%s addresses:\n", + i == 0 ? "Succeeded" : i == 1 ? "Failed" : "Deferred"); + acl_level++; + } /* Loop for each address on the chain. For deferred addresses, the whole address times out unless one of its retry addresses has a retry rule that @@ -589,8 +593,12 @@ for (int i = 0; i < 3; i++) int update_count = 0; int timedout_count = 0; - DEBUG(D_retry) debug_printf_indent(" %s%s\n", addr->address, - addr->retries ? "" : ": no retry items"); + DEBUG(D_retry) + { + debug_printf_indent("%s%s\n", addr->address, + addr->retries ? "" : ": no retry items"); + acl_level++; + } /* Loop for each retry item. */ @@ -883,6 +891,7 @@ for (int i = 0; i < 3; i++) (void)dbfn_write(dbm_file, rti->key, retry_record, sizeof(dbdata_retry) + message_length); } /* Loop for each retry item */ + DEBUG(D_retry) acl_level--; /* If all the non-delete retry items are timed out, the address is timed out, provided that we didn't skip any hosts because their retry @@ -957,13 +966,14 @@ for (int i = 0; i < 3; i++) paddr = &(endaddr->next); /* Advance to next address */ } /* Loop for all addresses */ + DEBUG(D_retry) acl_level--; } /* Loop for succeed, fail, defer */ /* Close and unlock the database */ if (dbm_file) dbfn_close(dbm_file); -DEBUG(D_retry) { acl_level--; debug_printf("end of retry processing\n"); } +DEBUG(D_retry) { acl_level--; debug_printf_indent("end of retry processing\n"); } } /* End of retry.c */ diff --git a/src/src/transport.c b/src/src/transport.c index 327d09cff..398022354 100644 --- a/src/src/transport.c +++ b/src/src/transport.c @@ -1724,13 +1724,18 @@ if ( continue_wait_db : !(dbp = dbfn_open(string_sprintf("wait-%.200s", transport_name), O_RDWR, &dbblock, TRUE, TRUE)) ) + { + DEBUG(D_transport) + debug_printf_indent("no messages waiting for %s\n", hostname); goto retfalse; + } /* See if there is a record for this host; if not, there's nothing to do. */ if (!(host_record = dbfn_read(dbp, hostname))) { - DEBUG(D_transport) debug_printf_indent("no messages waiting for %s\n", hostname); + DEBUG(D_transport) + debug_printf_indent("no messages waiting for %s\n", hostname); goto dbclose_false; } commit e790c070e6b3a0471642fa182948778304751c8d Author: Jeremy Harris Date: Sat Jul 13 16:38:45 2024 +0100 tidying diff --git a/src/src/filter.c b/src/src/filter.c index 813ffdd7c..76749510a 100644 --- a/src/src/filter.c +++ b/src/src/filter.c @@ -795,7 +795,7 @@ static void indent(void) { int i; -for (i = 0; i < output_indent; i++) debug_printf(" "); +DEBUG(D_filter) for (i = 0; i < output_indent; i++) debug_printf(" "); } @@ -1669,8 +1669,8 @@ if ((filter_test != FTEST_NONE && debug_selector != 0) || { indent(); debug_printf_indent("%sondition is %s: ", - toplevel? "C" : "Sub-c", - (yield == c->testfor)? "true" : "false"); + toplevel ? "C" : "Sub-c", + yield == c->testfor ? "true" : "false"); print_condition(c, TRUE); debug_printf_indent("\n"); } @@ -1818,11 +1818,11 @@ while (commands) { indent(); printf("%seliver message to: %s%s%s%s\n", - (commands->seen)? "D" : "Unseen d", + commands->seen ? "D" : "Unseen d", expargs[0], commands->noerror? " (noerror)" : "", - (s != NULL)? " errors_to " : "", - (s != NULL)? s : US""); + s ? " errors_to " : "", + s ? s : US""); } /* Real case. */ @@ -1830,11 +1830,11 @@ while (commands) else { DEBUG(D_filter) debug_printf_indent("Filter: %sdeliver message to: %s%s%s%s\n", - (commands->seen)? "" : "unseen ", + commands->seen ? "" : "unseen ", expargs[0], - commands->noerror? " (noerror)" : "", - (s != NULL)? " errors_to " : "", - (s != NULL)? s : US""); + commands->noerror ? " (noerror)" : "", + s ? " errors_to " : "", + s ? s : US""); /* Create the new address and add it to the chain, setting the af_ignore_error flag if necessary, and the errors address, which can be @@ -1858,11 +1858,13 @@ while (commands) { indent(); if (mode < 0) - printf("%save message to: %s%s\n", (commands->seen)? - "S" : "Unseen s", s, commands->noerror? " (noerror)" : ""); + printf("%save message to: %s%s\n", + commands->seen ? "S" : "Unseen s", + s, commands->noerror ? " (noerror)" : ""); else - printf("%save message to: %s %04o%s\n", (commands->seen)? - "S" : "Unseen s", s, mode, commands->noerror? " (noerror)" : ""); + printf("%save message to: %s %04o%s\n", + commands->seen ? "S" : "Unseen s", + s, mode, commands->noerror ? " (noerror)" : ""); } /* Real case: Ensure save argument starts with / if there is a home @@ -1874,8 +1876,8 @@ while (commands) deliver_home != NULL && deliver_home[0] != 0) s = string_sprintf("%s/%s", deliver_home, s); DEBUG(D_filter) debug_printf_indent("Filter: %ssave message to: %s%s\n", - (commands->seen)? "" : "unseen ", s, - commands->noerror? " (noerror)" : ""); + commands->seen ? "" : "unseen ", + s, commands->noerror ? " (noerror)" : ""); /* Create the new address and add it to the chain, setting the af_pfr and af_file flags, the af_ignore_error flag if necessary, and the @@ -1896,8 +1898,9 @@ while (commands) if (filter_test != FTEST_NONE) { indent(); - printf("%sipe message to: %s%s\n", (commands->seen)? - "P" : "Unseen p", s, commands->noerror? " (noerror)" : ""); + printf("%sipe message to: %s%s\n", + commands->seen ? "P" : "Unseen p", + s, commands->noerror? " (noerror)" : ""); } else /* Ensure pipe command starts with | */ { @@ -1953,7 +1956,7 @@ while (commands) if (filter_test != FTEST_NONE) { indent(); - printf("%sogfile %s\n", (commands->seen)? "Seen l" : "L", log_filename); + printf("%sogfile %s\n", commands->seen ? "Seen l" : "L", log_filename); } break; @@ -1963,14 +1966,14 @@ while (commands) if (filter_test != FTEST_NONE) { indent(); - printf("%sogwrite \"%s\"\n", (commands->seen)? "Seen l" : "L", + printf("%sogwrite \"%s\"\n", commands->seen ? "Seen l" : "L", string_printing(s)); } /* Attempt to write to a log file only if configured as permissible. Logging may be forcibly skipped for verifying or testing. */ - else if ((filter_options & RDO_LOG) != 0) /* Locked out */ + else if (filter_options & RDO_LOG) /* Locked out */ { DEBUG(D_filter) debug_printf_indent("filter log command aborted: euid=%ld\n", @@ -1978,7 +1981,7 @@ while (commands) *error_pointer = US"logwrite command forbidden"; return FF_ERROR; } - else if ((filter_options & RDO_REALLOG) != 0) + else if (filter_options & RDO_REALLOG) { int len; DEBUG(D_filter) debug_printf_indent("writing filter log as euid %ld\n", @@ -2094,7 +2097,7 @@ while (commands) if (filter_test != FTEST_NONE) { indent(); - printf("%sinish\n", (commands->seen)? "Seen f" : "F"); + printf("%sinish\n", commands->seen ? "Seen f" : "F"); } else DEBUG(D_filter) debug_printf_indent("Filter: %sfinish\n", @@ -2112,7 +2115,7 @@ while (commands) else { output_indent += 2; - ok = interpret_commands(commands->args[condition_value? 1:2].f, + ok = interpret_commands(commands->args[condition_value ? 1:2].f, generated); output_indent -= 2; } @@ -2134,7 +2137,8 @@ while (commands) if (filter_test != FTEST_NONE) printf("%s command ignored because return_path is empty\n", command_list[commands->command]); - else DEBUG(D_filter) debug_printf_indent("%s command ignored because return_path " + else DEBUG(D_filter) + debug_printf_indent("%s command ignored because return_path " "is empty\n", command_list[commands->command]); break; } @@ -2228,15 +2232,15 @@ while (commands) commands->noerror ? " (noerror)" : ""); for (i = 1; i < MAILARGS_STRING_COUNT; i++) { - const uschar *arg = commands->args[i].u; + const uschar * arg = commands->args[i].u; if (arg) { int len = Ustrlen(mailargs[i]); - int indent = (debug_selector != 0)? output_indent : 0; + int indent = debug_selector != 0 ? output_indent : 0; while (len++ < 7 + indent) printf(" "); printf("%s: %s%s\n", mailargs[i], string_printing(arg), - (commands->args[mailarg_index_expand].u != NULL && - Ustrcmp(mailargs[i], "file") == 0)? " (expanded)" : ""); + ( commands->args[mailarg_index_expand].u + && Ustrcmp(mailargs[i], "file") == 0) ? " (expanded)" : ""); } } if (commands->args[mailarg_index_return].u) commit 4cbf0dff96031f9b53cf6d8a3c4b2001d467a57d Author: Jeremy Harris Date: Tue Jul 16 21:24:48 2024 +0100 Use explicit O_CREAT for dbfn_open() diff --git a/src/src/acl.c b/src/src/acl.c index 0aa789dbf..69777cf9c 100644 --- a/src/src/acl.c +++ b/src/src/acl.c @@ -2584,7 +2584,7 @@ if ((t = tree_search(*anchor, key))) /* We aren't using a pre-computed rate, so get a previously recorded rate from the database, which will be updated and written back if required. */ -if (!(dbm = dbfn_open(US"ratelimit", O_RDWR, &dbblock, TRUE, TRUE))) +if (!(dbm = dbfn_open(US"ratelimit", O_RDWR|O_CREAT, &dbblock, TRUE, TRUE))) { store_pool = old_pool; sender_rate = NULL; @@ -2966,7 +2966,7 @@ while ((ele = string_nextinlist(&list, &slash, NULL, 0))) else goto badopt; -if (!(dbm = dbfn_open(US"seen", O_RDWR, &dbblock, TRUE, TRUE))) +if (!(dbm = dbfn_open(US"seen", O_RDWR|O_CREAT, &dbblock, TRUE, TRUE))) { HDEBUG(D_acl) debug_printf_indent("database for 'seen' not available\n"); *log_msgptr = US"database for 'seen' not available"; diff --git a/src/src/dbfn.c b/src/src/dbfn.c index d64f1927b..c2c92cf44 100644 --- a/src/src/dbfn.c +++ b/src/src/dbfn.c @@ -128,7 +128,7 @@ return TRUE; Arguments: name The single-component name of one of Exim's database files. flags Either O_RDONLY or O_RDWR, indicating the type of open required; - O_RDWR implies "create if necessary" + optionally O_CREAT dbblock Points to an open_db block to be filled in. lof If TRUE, write to the log for actual open failures (locking failures are always logged). @@ -165,16 +165,16 @@ unnecessarily, because usually the lock file will be there. If the directory exists, there is no error. */ dlen = snprintf(CS dirname, sizeof(dirname), "%s/db", spool_directory); -flen = Ustrlen(name); -snprintf(CS filename, sizeof(filename), "%.*s/%.*s.lockfile", - (int)sizeof(filename) - dlen - flen - 11, dirname, - flen, name); dbblock->lockfd = -1; if (!exim_lockfile_needed()) db_dir_make(panic); else { + flen = Ustrlen(name); + snprintf(CS filename, sizeof(filename), "%.*s/%.*s.lockfile", + (int)sizeof(filename) - dlen - flen - 11, dirname, + flen, name); if (!lockfile_take(dbblock, filename, flags == O_RDONLY, panic)) { DEBUG(D_hints_lookup) acl_level--; @@ -191,16 +191,15 @@ databases - often this is caused by non-matching db.h and the library. To make it easy to pin this down, there are now debug statements on either side of the open call. */ -flags &= O_RDONLY | O_RDWR; snprintf(CS filename, sizeof(filename), "%.*s/%s", dlen, dirname, name); priv_drop_temp(exim_uid, exim_gid); -dbblock->dbptr = exim_dbopen(filename, dirname, flags, EXIMDB_MODE); -if (!dbblock->dbptr && errno == ENOENT && flags == O_RDWR) +dbblock->dbptr = exim_dbopen(filename, dirname, flags & O_ACCMODE, EXIMDB_MODE); +if (!dbblock->dbptr && errno == ENOENT && flags & O_CREAT) { DEBUG(D_hints_lookup) debug_printf_indent("%s appears not to exist: trying to create\n", filename); - dbblock->dbptr = exim_dbopen(filename, dirname, flags|O_CREAT, EXIMDB_MODE); + dbblock->dbptr = exim_dbopen(filename, dirname, flags, EXIMDB_MODE); } save_errno = errno; priv_restore(); @@ -227,10 +226,11 @@ if (!dbblock->dbptr) } DEBUG(D_hints_lookup) - debug_printf_indent("opened hints database %s: flags=%s\n", filename, - flags == O_RDONLY ? "O_RDONLY" - : flags == O_RDWR ? "O_RDWR" - : "??"); + debug_printf_indent("opened hints database %s: flags=%s%s\n", filename, + (flags & O_ACCMODE) == O_RDONLY ? "O_RDONLY" + : (flags & O_ACCMODE) == O_RDWR ? "O_RDWR" + : "??", + flags & O_CREAT ? "|O_CREAT" : ""); /* Pass back the block containing the opened database handle and the open fd for the lock. */ @@ -260,12 +260,12 @@ dlen = snprintf(CS dirname, sizeof(dirname), "%s/db", spool_directory); snprintf(CS filename, sizeof(filename), "%.*s/%s", dlen, dirname, name); priv_drop_temp(exim_uid, exim_gid); -dbblock->dbptr = exim_dbopen_multi(filename, dirname, flags, EXIMDB_MODE); -if (!dbblock->dbptr && errno == ENOENT && flags == O_RDWR) +dbblock->dbptr = exim_dbopen_multi(filename, dirname, flags & O_ACCMODE, EXIMDB_MODE); +if (!dbblock->dbptr && errno == ENOENT && flags & O_CREAT) { DEBUG(D_hints_lookup) debug_printf_indent("%s appears not to exist: trying to create\n", filename); - dbblock->dbptr = exim_dbopen_multi(filename, dirname, O_RDWR|O_CREAT, EXIMDB_MODE); + dbblock->dbptr = exim_dbopen_multi(filename, dirname, flags, EXIMDB_MODE); } save_errno = errno; priv_restore(); @@ -289,8 +289,13 @@ if (!dbblock->dbptr) return NULL; } -DEBUG(D_hints_lookup) debug_printf_indent( - "opened hints database %s for transactions: NOLOCK flags=O_RDWR\n", filename); +DEBUG(D_hints_lookup) + debug_printf_indent("opened hints database %s for transactions: NOLOCK flags=%s%s\n", + filename, + (flags & O_ACCMODE) == O_RDONLY ? "O_RDONLY" + : (flags & O_ACCMODE) == O_RDWR ? "O_RDWR" + : "??", + flags & O_CREAT ? "|O_CREAT" : ""); /* Pass back the block containing the opened database handle */ @@ -660,7 +665,7 @@ while (Ufgets(buffer, 256, stdin) != NULL) } start = clock(); - odb = dbfn_open(s, O_RDWR, dbblock + i, TRUE, TRUE); + odb = dbfn_open(s, O_RDWR|O_CREAT, dbblock + i, TRUE, TRUE); stop = clock(); if (odb) diff --git a/src/src/deliver.c b/src/src/deliver.c index 7576682f4..b04fe6f12 100644 --- a/src/src/deliver.c +++ b/src/src/deliver.c @@ -7453,7 +7453,7 @@ while (addr_new) /* Loop until all addresses dealt with */ } else if (!exim_lockfile_needed() && continue_transport) { - dbm_file = dbfn_open_multi(US"retry", O_RDONLY, &dbblock); + dbm_file = dbfn_open_multi(US"retry", O_RDWR, &dbblock); continue_retry_db = dbm_file ? dbm_file : (open_db *)-1; } else diff --git a/src/src/enq.c b/src/src/enq.c index 43f53a585..057dffae1 100644 --- a/src/src/enq.c +++ b/src/src/enq.c @@ -44,12 +44,9 @@ open_db *dbm_file; DEBUG(D_transport) debug_printf("check serialized: %s\n", key); -/* Open and lock the waiting information database. The absence of O_CREAT is -deliberate; the dbfn_open() function - which is an Exim function - always tries -to create if it can't open a read/write file. It expects only O_RDWR or -O_RDONLY as its argument. */ +/* Open and lock the waiting information database. */ -if (!(dbm_file = dbfn_open(US"misc", O_RDWR, &dbblock, TRUE, TRUE))) +if (!(dbm_file = dbfn_open(US"misc", O_RDWR|O_CREAT, &dbblock, TRUE, TRUE))) return FALSE; /* See if there is a record for this host or queue run; if there is, we cannot diff --git a/src/src/exim_dbutil.c b/src/src/exim_dbutil.c index d3d854dd5..dd1444593 100644 --- a/src/src/exim_dbutil.c +++ b/src/src/exim_dbutil.c @@ -271,7 +271,7 @@ the lock file. Arguments: name The single-component name of one of Exim's database files. - flags O_RDONLY or O_RDWR + flags O_RDONLY or O_RDWR, O_CREAT dbblock Points to an open_db block to be filled in. lof Unused. panic Unused @@ -340,8 +340,6 @@ if (exim_lockfile_needed()) if (asprintf(CSS &filename, "%s/%s", dirname, name) < 0) return NULL; -if (flags & O_RDWR) flags |= O_CREAT; - if (!(dbblock->dbptr = exim_dbopen(filename, dirname, flags, 0))) { printf("** Failed to open hintsdb file %s for %s: %s%s\n", filename, @@ -846,7 +844,7 @@ for(; (reset_point = store_mark()); store_reset(reset_point)) { int verify = 1; - if (!(dbm = dbfn_open(aname, O_RDWR, &dbblock, FALSE, TRUE))) + if (!(dbm = dbfn_open(aname, O_RDWR|O_CREAT, &dbblock, FALSE, TRUE))) continue; if (Ustrcmp(field, "d") == 0) @@ -1209,7 +1207,7 @@ oldest = time(NULL) - maxkeep; printf("Tidying Exim hints database %s/db/%s\n", argv[1], argv[2]); spool_directory = argv[1]; -if (!(dbm = dbfn_open(argv[2], O_RDWR, &dbblock, FALSE, TRUE))) +if (!(dbm = dbfn_open(argv[2], O_RDWR|O_CREAT, &dbblock, FALSE, TRUE))) exit(EXIT_FAILURE); /* Prepare for building file names */ diff --git a/src/src/retry.c b/src/src/retry.c index 071c94cd8..e86b1afe8 100644 --- a/src/src/retry.c +++ b/src/src/retry.c @@ -617,7 +617,7 @@ for (int i = 0; i < 3; i++) reached their retry next try time. */ if (!dbm_file) - dbm_file = dbfn_open(US"retry", O_RDWR, &dbblock, TRUE, TRUE); + dbm_file = dbfn_open(US"retry", O_RDWR|O_CREAT, &dbblock, TRUE, TRUE); if (!dbm_file) { diff --git a/src/src/tls-gnu.c b/src/src/tls-gnu.c index f443a6e27..25690aed2 100644 --- a/src/src/tls-gnu.c +++ b/src/src/tls-gnu.c @@ -3385,7 +3385,7 @@ if (gnutls_session_get_flags(session) & GNUTLS_SFLAGS_SESSION_TICKET) memcpy(dt->session, tkt.data, tkt.size); gnutls_free(tkt.data); - if ((dbm_file = dbfn_open(US"tls", O_RDWR, &dbblock, FALSE, FALSE))) + if ((dbm_file = dbfn_open(US"tls", O_RDWR|O_CREAT, &dbblock, FALSE, FALSE))) { /* key for the db is the IP */ dbfn_write(dbm_file, tlsp->resume_index, dt, dlen); diff --git a/src/src/tls-openssl.c b/src/src/tls-openssl.c index c97106fe0..0a5e2a7a0 100644 --- a/src/src/tls-openssl.c +++ b/src/src/tls-openssl.c @@ -3933,7 +3933,7 @@ if (tlsp->host_resumable) tlsp->resumption |= RESUME_CLIENT_REQUESTED; DEBUG(D_tls) debug_printf("checking for resumable session for %s\n", tlsp->resume_index); - if ((dbm_file = dbfn_open(US"tls", O_RDWR, &dbblock, FALSE, FALSE))) + if ((dbm_file = dbfn_open(US"tls", O_RDWR|O_CREAT, &dbblock, FALSE, FALSE))) { if ((dt = dbfn_read_with_length(dbm_file, tlsp->resume_index, &len))) { @@ -4016,7 +4016,7 @@ if (SSL_SESSION_is_resumable(ss)) /* 1.1.1 */ dt->ocsp = tlsp->ocsp; (void) i2d_SSL_SESSION(ss, &s); /* s gets bumped to end */ - if ((dbm_file = dbfn_open(US"tls", O_RDWR, &dbblock, FALSE, FALSE))) + if ((dbm_file = dbfn_open(US"tls", O_RDWR|O_CREAT, &dbblock, FALSE, FALSE))) { dbfn_write(dbm_file, tlsp->resume_index, dt, dlen); dbfn_close(dbm_file); diff --git a/src/src/transport.c b/src/src/transport.c index 398022354..170997172 100644 --- a/src/src/transport.c +++ b/src/src/transport.c @@ -1516,7 +1516,7 @@ DEBUG(D_transport) debug_printf("updating wait-%s database\n", tpname); if ( continue_wait_db ? !dbfn_transaction_start(dbp = continue_wait_db) : !(dbp = dbfn_open(string_sprintf("wait-%.200s", tpname), - O_RDWR, &dbblock, TRUE, TRUE)) + O_RDWR|O_CREAT, &dbblock, TRUE, TRUE)) ) return; diff --git a/src/src/transports/smtp.c b/src/src/transports/smtp.c index 581e94f76..5b54fa1ae 100644 --- a/src/src/transports/smtp.c +++ b/src/src/transports/smtp.c @@ -913,7 +913,7 @@ sx->ehlo_resp.limit_rcpt = sx->peer_limit_rcpt; sx->ehlo_resp.limit_rcptdom = sx->peer_limit_rcptdom; # endif -if ((dbm_file = dbfn_open(US"misc", O_RDWR, &dbblock, TRUE, TRUE))) +if ((dbm_file = dbfn_open(US"misc", O_RDWR|O_CREAT, &dbblock, TRUE, TRUE))) { uschar * ehlo_resp_key = ehlo_cache_key(sx); dbdata_ehlo_resp er = { .data = sx->ehlo_resp }; @@ -943,7 +943,7 @@ invalidate_ehlo_cache_entry(smtp_context * sx) open_db dbblock, * dbm_file; if ( sx->early_pipe_active - && (dbm_file = dbfn_open(US"misc", O_RDWR, &dbblock, TRUE, TRUE))) + && (dbm_file = dbfn_open(US"misc", O_RDWR|O_CREAT, &dbblock, TRUE, TRUE))) { uschar * ehlo_resp_key = ehlo_cache_key(sx); HDEBUG(D_transport) @@ -981,7 +981,7 @@ else { DEBUG(D_transport) debug_printf("ehlo-resp record too old\n"); dbfn_close(dbm_file); - if ((dbm_file = dbfn_open(US"misc", O_RDWR, &dbblock, TRUE, TRUE))) + if ((dbm_file = dbfn_open(US"misc", O_RDWR|O_CREAT, &dbblock, TRUE, TRUE))) dbfn_delete(dbm_file, ehlo_resp_key); } else diff --git a/src/src/verify.c b/src/src/verify.c index ab56ed374..25fc700f9 100644 --- a/src/src/verify.c +++ b/src/src/verify.c @@ -121,7 +121,7 @@ if (options & vopt_callout_no_cache) { HDEBUG(D_verify) debug_printf_indent("callout cache: disabled by no_cache\n"); } -else if (!(dbm_file = dbfn_open(US"callout", O_RDWR, &dbblock, FALSE, TRUE))) +else if (!(dbm_file = dbfn_open(US"callout", O_RDWR|O_CREAT, &dbblock, FALSE, TRUE))) { HDEBUG(D_verify) debug_printf_indent("callout cache: not available\n"); } @@ -294,7 +294,7 @@ implying some kind of I/O error. We don't want to write the cache in that case. Otherwise the value is ccache_accept, ccache_reject, or ccache_reject_mfnull. */ if (dom_rec->result != ccache_unknown) - if (!(dbm_file = dbfn_open(US"callout", O_RDWR, &dbblock, FALSE, TRUE))) + if (!(dbm_file = dbfn_open(US"callout", O_RDWR|O_CREAT, &dbblock, FALSE, TRUE))) { HDEBUG(D_verify) debug_printf_indent("callout cache: not available\n"); } @@ -316,7 +316,7 @@ is disabled. */ if (done && addr_rec->result != ccache_unknown) { if (!dbm_file) - dbm_file = dbfn_open(US"callout", O_RDWR, &dbblock, FALSE, TRUE); + dbm_file = dbfn_open(US"callout", O_RDWR|O_CREAT, &dbblock, FALSE, TRUE); if (!dbm_file) { HDEBUG(D_verify) debug_printf_indent("no callout cache available\n"); @@ -3497,7 +3497,7 @@ dbdata_callout_cache_address * cache_address_record; if (!pos_cache && !neg_cache) return FALSE; -if (!(dbm_file = dbfn_open(US"callout", O_RDWR, &dbblock, FALSE, TRUE))) +if (!(dbm_file = dbfn_open(US"callout", O_RDWR|O_CREAT, &dbblock, FALSE, TRUE))) { HDEBUG(D_verify) debug_printf_indent("quota cache: not available\n"); return FALSE; @@ -3525,7 +3525,7 @@ dbdata_callout_cache_address cache_address_record; if (!pos_cache && !neg_cache) return; -if (!(dbm_file = dbfn_open(US"callout", O_RDWR, &dbblock, FALSE, TRUE))) +if (!(dbm_file = dbfn_open(US"callout", O_RDWR|O_CREAT, &dbblock, FALSE, TRUE))) { HDEBUG(D_verify) debug_printf_indent("quota cache: not available\n"); return; commit 3cee6033bae86d254e51b583d34f6b559a6d95ea Author: Jeremy Harris Date: Thu Jul 18 14:50:14 2024 +0100 HintsDB: Support transactions with a TDB backend. diff --git a/src/src/dbfn.c b/src/src/dbfn.c index c2c92cf44..1f6989164 100644 --- a/src/src/dbfn.c +++ b/src/src/dbfn.c @@ -38,8 +38,14 @@ are separate open and close functions. However, the calling modules should arrange to hold the locks for the bare minimum of time. API: - dbfn_open - dbfn_close + exim_lockfile_needed facilities predicate + dbfn_open takes lockfile or opens transaction + dbfn_open_multi only if transactions supported; + no lock or transaction taken + dbfn_close release lockfile or transaction + dbfn_close_multi + dbfn_transaction_start only if transactions supported + dbfn_transaction_commit dbfn_read_with_length dbfn_read_enforce_length dbfn_write @@ -166,6 +172,7 @@ exists, there is no error. */ dlen = snprintf(CS dirname, sizeof(dirname), "%s/db", spool_directory); +dbblock->readonly = (flags & O_ACCMODE) == O_RDONLY; dbblock->lockfd = -1; if (!exim_lockfile_needed()) db_dir_make(panic); @@ -194,7 +201,10 @@ open call. */ snprintf(CS filename, sizeof(filename), "%.*s/%s", dlen, dirname, name); priv_drop_temp(exim_uid, exim_gid); -dbblock->dbptr = exim_dbopen(filename, dirname, flags & O_ACCMODE, EXIMDB_MODE); +dbblock->dbptr = dbblock->readonly && !exim_lockfile_needed() + ? exim_dbopen_multi(filename, dirname, flags & O_ACCMODE, EXIMDB_MODE) + : exim_dbopen(filename, dirname, flags & O_ACCMODE, EXIMDB_MODE); + if (!dbblock->dbptr && errno == ENOENT && flags & O_CREAT) { DEBUG(D_hints_lookup) @@ -220,21 +230,13 @@ if (!dbblock->dbptr) filename)); (void)close(dbblock->lockfd); dbblock->lockfd = -1; - errno = save_errno; - DEBUG(D_hints_lookup) acl_level--; - return NULL; + dbblock = NULL; } -DEBUG(D_hints_lookup) - debug_printf_indent("opened hints database %s: flags=%s%s\n", filename, - (flags & O_ACCMODE) == O_RDONLY ? "O_RDONLY" - : (flags & O_ACCMODE) == O_RDWR ? "O_RDWR" - : "??", - flags & O_CREAT ? "|O_CREAT" : ""); - /* Pass back the block containing the opened database handle and the open fd for the lock. */ +DEBUG(D_hints_lookup) acl_level--; return dbblock; } @@ -284,21 +286,11 @@ if (!dbblock->dbptr) DEBUG(D_hints_lookup) debug_printf_indent("%s\n", CS string_open_failed("DB file %s", filename)); - errno = save_errno; - DEBUG(D_hints_lookup) acl_level--; - return NULL; + dbblock = NULL; } -DEBUG(D_hints_lookup) - debug_printf_indent("opened hints database %s for transactions: NOLOCK flags=%s%s\n", - filename, - (flags & O_ACCMODE) == O_RDONLY ? "O_RDONLY" - : (flags & O_ACCMODE) == O_RDWR ? "O_RDWR" - : "??", - flags & O_CREAT ? "|O_CREAT" : ""); - /* Pass back the block containing the opened database handle */ - +DEBUG(D_hints_lookup) acl_level--; return dbblock; } @@ -307,13 +299,13 @@ BOOL dbfn_transaction_start(open_db * dbp) { DEBUG(D_hints_lookup) debug_printf_indent("dbfn_transaction_start\n"); -return exim_dbtransaction_start(dbp->dbptr); +if (!dbp->readonly) return exim_dbtransaction_start(dbp->dbptr); } void dbfn_transaction_commit(open_db * dbp) { DEBUG(D_hints_lookup) debug_printf_indent("dbfn_transaction_commit\n"); -exim_dbtransaction_commit(dbp->dbptr); +if (!dbp->readonly) exim_dbtransaction_commit(dbp->dbptr); } @@ -334,14 +326,15 @@ dbfn_close(open_db * dbp) { int * fdp = &dbp->lockfd; -exim_dbclose(dbp->dbptr); +if (dbp->readonly && !exim_lockfile_needed()) + exim_dbclose_multi(dbp->dbptr); +else + exim_dbclose(dbp->dbptr); + if (*fdp >= 0) (void)close(*fdp); DEBUG(D_hints_lookup) - { debug_printf_indent("closed hints database%s\n", *fdp < 0 ? "" : " and lockfile"); - acl_level--; - } *fdp = -1; } @@ -351,10 +344,7 @@ dbfn_close_multi(open_db * dbp) { exim_dbclose_multi(dbp->dbptr); DEBUG(D_hints_lookup) - { debug_printf_indent("closed hints database\n"); - acl_level--; - } } diff --git a/src/src/dbfunctions.h b/src/src/dbfunctions.h index 0aa3b777c..08f8c2311 100644 --- a/src/src/dbfunctions.h +++ b/src/src/dbfunctions.h @@ -28,16 +28,5 @@ void dbfn_transaction_commit(open_db *); #define dbfn_read(a, b) dbfn_read_with_length(a, b, NULL) -/* Berkeley DB uses a callback function to pass back error details. Its API -changed at release 4.3. */ - -#if defined(USE_DB) && defined(DB_VERSION_STRING) -# if DB_VERSION_MAJOR > 4 || (DB_VERSION_MAJOR == 4 && DB_VERSION_MINOR >= 3) -void dbfn_bdb_error_callback(const DB_ENV *, const char *, const char *); -# else -void dbfn_bdb_error_callback(const char *, char *); -# endif -#endif - #endif /* End of dbfunctions.h */ diff --git a/src/src/deliver.c b/src/src/deliver.c index b04fe6f12..ee7650243 100644 --- a/src/src/deliver.c +++ b/src/src/deliver.c @@ -2896,7 +2896,12 @@ while (addr_local) of these checks, rather than for all local deliveries, because some local deliveries (e.g. to pipes) can take a substantial time. */ - if (!(dbm_file = dbfn_open(US"retry", O_RDONLY, &dbblock, FALSE, TRUE))) + if (continue_retry_db && continue_retry_db != (open_db *)-1) + { + DEBUG(D_hints_lookup) debug_printf("using cached retry hintsdb handle\n"); + dbm_file = continue_retry_db; + } + else if (!(dbm_file = dbfn_open(US"retry", O_RDONLY, &dbblock, FALSE, TRUE))) DEBUG(D_deliver|D_retry|D_hints_lookup) debug_printf("no retry data available\n"); @@ -2905,61 +2910,66 @@ while (addr_local) while (addr2) { BOOL ok = TRUE; /* to deliver this address */ - uschar *retry_key; - - /* Set up the retry key to include the domain or not, and change its - leading character from "R" to "T". Must make a copy before doing this, - because the old key may be pointed to from a "delete" retry item after - a routing delay. */ - retry_key = string_copy( - tp->retry_use_local_part ? addr2->address_retry_key : - addr2->domain_retry_key); - *retry_key = 'T'; + if (f.queue_2stage) + { + DEBUG(D_deliver) + debug_printf_indent("no router retry check (ph1 qrun)\n"); + } + else + { + /* Set up the retry key to include the domain or not, and change its + leading character from "R" to "T". Must make a copy before doing this, + because the old key may be pointed to from a "delete" retry item after + a routing delay. */ + uschar * retry_key = string_copy(tp->retry_use_local_part + ? addr2->address_retry_key : addr2->domain_retry_key); + *retry_key = 'T'; - /* Inspect the retry data. If there is no hints file, delivery happens. */ + /* Inspect the retry data. If there is no hints file, delivery happens. */ - if (dbm_file) - { - dbdata_retry * retry_record = dbfn_read(dbm_file, retry_key); + if (dbm_file) + { + dbdata_retry * retry_record = dbfn_read(dbm_file, retry_key); - /* If there is no retry record, delivery happens. If there is, - remember it exists so it can be deleted after a successful delivery. */ + /* If there is no retry record, delivery happens. If there is, + remember it exists so it can be deleted after a successful delivery. */ - if (retry_record) - { - setflag(addr2, af_lt_retry_exists); + if (retry_record) + { + setflag(addr2, af_lt_retry_exists); - /* A retry record exists for this address. If queue running and not - forcing, inspect its contents. If the record is too old, or if its - retry time has come, or if it has passed its cutoff time, delivery - will go ahead. */ + /* A retry record exists for this address. If queue running and not + forcing, inspect its contents. If the record is too old, or if its + retry time has come, or if it has passed its cutoff time, delivery + will go ahead. */ - DEBUG(D_retry) - { - debug_printf("retry record exists: age=%s ", - readconf_printtime(now - retry_record->time_stamp)); - debug_printf("(max %s)\n", readconf_printtime(retry_data_expire)); - debug_printf(" time to retry = %s expired = %d\n", - readconf_printtime(retry_record->next_try - now), - retry_record->expired); - } + DEBUG(D_retry) + { + debug_printf("retry record exists: age=%s ", + readconf_printtime(now - retry_record->time_stamp)); + debug_printf("(max %s)\n", readconf_printtime(retry_data_expire)); + debug_printf(" time to retry = %s expired = %d\n", + readconf_printtime(retry_record->next_try - now), + retry_record->expired); + } - if (f.queue_running && !f.deliver_force) - { - ok = (now - retry_record->time_stamp > retry_data_expire) - || (now >= retry_record->next_try) - || retry_record->expired; + if (f.queue_running && !f.deliver_force) + { + ok = (now - retry_record->time_stamp > retry_data_expire) + || (now >= retry_record->next_try) + || retry_record->expired; - /* If we haven't reached the retry time, there is one more check - to do, which is for the ultimate address timeout. */ + /* If we haven't reached the retry time, there is one more check + to do, which is for the ultimate address timeout. */ - if (!ok) - ok = retry_ultimate_address_timeout(retry_key, addr2->domain, - retry_record, now); - } - } - else DEBUG(D_retry) debug_printf("no retry record exists\n"); + if (!ok) + ok = retry_ultimate_address_timeout(retry_key, addr2->domain, + retry_record, now); + } + } + else DEBUG(D_retry) debug_printf("no retry record exists\n"); + } } /* This address is to be delivered. Leave it on the chain. */ @@ -2985,7 +2995,11 @@ while (addr_local) } } - if (dbm_file) dbfn_close(dbm_file); + if (dbm_file) + if (dbm_file != continue_retry_db) + { dbfn_close(dbm_file); dbm_file = NULL; } + else + DEBUG(D_hints_lookup) debug_printf("retaining retry hintsdb handle\n"); /* If there are no addresses left on the chain, they all deferred. Loop for the next set of addresses. */ @@ -7440,8 +7454,13 @@ while (addr_new) /* Loop until all addresses dealt with */ { /* If we have transaction-capable hintsdbs, open the retry db without locking, and leave open for the transport process and for subsequent - deliveries. If the open fails, tag that explicitly for the transport but - retry the open next time around, in case it was created in the interim. */ + deliveries. Use a writeable open as we can keep it open all the way through + to writing retry records if needed due to message fails. + If the open fails, tag that explicitly for the transport but retry the open + next time around, in case it was created in the interim. + If non-transaction, we are only reading records at this stage and + we close the db before running the transport. + Either way we do a non-creating open. */ if (continue_retry_db == (open_db *)-1) continue_retry_db = NULL; @@ -7451,7 +7470,7 @@ while (addr_new) /* Loop until all addresses dealt with */ DEBUG(D_hints_lookup) debug_printf("using cached retry hintsdb handle\n"); dbm_file = continue_retry_db; } - else if (!exim_lockfile_needed() && continue_transport) + else if (!exim_lockfile_needed()) { dbm_file = dbfn_open_multi(US"retry", O_RDWR, &dbblock); continue_retry_db = dbm_file ? dbm_file : (open_db *)-1; @@ -7471,7 +7490,7 @@ while (addr_new) /* Loop until all addresses dealt with */ { int rc; tree_node * tnode; - dbdata_retry * domain_retry_record, * address_retry_record; + dbdata_retry * domain_retry_record = NULL, * address_retry_record = NULL; addr = addr_new; addr_new = addr->next; @@ -7700,76 +7719,82 @@ while (addr_new) /* Loop until all addresses dealt with */ continue; } - /* Get the routing retry status, saving the two retry keys (with and - without the local part) for subsequent use. If there is no retry record for - the standard address routing retry key, we look for the same key with the - sender attached, because this form is used by the smtp transport after a - 4xx response to RCPT when address_retry_include_sender is true. */ - - DEBUG(D_retry) + if (f.queue_2stage) { - debug_printf_indent("checking router retry status\n"); - acl_level++; + DEBUG(D_deliver) + debug_printf_indent("no router retry check (ph1 qrun)\n"); } - addr->domain_retry_key = string_sprintf("R:%s", addr->domain); - addr->address_retry_key = string_sprintf("R:%s@%s", addr->local_part, - addr->domain); - - if (dbm_file) + else { - domain_retry_record = dbfn_read(dbm_file, addr->domain_retry_key); - if ( domain_retry_record - && now - domain_retry_record->time_stamp > retry_data_expire - ) + /* Get the routing retry status, saving the two retry keys (with and + without the local part) for subsequent use. If there is no retry record + for the standard address routing retry key, we look for the same key with + the sender attached, because this form is used by the smtp transport after + a 4xx response to RCPT when address_retry_include_sender is true. */ + + DEBUG(D_deliver|D_retry) { - DEBUG(D_deliver|D_retry) - debug_printf_indent("domain retry record present but expired\n"); - domain_retry_record = NULL; /* Ignore if too old */ + debug_printf_indent("checking router retry status\n"); + acl_level++; } + addr->domain_retry_key = string_sprintf("R:%s", addr->domain); + addr->address_retry_key = string_sprintf("R:%s@%s", addr->local_part, + addr->domain); - address_retry_record = dbfn_read(dbm_file, addr->address_retry_key); - if ( address_retry_record - && now - address_retry_record->time_stamp > retry_data_expire - ) + if (dbm_file) { - DEBUG(D_deliver|D_retry) - debug_printf_indent("address retry record present but expired\n"); - address_retry_record = NULL; /* Ignore if too old */ - } + domain_retry_record = dbfn_read(dbm_file, addr->domain_retry_key); + if ( domain_retry_record + && now - domain_retry_record->time_stamp > retry_data_expire + ) + { + DEBUG(D_deliver|D_retry) + debug_printf_indent("domain retry record present but expired\n"); + domain_retry_record = NULL; /* Ignore if too old */ + } - if (!address_retry_record) - { - uschar *altkey = string_sprintf("%s:<%s>", addr->address_retry_key, - sender_address); - address_retry_record = dbfn_read(dbm_file, altkey); - if ( address_retry_record - && now - address_retry_record->time_stamp > retry_data_expire) + address_retry_record = dbfn_read(dbm_file, addr->address_retry_key); + if ( address_retry_record + && now - address_retry_record->time_stamp > retry_data_expire + ) { DEBUG(D_deliver|D_retry) - debug_printf_indent("address retry record present but expired\n"); - address_retry_record = NULL; /* Ignore if too old */ + debug_printf_indent("address retry record present but expired\n"); + address_retry_record = NULL; /* Ignore if too old */ } - } - } - else - domain_retry_record = address_retry_record = NULL; - DEBUG(D_deliver|D_retry) - { - if (!domain_retry_record) - debug_printf_indent("no domain retry record\n"); - else - debug_printf_indent("have domain retry record; next_try = now%+d\n", - f.running_in_test_harness ? 0 : - (int)(domain_retry_record->next_try - now)); + if (!address_retry_record) + { + uschar *altkey = string_sprintf("%s:<%s>", addr->address_retry_key, + sender_address); + address_retry_record = dbfn_read(dbm_file, altkey); + if ( address_retry_record + && now - address_retry_record->time_stamp > retry_data_expire) + { + DEBUG(D_deliver|D_retry) + debug_printf_indent("address retry record present but expired\n"); + address_retry_record = NULL; /* Ignore if too old */ + } + } + } - if (!address_retry_record) - debug_printf_indent("no address retry record\n"); - else - debug_printf_indent("have address retry record; next_try = now%+d\n", - f.running_in_test_harness ? 0 : - (int)(address_retry_record->next_try - now)); - acl_level--; + DEBUG(D_deliver|D_retry) + { + if (!domain_retry_record) + debug_printf_indent("no domain retry record\n"); + else + debug_printf_indent("have domain retry record; next_try = now%+d\n", + f.running_in_test_harness ? 0 : + (int)(domain_retry_record->next_try - now)); + + if (!address_retry_record) + debug_printf_indent("no address retry record\n"); + else + debug_printf_indent("have address retry record; next_try = now%+d\n", + f.running_in_test_harness ? 0 : + (int)(address_retry_record->next_try - now)); + acl_level--; + } } /* If we are sending a message down an existing SMTP connection, we must @@ -7873,11 +7898,15 @@ while (addr_new) /* Loop until all addresses dealt with */ } } - /* The database is closed while routing is actually happening. Requests to - update it are put on a chain and all processed together at the end. */ + /* If not transaction-capable, the database is closed while routing is + actually happening. Requests to update it are put on a chain and all processed + together at the end. */ - if (dbm_file && !continue_retry_db) - { dbfn_close(dbm_file); dbm_file = NULL; } + if (dbm_file) + if (exim_lockfile_needed()) + { dbfn_close(dbm_file); dbm_file = NULL; } + else + DEBUG(D_hints_lookup) debug_printf("retaining retry hintsdb handle\n"); /* If queue_domains is set, we don't even want to try routing addresses in those domains. During queue runs, queue_domains is forced to be unset. @@ -8048,9 +8077,6 @@ while (addr_new) /* Loop until all addresses dealt with */ } /* Loop to process any child addresses that the routers created, and any rerouted addresses that got put back on the new chain. */ -if (dbm_file) /* Can only be continue_retry_db */ - { dbfn_close_multi(continue_retry_db); continue_retry_db = dbm_file = NULL; } - /* Debugging: show the results of the routing */ DEBUG(D_deliver|D_retry|D_route) @@ -8149,12 +8175,7 @@ if ( mua_wrapper /* If this is a run to continue deliveries to an external channel that is -already set up, defer any local deliveries. - -jgh 2020/12/20: I don't see why; locals should be quick. -The defer goes back to version 1.62 in 1997. A local being still deliverable -during a continued run might result from something like a defer during the -original delivery, eg. in a DB lookup. Unlikely but possible. +already set up, defer any local deliveries because we are handling remotes. To avoid delaying a local when combined with a callout-hold for a remote delivery, test continue_sequence rather than continue_transport. */ @@ -8402,9 +8423,10 @@ if (mua_wrapper) /* In a normal configuration, we now update the retry database. This is done in one fell swoop at the end in order not to keep opening and closing (and -locking) the database. The code for handling retries is hived off into a -separate module for convenience. We pass it the addresses of the various -chains, because deferred addresses can get moved onto the failed chain if the +locking) the database (at least, for non-transaction-capable DBs. +The code for handling retries is hived off into a separate module for +convenience. We pass it the addresses of the various chains, +because deferred addresses can get moved onto the failed chain if the retry cutoff time has expired for all alternative destinations. Bypass the updating of the database if the -N flag is set, which is a debugging thing that prevents actual delivery. */ @@ -8506,6 +8528,13 @@ f.disable_logging = FALSE; /* In case left set */ DELIVERY_TIDYUP: +if (dbm_file) /* Can only be continue_retry_db */ + { + DEBUG(D_hints_lookup) debug_printf("final close of cached retry db\n"); + dbfn_close_multi(continue_retry_db); + continue_retry_db = dbm_file = NULL; + } + /* If there are now no deferred addresses, we are done. Preserve the message log if so configured, and we are using them. Otherwise, sling it. Then delete the message itself. */ diff --git a/src/src/exim_dbmbuild.c b/src/src/exim_dbmbuild.c index 05387aa3f..f8fbd2b8c 100644 --- a/src/src/exim_dbmbuild.c +++ b/src/src/exim_dbmbuild.c @@ -88,6 +88,27 @@ if it is made static. */ const uschar *hex_digits = CUS"0123456789abcdef"; +/******************* +* Debug output * +*******************/ + +unsigned int debug_selector = 0; /* set -1 for debugging */ + +void +debug_printf(const char * fmt, ...) +{ +va_list ap; +va_start(ap, fmt); vfprintf(stderr, fmt, ap); va_end(ap); +} +void +debug_printf_indent(const char * fmt, ...) +{ +va_list ap; +va_start(ap, fmt); vfprintf(stderr, fmt, ap); va_end(ap); +} + + + #ifdef STRERROR_FROM_ERRLIST /* Some old-fashioned systems still around (e.g. SunOS4) don't have strerror() in their libraries, but can provide the same facility by this simple diff --git a/src/src/exim_dbutil.c b/src/src/exim_dbutil.c index dd1444593..f3123c80b 100644 --- a/src/src/exim_dbutil.c +++ b/src/src/exim_dbutil.c @@ -112,6 +112,25 @@ exit(EXIT_FAILURE); +/******************* +* Debug output * +*******************/ + +unsigned int debug_selector = 0; /* set -1 for debugging */ + +void +debug_printf(const char * fmt, ...) +{ +va_list ap; +va_start(ap, fmt); vfprintf(stderr, fmt, ap); va_end(ap); +} +void +debug_printf_indent(const char * fmt, ...) +{ +va_list ap; +va_start(ap, fmt); vfprintf(stderr, fmt, ap); va_end(ap); +} + /************************************************* * Sort out the command arguments * *************************************************/ @@ -287,7 +306,6 @@ dbfn_open(const uschar * name, int flags, open_db * dbblock, { int rc; struct flock lock_data; -BOOL read_only = (flags & (O_WRONLY|O_RDWR)) == O_RDONLY; uschar * dirname, * filename; /* The first thing to do is to open a separate file on which to lock. This @@ -301,6 +319,7 @@ if ( asprintf(CSS &dirname, "%s/db", spool_directory) < 0 || asprintf(CSS &filename, "%s/%s.lockfile", dirname, name) < 0) return NULL; +dbblock->readonly = (flags & (O_WRONLY|O_RDWR)) == O_RDONLY; dbblock->lockfd = -1; if (exim_lockfile_needed()) { @@ -314,7 +333,7 @@ if (exim_lockfile_needed()) /* Now we must get a lock on the opened lock file; do this with a blocking lock that times out. */ - lock_data.l_type = read_only ? F_RDLCK : F_WRLCK; + lock_data.l_type = dbblock->readonly ? F_RDLCK : F_WRLCK; lock_data.l_whence = lock_data.l_start = lock_data.l_len = 0; sigalrm_seen = FALSE; @@ -327,7 +346,7 @@ if (exim_lockfile_needed()) if (rc < 0) { printf("** Failed to get %s lock for %s: %s", - read_only ? "read" : "write", + dbblock->readonly ? "read" : "write", filename, errno == ETIMEDOUT ? "timed out" : strerror(errno)); (void)close(dbblock->lockfd); @@ -340,10 +359,12 @@ if (exim_lockfile_needed()) if (asprintf(CSS &filename, "%s/%s", dirname, name) < 0) return NULL; -if (!(dbblock->dbptr = exim_dbopen(filename, dirname, flags, 0))) +if (!(dbblock->dbptr = dbblock->readonly && !exim_lockfile_needed() + ? exim_dbopen_multi(filename, dirname, flags, 0) + : exim_dbopen(filename, dirname, flags, 0))) { printf("** Failed to open hintsdb file %s for %s: %s%s\n", filename, - read_only ? "reading" : "writing", strerror(errno), + dbblock->readonly ? "reading" : "writing", strerror(errno), #ifdef USE_DB " (or Berkeley DB error while opening)" #else @@ -374,8 +395,13 @@ Returns: nothing void dbfn_close(open_db * dbp) { -exim_dbclose(dbp->dbptr); -if (dbp->lockfd >= 0) (void) close(dbp->lockfd); +if (dbp->readonly && !exim_lockfile_needed()) + exim_dbclose_multi(dbp->dbptr); +else + exim_dbclose(dbp->dbptr); + +if (dbp->lockfd >= 0) + (void) close(dbp->lockfd); } @@ -422,6 +448,7 @@ we should store the taint status along with the data. */ dlen = exim_datum_size_get(&result_datum); yield = store_get(dlen, GET_TAINTED); memcpy(yield, exim_datum_data_get(&result_datum), dlen); +DEBUG(D_hints_lookup) debug_printf_indent("dbfn_read: size %u return\n", dlen); if (length) *length = dlen; exim_datum_free(&result_datum); /* Some DBM libs require freeing */ diff --git a/src/src/hintsdb.h b/src/src/hintsdb.h index ba50ae1b9..a35791409 100644 --- a/src/src/hintsdb.h +++ b/src/src/hintsdb.h @@ -25,7 +25,11 @@ The API is: Functions: exim_lockfile_needed API semantics predicate exim_dbopen + exim_dbopen_multi only for no-lockfile-needed exim_dbclose + exim_dbclose_multi only for no-lockfile-needed + exim_dbtransaction_start only for no-lockfile-needed + exim_dbtransaction_commit only for no-lockfile-needed exim_dbget exim_dbput exim_dbputb non-overwriting put @@ -66,6 +70,9 @@ required by Exim's process transitions)? #ifndef HINTSDB_H #define HINTSDB_H +/* Include file ordering problem */ +extern void debug_printf_indent(const char *, ...) PRINTF_FUNCTION(1,2); + #ifdef USE_SQLITE # if defined(USE_DB) || defined(USE_GDBM) || defined(USE_TDB) @@ -103,20 +110,6 @@ the default is the NDBM interface (which seems to be a wrapper for GDBM) */ -#if defined(COMPILE_UTILITY) || defined(MACRO_PREDEF) - -static inline EXIM_DB * -exim_dbopen(const uschar * name, const uschar * dirname, int flags, - unsigned mode) -{ -return exim_dbopen__(name, dirname, flags, mode); -} - -static inline void -exim_dbclose(EXIM_DB * dbp) -{ exim_dbclose__(dbp); } - -#else /* exim mainline code */ /* Wrappers for open/close with debug tracing */ @@ -147,14 +140,43 @@ DEBUG(D_hints_lookup) debug_printf_indent("returned from EXIM_DBOPEN: %p\n", dbp return dbp; } +static inline EXIM_DB * +exim_dbopen_multi(const uschar * name, const uschar * dirname, int flags, + unsigned mode) +{ +void * dbp; +DEBUG(D_hints_lookup) + debug_printf_indent("EXIM_DBOPEN_MULTI: file <%s> dir <%s> flags=%s\n", + name, dirname, + flags == O_RDONLY ? "O_RDONLY" + : flags == O_RDWR ? "O_RDWR" + : flags == (O_RDWR|O_CREAT) ? "O_RDWR|O_CREAT" + : "??"); +if (is_tainted(name) || is_tainted(dirname)) + { + log_write(0, LOG_MAIN|LOG_PANIC, "Tainted name for DB file not permitted"); + dbp = NULL; + } +else + dbp = exim_dbopen_multi__(name, dirname, flags, mode); + +DEBUG(D_hints_lookup) debug_printf_indent("returned from EXIM_DBOPEN_MULTI: %p\n", dbp); +return dbp; +} + static inline void exim_dbclose(EXIM_DB * dbp) { DEBUG(D_hints_lookup) debug_printf_indent("EXIM_DBCLOSE(%p)\n", dbp); exim_dbclose__(dbp); } +static inline void +exim_dbclose_multi(EXIM_DB * dbp) +{ +DEBUG(D_hints_lookup) debug_printf_indent("EXIM_DBCLOSE_MULTI(%p)\n", dbp); +exim_dbclose_multi__(dbp); +} -#endif /* defined(COMPILE_UTILITY) || defined(MACRO_PREDEF) */ /********************* End of dbm library definitions **********************/ diff --git a/src/src/hintsdb/hints_bdb.h b/src/src/hintsdb/hints_bdb.h index e629cce4e..48573e907 100644 --- a/src/src/hintsdb/hints_bdb.h +++ b/src/src/hintsdb/hints_bdb.h @@ -48,6 +48,21 @@ definition of DB_VERSION_STRING, which is present in versions 2.x onwards. */ # define DB_FORCESYNC 0 # endif + + +/* Berkeley DB uses a callback function to pass back error details. Its API +changed at release 4.3. */ + +#if defined(DB_VERSION_STRING) +# if DB_VERSION_MAJOR > 4 || (DB_VERSION_MAJOR == 4 && DB_VERSION_MINOR >= 3) +static void dbfn_bdb_error_callback(const DB_ENV *, const char *, const char *); +# else +static void dbfn_bdb_error_callback(const char *, char *); +# endif +#endif + + + /* Error callback */ /* For Berkeley DB >= 2, we can define a function to be called in case of DB errors. This should help with debugging strange DB problems, e.g. getting "File @@ -73,9 +88,9 @@ return TRUE; } static inline EXIM_DB * -exim_dbopen_multi(const uschar * name, const uschar * dirname, int flags, +exim_dbopen_multi__(const uschar * name, const uschar * dirname, int flags, unsigned mode) { return NULL; } -static inline void exim_dbclose_multi(EXIM_DB * dbp) {} +static inline void exim_dbclose_multi__(EXIM_DB * dbp) {} static inline BOOL exim_dbtransaction_start(EXIM_DB * dbp) { return FALSE; } static inline void exim_dbtransaction_commit(EXIM_DB * dbp) {} @@ -106,6 +121,9 @@ if (db_create(&b, dbp, 0) == 0) mode) == 0 ) return dbp; + else DEBUG(D_hints_lookup) + debug_printf_indent("bdb_open(flags 0x%x mode %04o) %s\n", + flags, mode, strerror(errno)); b->close(b, 0); } @@ -233,9 +251,9 @@ return TRUE; } static inline EXIM_DB * -exim_dbopen_multi(const uschar * name, const uschar * dirname, int flags, +exim_dbopen_multi__(const uschar * name, const uschar * dirname, int flags, unsigned mode) { return NULL; } -static inline void exim_dbclose_multi(EXIM_DB * dbp) {} +static inline void exim_dbclose_multi__(EXIM_DB * dbp) {} static inline BOOL exim_dbtransaction_start(EXIM_DB * dbp) { return FALSE; } static inline void exim_dbtransaction_commit(EXIM_DB * dbp) {} diff --git a/src/src/hintsdb/hints_gdbm.h b/src/src/hintsdb/hints_gdbm.h index b406d45e0..6b3789979 100644 --- a/src/src/hintsdb/hints_gdbm.h +++ b/src/src/hintsdb/hints_gdbm.h @@ -41,9 +41,9 @@ return TRUE; } static inline EXIM_DB * -exim_dbopen_multi(const uschar * name, const uschar * dirname, int flags, +exim_dbopen_multi__(const uschar * name, const uschar * dirname, int flags, unsigned mode) { return NULL; } -static inline void exim_dbclose_multi(EXIM_DB * dbp) {} +static inline void exim_dbclose_multi__(EXIM_DB * dbp) {} static inline BOOL exim_dbtransaction_start(EXIM_DB * dbp) { return FALSE; } static inline void exim_dbtransaction_commit(EXIM_DB * dbp) {} @@ -60,7 +60,12 @@ if (dbp) flags & O_CREAT ? GDBM_WRCREAT : flags & (O_RDWR|O_WRONLY) ? GDBM_WRITER : GDBM_READER, mode, 0); - if (dbp->gdbm) return dbp; + if (dbp->gdbm) + return dbp; + + DEBUG(D_hints_lookup) + debug_printf_indent("gdbm_open(flags 0x%x mode %04o) %s\n", + flags, mode, strerror(errno)); free(dbp); } return NULL; diff --git a/src/src/hintsdb/hints_ndbm.h b/src/src/hintsdb/hints_ndbm.h index fb1db57a8..389abd9e2 100644 --- a/src/src/hintsdb/hints_ndbm.h +++ b/src/src/hintsdb/hints_ndbm.h @@ -36,9 +36,9 @@ return TRUE; } static inline EXIM_DB * -exim_dbopen_multi(const uschar * name, const uschar * dirname, int flags, +exim_dbopen_multi__(const uschar * name, const uschar * dirname, int flags, unsigned mode) { return NULL; } -static inline void exim_dbclose_multi(EXIM_DB * dbp) {} +static inline void exim_dbclose_multi__(EXIM_DB * dbp) {} static inline BOOL exim_dbtransaction_start(EXIM_DB * dbp) { return FALSE; } static inline void exim_dbtransaction_commit(EXIM_DB * dbp) {} @@ -54,9 +54,10 @@ exim_dbopen__(const uschar * name, const uschar * dirname, int flags, struct stat st; if (!(flags & O_CREAT) || lstat(CCS name, &st) != 0 && errno == ENOENT) return dbm_open(CS name, flags, mode); -#ifndef COMPILE_UTILITY -debug_printf("%s %d errno %s\n", __FUNCTION__, __LINE__, strerror(errno)); -#endif + +DEBUG(D_hints_lookup) + debug_printf_indent("ndbm_open(flags 0x%x mode %04o) %s\n", + flags, mode, strerror(errno)); errno = (st.st_mode & S_IFMT) == S_IFDIR ? EISDIR : EEXIST; return NULL; } diff --git a/src/src/hintsdb/hints_sqlite.h b/src/src/hintsdb/hints_sqlite.h index da3bc2bff..262a414f6 100644 --- a/src/src/hintsdb/hints_sqlite.h +++ b/src/src/hintsdb/hints_sqlite.h @@ -35,11 +35,12 @@ return FALSE; /* We do transaction; no extra locking needed */ /* EXIM_DBOPEN - return pointer to an EXIM_DB, NULL if failed */ static inline EXIM_DB * -exim_dbopen_multi(const uschar * name, const uschar * dirname, int flags, +exim_dbopen_multi__(const uschar * name, const uschar * dirname, int flags, unsigned mode) { EXIM_DB * dbp; int ret, sflags = flags & O_RDWR ? SQLITE_OPEN_READWRITE : SQLITE_OPEN_READONLY; + if (flags & O_CREAT) sflags |= SQLITE_OPEN_CREATE; if ((ret = sqlite3_open_v2(CCS name, &dbp, sflags, NULL)) == SQLITE_OK) { @@ -51,8 +52,9 @@ if ((ret = sqlite3_open_v2(CCS name, &dbp, sflags, NULL)) == SQLITE_OK) if (ret != SQLITE_OK) sqlite3_close(dbp); } -//else -// fprintf(stderr, "sqlite3_open_v2: %s\n", sqlite3_errmsg(dbp)); +else DEBUG(D_hints_lookup) + debug_printf_indent("sqlite_open(flags 0x%x mode %04o) %s\n", + flags, mode, sqlite3_errmsg(dbp)); return ret == SQLITE_OK ? dbp : NULL; } @@ -286,7 +288,7 @@ store_free(cursor); /* EXIM_DBCLOSE */ static inline void -exim_dbclose_multi(EXIM_DB * dbp) +exim_dbclose_multi__(EXIM_DB * dbp) { sqlite3_close(dbp); } diff --git a/src/src/hintsdb/hints_tdb.h b/src/src/hintsdb/hints_tdb.h index 436597115..2b4fcc7f6 100644 --- a/src/src/hintsdb/hints_tdb.h +++ b/src/src/hintsdb/hints_tdb.h @@ -11,9 +11,6 @@ backend provider. */ /* ************************* tdb interface ************************ */ -/*XXX https://manpages.org/tdb/3 mentions concurrent writes. -Could we lose the file lock? */ - # include /* Basic DB type */ @@ -34,22 +31,39 @@ tdb_traverse to be called) */ static inline BOOL exim_lockfile_needed(void) { -return TRUE; +return FALSE; /* Transactions are supported */ } -static inline EXIM_DB * -exim_dbopen_multi(const uschar * name, const uschar * dirname, int flags, - unsigned mode) { return NULL; } -static inline void exim_dbclose_multi(EXIM_DB * dbp) {} -static inline BOOL exim_dbtransaction_start(EXIM_DB * dbp) { return FALSE; } -static inline void exim_dbtransaction_commit(EXIM_DB * dbp) {} /* EXIM_DBOPEN - return pointer to an EXIM_DB, NULL if failed */ static inline EXIM_DB * exim_dbopen__(const uschar * name, const uschar * dirname, int flags, unsigned mode) { -return tdb_open(CS name, 0, TDB_DEFAULT, flags, mode); +EXIM_DB * db = tdb_open(CS name, 0, TDB_DEFAULT, flags, mode); +int e; + +DEBUG(D_hints_lookup) if (!db) + debug_printf_indent("tdb_open(flags 0x%x mode %04o) %s\n", + flags, mode, strerror(errno)); +if (!db || tdb_transaction_start(db) == 0) return db; +e = errno; +DEBUG(D_hints_lookup) if (db) + debug_printf_indent("tdb_transaction_start: %s\n", tdb_errorstr(db)); +tdb_close(db); +errno = e; +return NULL; +} + +static inline EXIM_DB * +exim_dbopen_multi__(const uschar * name, const uschar * dirname, int flags, + unsigned mode) +{ +EXIM_DB * db = tdb_open(CS name, 0, TDB_DEFAULT, flags, mode); +DEBUG(D_hints_lookup) if (!db) + debug_printf_indent("tdb_open(flags 0x%x mode %04o) %s\n", + flags, mode, strerror(errno)); +return db; } /* EXIM_DBGET - returns TRUE if successful, FALSE otherwise */ @@ -60,10 +74,36 @@ exim_dbget(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * res) return res->dptr != NULL; } + +static inline BOOL +exim_dbtransaction_start(EXIM_DB * db) +{ +BOOL ok = tdb_transaction_start(db) == 0; +DEBUG(D_hints_lookup) if (!ok) + debug_printf_indent("tdb_transaction_start: %s\n", tdb_errorstr(db)); +return ok; +} + +static inline void +exim_dbtransaction_commit(EXIM_DB * db) +{ +BOOL ok = tdb_transaction_commit(db) == 0; +DEBUG(D_hints_lookup) if (!ok) + debug_printf_indent("tdb_transaction_commit: %s\n", tdb_errorstr(db)); +return; +} + + + /* EXIM_DBPUT - returns nothing useful, assumes replace mode */ static inline int exim_dbput(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * data) -{ return tdb_store(dbp, *key, *data, TDB_REPLACE); } +{ +int rc = tdb_store(dbp, *key, *data, TDB_REPLACE); +DEBUG(D_hints_lookup) if (rc != 0) + debug_printf_indent("tdb_store: %s\n", tdb_errorstr(dbp)); +return rc; +} /* EXIM_DBPUTB - non-overwriting for use by dbmbuild */ static inline int @@ -109,12 +149,33 @@ return key->dptr != NULL; /* EXIM_DBDELETE_CURSOR - terminate scanning operation. */ static inline void exim_dbdelete_cursor(EXIM_CURSOR * cursor) -{ store_free(cursor); } +{ +#ifdef COMPILE_UTILITY +free(cursor); +#else +store_free(cursor); +#endif +} /* EXIM_DBCLOSE */ +static inline void +exim_dbclose_multi__(EXIM_DB * db) +{ +int rc = tdb_close(db); +DEBUG(D_hints_lookup) if (rc != 0) + debug_printf_indent("tdb_close: %s\n", tdb_errorstr(db)); +} + static inline void exim_dbclose__(EXIM_DB * db) -{ tdb_close(db); } +{ +int rc = tdb_transaction_commit(db); +DEBUG(D_hints_lookup) if (rc != 0) + debug_printf_indent("tdb_transaction_commit: %s\n", tdb_errorstr(db)); +rc = tdb_close(db); +DEBUG(D_hints_lookup) if (rc != 0) + debug_printf_indent("tdb_close: %s\n", tdb_errorstr(db)); +} /* Datum access */ diff --git a/src/src/hintsdb_structs.h b/src/src/hintsdb_structs.h index 5f50dfd5f..5adb68eb5 100644 --- a/src/src/hintsdb_structs.h +++ b/src/src/hintsdb_structs.h @@ -21,6 +21,7 @@ that relates to it. */ typedef struct { void * dbptr; int lockfd; + BOOL readonly; } open_db; diff --git a/src/src/queue.c b/src/src/queue.c index 3073ee780..57e5eb769 100644 --- a/src/src/queue.c +++ b/src/src/queue.c @@ -513,23 +513,31 @@ for (int i = queue_run_in_order ? -1 : 0; (double)load_average/1000.0, (double)deliver_queue_load_max/1000.0); - /* If initial of a 2-phase run, maintain a set of child procs - to get disk parallelism */ + /* If initial of a 2-phase run (and not under the test-harness) + maintain a set of child procs to get disk parallelism */ if (q->queue_2stage && !queue_run_in_order) { int i; - if (qpid[f.running_in_test_harness ? 0 : nelem(qpid) - 1]) - { - DEBUG(D_queue_run) debug_printf("q2stage waiting for child %d\n", (int)qpid[0]); + if (qpid[ +#ifndef MEASURE_TIMING + f.running_in_test_harness ? 0 : +#endif + nelem(qpid) - 1]) + { /* The child table is maxed out; wait for the oldest */ + DEBUG(D_queue_run) + debug_printf("q2stage waiting for child %d\n", (int)qpid[0]); waitpid(qpid[0], NULL, 0); - DEBUG(D_queue_run) debug_printf("q2stage reaped child %d\n", (int)qpid[0]); - if (f.running_in_test_harness) i = 0; - else for (i = 0; i < nelem(qpid) - 1; i++) qpid[i] = qpid[i+1]; + DEBUG(D_queue_run) + debug_printf("q2stage reaped child %d\n", (int)qpid[0]); +#ifndef MEASURE_TIMING + if (f.running_in_test_harness) i = 0; else +#endif + for (i = 0; i < nelem(qpid) - 1; i++) qpid[i] = qpid[i+1]; qpid[i] = 0; } else - for (i = 0; qpid[i]; ) i++; + for (i = 0; qpid[i]; ) i++; /* find first spare slot */ if ((qpid[i] = exim_fork(US"qrun-phase-one"))) continue; /* parent loops around */ } @@ -565,7 +573,8 @@ for (int i = queue_run_in_order ? -1 : 0; follow. If the message is chosen for delivery, the header is read again in the deliver_message() function, in a subprocess. */ - if (spool_read_header(fq->text, FALSE, TRUE) != spool_read_OK) goto go_around; + if (spool_read_header(fq->text, FALSE, TRUE) != spool_read_OK) + goto go_around; f.dont_deliver = orig_dont_deliver; /* Now decide if we want to deliver this message. As we have read the @@ -739,6 +748,7 @@ single_item_retry: set_process_info("running queue"); /* If initial of a 2-phase run, we are a child - so just exit */ + if (q->queue_2stage && !queue_run_in_order) exim_exit(EXIT_SUCCESS); diff --git a/src/src/retry.c b/src/src/retry.c index e86b1afe8..c32bd85b7 100644 --- a/src/src/retry.c +++ b/src/src/retry.c @@ -153,19 +153,23 @@ retry_check_address(const uschar *domain, host_item *host, uschar *portstring, BOOL yield = FALSE; time_t now = time(NULL); const uschar * host_key, * message_key; -open_db dbblock, * dbm_file; +open_db dbblock, * dbm_file = NULL; tree_node * node; dbdata_retry * host_retry_record, * message_retry_record; *retry_host_key = *retry_message_key = NULL; -DEBUG(D_transport|D_retry) debug_printf("checking retry status of %s\n", host->name); - /* Do nothing if status already set; otherwise initialize status as usable. */ if (host->status != hstatus_unknown) return FALSE; host->status = hstatus_usable; +DEBUG(D_transport|D_retry) + { + debug_printf_indent("checking retry status of %s\n", host->name); + acl_level++; + } + /* Generate the host key for the unusable tree and the retry database. Ensure host names are lower cased (that's what %S does). Generate the message-specific key too. @@ -182,11 +186,12 @@ the retry database when it is updated). */ if ((node = tree_search(tree_unusable, host_key))) { - DEBUG(D_transport|D_retry) debug_printf("found in tree of unusables\n"); - host->status = (node->data.val > 255)? - hstatus_unusable_expired : hstatus_unusable; + DEBUG(D_transport|D_retry) + debug_printf_indent("found in tree of unusables\n"); + host->status = node->data.val > 255 + ? hstatus_unusable_expired : hstatus_unusable; host->why = node->data.val & 255; - return FALSE; + goto out; } /* Open the retry database, giving up if there isn't one. Otherwise, search for @@ -194,40 +199,49 @@ the retry records, and then close the database again. */ if (!continue_retry_db) dbm_file = dbfn_open(US"retry", O_RDONLY, &dbblock, FALSE, TRUE); -else if ((dbm_file = continue_retry_db) == (open_db *)-1) - dbm_file = NULL; +else if (continue_retry_db != (open_db *)-1) + { + DEBUG(D_hints_lookup) + debug_printf_indent(" using cached retry hintsdb handle\n"); + dbm_file = continue_retry_db; + } +else DEBUG(D_hints_lookup) + debug_printf_indent(" using cached retry hintsdb nonpresence\n"); if (!dbm_file) { DEBUG(D_deliver|D_retry|D_hints_lookup) - debug_printf("no retry data available\n"); - return FALSE; + debug_printf_indent("no retry data available\n"); + goto out; } host_retry_record = dbfn_read(dbm_file, host_key); message_retry_record = dbfn_read(dbm_file, message_key); if (!continue_retry_db) dbfn_close(dbm_file); +else + DEBUG(D_hints_lookup) debug_printf_indent("retaining retry hintsdb handle\n"); /* Ignore the data if it is too old - too long since it was written */ if (!host_retry_record) { - DEBUG(D_transport|D_retry) debug_printf("no host retry record\n"); + DEBUG(D_transport|D_retry) debug_printf_indent("no host retry record\n"); } else if (now - host_retry_record->time_stamp > retry_data_expire) { host_retry_record = NULL; - DEBUG(D_transport|D_retry) debug_printf("host retry record too old\n"); + DEBUG(D_transport|D_retry) debug_printf_indent("host retry record too old\n"); } if (!message_retry_record) { - DEBUG(D_transport|D_retry) debug_printf("no message retry record\n"); + DEBUG(D_transport|D_retry) debug_printf_indent("no message retry record\n"); } else if (now - message_retry_record->time_stamp > retry_data_expire) { message_retry_record = NULL; - DEBUG(D_transport|D_retry) debug_printf("message retry record too old\n"); + DEBUG(D_transport|D_retry) + debug_printf_indent("message retry record too old\n"); } /* If there's a host-specific retry record, check for reaching the retry @@ -249,7 +263,7 @@ if (host_retry_record) if (!host_retry_record->expired && retry_ultimate_address_timeout(host_key, domain, host_retry_record, now)) - return FALSE; + goto out; /* We have not hit the ultimate address timeout; host is unusable. */ @@ -257,7 +271,7 @@ if (host_retry_record) hstatus_unusable_expired : hstatus_unusable; host->why = hwhy_retry; host->last_try = host_retry_record->last_try; - return FALSE; + goto out; } /* Host is usable; set return TRUE if expired. */ @@ -280,10 +294,12 @@ if (message_retry_record) host->status = hstatus_unusable; host->why = hwhy_retry; } - return FALSE; + yield = FALSE; goto out; } } +out: +DEBUG(D_transport|D_retry) acl_level--; return yield; } @@ -531,6 +547,7 @@ return yield; /* Update the retry data for any directing/routing/transporting that was deferred, or delete it for those that succeeded after a previous defer. This is done all in one go to minimize opening/closing/locking of the database file. +Called (only) from deliver_message(). Note that, because SMTP delivery involves a list of destinations to try, there may be defer-type retry information for some of them even when the message was @@ -590,8 +607,7 @@ for (int i = 0; i < 3; i++) for (addr = endaddr; addr; addr = addr->parent) { - int update_count = 0; - int timedout_count = 0; + int update_count = 0, timedout_count = 0; DEBUG(D_retry) { @@ -617,14 +633,20 @@ for (int i = 0; i < 3; i++) reached their retry next try time. */ if (!dbm_file) - dbm_file = dbfn_open(US"retry", O_RDWR|O_CREAT, &dbblock, TRUE, TRUE); - - if (!dbm_file) - { - DEBUG(D_deliver|D_retry|D_hints_lookup) - debug_printf_indent("retry database not available for updating\n"); - return; - } + if (continue_retry_db && continue_retry_db != (open_db *)-1) + { + DEBUG(D_hints_lookup) + debug_printf_indent("using cached retry hintsdb handle\n"); + dbm_file = continue_retry_db; + } + else if (!(dbm_file = exim_lockfile_needed() + ? dbfn_open(US"retry", O_RDWR|O_CREAT, &dbblock, TRUE, TRUE) + : dbfn_open_multi(US"retry", O_RDWR|O_CREAT, &dbblock))) + { + DEBUG(D_deliver|D_retry|D_hints_lookup) + debug_printf_indent("retry db not available for updating\n"); + return; + } /* If there are no deferred addresses, that is, if this message is completing, and the retry item is for a message-specific SMTP error, @@ -699,6 +721,12 @@ for (int i = 0; i < 3; i++) message_length = EXIM_DB_RLIMIT; } + /* For a transaction-capable DB, open one for the read,write + sequence used for this retry record */ + + if (!exim_lockfile_needed()) + dbfn_transaction_start(dbm_file); + /* Read a retry record from the database or construct a new one. Ignore an old one if it is too old since it was last updated. */ @@ -888,8 +916,12 @@ for (int i = 0; i < 3; i++) debug_printf(" %s\n", retry_record->text); } - (void)dbfn_write(dbm_file, rti->key, retry_record, - sizeof(dbdata_retry) + message_length); + if (dbfn_write(dbm_file, rti->key, retry_record, + sizeof(dbdata_retry) + message_length) != 0) + DEBUG(D_retry) debug_printf_indent("retry record write failed\n"); + + if (!exim_lockfile_needed()) + dbfn_transaction_commit(dbm_file); } /* Loop for each retry item */ DEBUG(D_retry) acl_level--; @@ -971,9 +1003,17 @@ for (int i = 0; i < 3; i++) /* Close and unlock the database */ -if (dbm_file) dbfn_close(dbm_file); +if (dbm_file) + if (dbm_file != continue_retry_db) + if (exim_lockfile_needed()) + dbfn_close(dbm_file); + else + dbfn_close_multi(dbm_file); + else DEBUG(D_hints_lookup) + debug_printf_indent("retaining retry hintsdb handle\n"); -DEBUG(D_retry) { acl_level--; debug_printf_indent("end of retry processing\n"); } +DEBUG(D_retry) + { acl_level--; debug_printf_indent("end of retry processing\n"); } } /* End of retry.c */ commit 64c6aae63ba5a5f2f0dbc5e90323091d253b1d22 Author: Jeremy Harris Date: Fri Jul 19 06:04:30 2024 +0100 Fix build with Solaris compiler Broken-by: 3cee6033bae8 diff --git a/src/src/exim.h b/src/src/exim.h index 9bf5dcdfc..c4d80c694 100644 --- a/src/src/exim.h +++ b/src/src/exim.h @@ -525,14 +525,18 @@ config.h, mytypes.h, and store.h, so we don't need to mention them explicitly. #include "path_max.h" #include "macros.h" #include "blob.h" -#include "hintsdb.h" +#ifndef MACRO_PREDEF +# include "hintsdb.h" +#endif #include "hintsdb_structs.h" #include "structs.h" #include "blob.h" #include "hash.h" #include "globals.h" #include "functions.h" -#include "dbfunctions.h" +#ifndef MACRO_PREDEF +# include "dbfunctions.h" +#endif #include "osfunctions.h" #ifdef EXPERIMENTAL_BRIGHTMAIL diff --git a/src/src/hintsdb/hints_bdb.h b/src/src/hintsdb/hints_bdb.h index 48573e907..99878a2d6 100644 --- a/src/src/hintsdb/hints_bdb.h +++ b/src/src/hintsdb/hints_bdb.h @@ -72,9 +72,7 @@ at DB release 4.3. */ static inline void dbfn_bdb_error_callback(const DB_ENV * dbenv, const char * pfx, const char * msg) { -#ifndef MACRO_PREDEF log_write(0, LOG_MAIN, "Berkeley DB error: %s", msg); -#endif }