Add a "message_logs" configuration option to allow disabling per-message logs for performance. Turning off per-message logs currently breaks queue_run_first_delivery (-qi) functionality, because this uses the absence of the per-message log to indicate that this will be the first delivery attempt for the message; since per-message logs aren't created, -qi will scan the whole queue. This patch also breaks the return_output option on pipes because they rely on the presence of the msglog subdirectory which won't be created with no_message_logs in action. --- doc/spec.txt.orig2 Wed Jun 26 12:44:20 2002 +++ doc/spec.txt Wed Jun 26 13:03:09 2002 @@ -7502,6 +7502,16 @@ variables such as $tod_log can be used, because the spaces and colons will become hyphens. +message_logs Type: boolean Default: true + + If this option is turned off then per-message log files are not created in + the msglog spool sub-directory. This can reduce the amount of disk I/O + required by Exim, by reducing the number of files involved in handling a + message from a minimum of four (header spool file, body spool file, + delivery journal, and per-message log) to three. The other major I/O + activities are the hints database and the main log (which this option does + not affect). + message_size_limit Type: string* Default: 50M This option limits the maximum size of message that Exim will process. The @@ -17268,6 +17277,8 @@ the main log. A message log is deleted when processing of the message is complete, unless "preserve_message_logs" is set, but this should be used only with great care because they can fill up your disc very quickly. +Alternatively, you can unset "message_logs" in order to eliminate the +additional I/O load that they cause. --- src/queue.c.orig2 Wed Jun 26 12:58:07 2002 +++ src/queue.c Wed Jun 26 13:00:32 2002 @@ -404,6 +404,9 @@ because of 2-stage queue runs; the first stage will create the msglog, but won't write anything to it. */ + /* XXX: This is broken when message_logs is false, in which case Exim + should probably fall back to looking at the header files. */ + if (queue_run_first_delivery) { sprintf(CS buffer, "%s/msglog/%s/%s", spool_directory, message_subdir, --- src/readconf.c.orig2 Thu Jun 13 09:32:06 2002 +++ src/readconf.c Thu Jun 27 19:44:00 2002 @@ -118,6 +118,7 @@ #ifdef LOOKUP_MYSQL { "mysql_servers", opt_stringptr, &mysql_servers }, #endif + { "message_logs", opt_bool, &message_logs }, { "never_users", opt_uidlist, &never_users }, #ifdef LOOKUP_ORACLE { "oracle_servers", opt_stringptr, &oracle_servers }, --- src/globals.c.orig2 Mon Jun 24 16:07:41 2002 +++ src/globals.c Mon Jun 24 16:10:04 2002 @@ -526,6 +526,7 @@ uschar *message_id_external; int message_linecount = 0; FILE *message_log = NULL; +BOOL message_logs = TRUE; int message_size = 0; uschar *message_size_limit = US"50M"; uschar message_subdir[2] = { 0, 0 }; --- src/globals.h.orig2 Mon Jun 24 16:07:50 2002 +++ src/globals.h Mon Jun 24 16:10:44 2002 @@ -317,6 +317,7 @@ extern uschar *message_id_text; /* Expanded to form message_id */ extern int message_linecount; /* As it says */ extern FILE *message_log; /* Open message log while delivering */ +extern BOOL message_logs; /* Create per-message log files */ extern int message_size; /* Size of message */ extern uschar *message_size_limit; /* As it says */ extern uschar message_subdir[]; /* Subdirectory for messages */ --- src/functions.h.orig2 Thu Jun 13 09:32:04 2002 +++ src/functions.h Tue Jul 2 14:58:09 2002 @@ -121,6 +121,7 @@ extern void md5_end(md5 *, const uschar *, int, uschar *); extern void md5_mid(md5 *, const uschar *); extern void md5_start(md5 *); +extern void message_log_write(const char *format, ...); extern void millisleep(int); extern uschar *moan_check_errorcopy(uschar *); extern BOOL moan_skipped_syntax_errors(uschar *, error_block *, uschar *, --- src/transports/smtp.c.orig2 Thu Jun 13 09:32:07 2002 +++ src/transports/smtp.c Tue Jul 2 14:48:32 2002 @@ -423,7 +423,7 @@ if (addr->basic_errno > 0) message = string_sprintf("%s: %s", message, strerror(addr->basic_errno)); log_write(0, LOG_MAIN, "%s", message); - fprintf(message_log, "%s %s\n", tod_stamp(tod_log), message); + message_log_write("%s %s\n", tod_stamp(tod_log), message); } else { @@ -431,13 +431,12 @@ host->name, host->address, strerror(addr->basic_errno)); - fprintf(message_log, "%s %s [%s]: %s\n", + message_log_write("%s %s [%s]: %s\n", tod_stamp(tod_log), host->name, host->address, strerror(addr->basic_errno)); } -fflush(message_log); } @@ -579,7 +578,7 @@ string_sprintf("SMTP error from remote mailer after RCPT TO:<%s>: " "host %s [%s]: %s", addr->rcpt, host->name, host->address, string_printing(buffer)); - fprintf(message_log, "%s %s\n", tod_stamp(tod_log), addr->message); + message_log_write("%s %s\n", tod_stamp(tod_log), addr->message); /* The response was 5xx */ @@ -1491,7 +1490,7 @@ if (save_errno > 0) message = US string_sprintf("%s: %s", message, strerror(save_errno)); if (host->next != NULL) log_write(0, LOG_MAIN, "%s", message); - fprintf(message_log, "%s %s\n", tod_stamp(tod_log), message); + message_log_write("%s %s\n", tod_stamp(tod_log), message); *message_defer = TRUE; } } --- src/deliver.c.orig2 Mon Jun 24 16:07:57 2002 +++ src/deliver.c Mon Jun 24 16:44:23 2002 @@ -75,6 +75,35 @@ /************************************************* +* Write to a message log * +*************************************************/ + +/* The arguments are formatted as for printf() and written to the +per-message log file, depending on the configuration. + + format a printf() format + ... arguments for format + +Returns: nothing +*/ + +void +message_log_write(const char *format, ...) +{ +va_list ap; + +if (message_log != NULL) + { + va_start(ap, format); + vfprintf(message_log, format, ap); + fflush(message_log); + va_end(ap) + } +} + + + +/************************************************* * Make a new address item * *************************************************/ @@ -510,8 +539,7 @@ if (Ustrcmp(aa->orig, addr->orig) == 0) break; if (aa != NULL) continue; - fprintf(message_log, "%s %s: children all complete\n", now, addr->orig); - fflush(message_log); + message_log_write("%s %s: children all complete\n", now, addr->orig); DEBUG(D_deliver) debug_printf("%s: children all complete\n", addr->orig); } } @@ -685,18 +713,16 @@ if (addr->parent == NULL) { - fprintf(message_log, "%s %s: %s%s succeeded\n", now, addr->orig, + message_log_write("%s %s: %s%s succeeded\n", now, addr->orig, driver_name, driver_kind); } else { - fprintf(message_log, "%s %s <%s>: %s%s succeeded\n", now, addr->orig, + message_log_write("%s %s <%s>: %s%s succeeded\n", now, addr->orig, addr->parent->orig, driver_name, driver_kind); child_done(addr, now); } - fflush(message_log); - /* Log the delivery on the main log. We use an extensible string to build up the log line, and reset the store afterwards. Remote deliveries should always have a pointer to the host item that succeeded; local deliveries can have a @@ -867,10 +893,7 @@ up with retry-time defers after the first delivery attempt. */ if (deliver_firsttime || addr->basic_errno > ERRNO_RETRY_BASE) - { - fprintf(message_log, "%s %s\n", now, s); - fflush(message_log); - } + message_log_write("%s %s\n", now, s); /* Write the main log and reset the store */ @@ -962,11 +985,9 @@ just to make it clearer. */ if (driver_name == NULL) - fprintf(message_log, "%s %s failed for %s\n", now, driver_kind, s); + message_log_write("%s %s failed for %s\n", now, driver_kind, s); else - fprintf(message_log, "%s %s\n", now, s); - - fflush(message_log); + message_log_write("%s %s\n", now, s); log_write(0, LOG_MAIN, "** %s", s); store_reset(reset_point); @@ -4096,54 +4117,57 @@ update_spool = TRUE; } +if (message_logs) + { + /* Open the message log file. This records details of deliveries, deferments, + and failures for the benefit of the mail administrator. The log is not used by + exim itself to track the progress of a message; that is done by rewriting the + header spool file. */ + + sprintf(CS spoolname, "%s/msglog/%s/%s", spool_directory, message_subdir, id); + fd = Uopen(spoolname, O_WRONLY|O_APPEND|O_CREAT, SPOOL_MODE); -/* Open the message log file. This records details of deliveries, deferments, -and failures for the benefit of the mail administrator. The log is not used by -exim itself to track the progress of a message; that is done by rewriting the -header spool file. */ - -sprintf(CS spoolname, "%s/msglog/%s/%s", spool_directory, message_subdir, id); -fd = Uopen(spoolname, O_WRONLY|O_APPEND|O_CREAT, SPOOL_MODE); - -if (fd < 0) - { - if(errno == ENOENT) - { - uschar temp[16]; - sprintf(CS temp, "msglog/%s", message_subdir); - if (message_subdir[0] == 0) temp[6] = 0; - (void)directory_make(spool_directory, temp, MSGLOG_DIRECTORY_MODE, TRUE); - fd = Uopen(spoolname, O_WRONLY|O_APPEND|O_CREAT, SPOOL_MODE); - } if (fd < 0) { - log_write(0, LOG_MAIN|LOG_PANIC, "Couldn't open message log %s: %s", - spoolname, strerror(errno)); - return continue_closedown(); + if(errno == ENOENT) + { + uschar temp[16]; + sprintf(CS temp, "msglog/%s", message_subdir); + if (message_subdir[0] == 0) temp[6] = 0; + (void)directory_make(spool_directory, temp, MSGLOG_DIRECTORY_MODE, TRUE); + fd = Uopen(spoolname, O_WRONLY|O_APPEND|O_CREAT, SPOOL_MODE); + } + if (fd < 0) + { + log_write(0, LOG_MAIN|LOG_PANIC, "Couldn't open message log %s: %s", + spoolname, strerror(errno)); + return continue_closedown(); + } } - } -/* Set the close-on-exec flag. */ + /* Set the close-on-exec flag. */ -fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | FD_CLOEXEC); + fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | FD_CLOEXEC); -/* Exim is running as root here; the message log is therefore owned by root -when it is created. Change it to the exim uid/gid, and double-check the mode -because the group setting doesn't always get set automatically. */ + /* Exim is running as root here; the message log is therefore owned by root + when it is created. Change it to the exim uid/gid, and double-check the mode + because the group setting doesn't always get set automatically. */ -fchown(fd, exim_uid, exim_gid); -fchmod(fd, SPOOL_MODE); + fchown(fd, exim_uid, exim_gid); + fchmod(fd, SPOOL_MODE); -/* Now make a C stream out of it. */ + /* Now make a C stream out of it. */ -message_log = fdopen(fd, "a"); -if (message_log == NULL) - { - log_write(0, LOG_MAIN|LOG_PANIC, "Couldn't fdopen message log %s: %s", - spoolname, strerror(errno)); - return continue_closedown(); - } + message_log = fdopen(fd, "a"); + if (message_log == NULL) + { + log_write(0, LOG_MAIN|LOG_PANIC, "Couldn't fdopen message log %s: %s", + spoolname, strerror(errno)); + return continue_closedown(); + } + /* That's the message log dealt with. */ + } /* If asked to give up on a message, log who did it, and set the action for all the addresses. */ @@ -5682,7 +5706,7 @@ (void)spool_write_header(message_id, SW_DELIVERING, NULL); s = US" (frozen)"; } - fprintf(message_log, "Process failed (%d) when writing error message " + message_log_write("Process failed (%d) when writing error message " "to %s%s", rc, bounce_recipient, s); log_write(0, LOG_MAIN, "Process failed (%d) when writing error message " "to %s%s", rc, bounce_recipient, s); @@ -5711,24 +5735,28 @@ if (addr_defer == NULL) { int rc; - sprintf(CS spoolname, "%s/msglog/%s/%s", spool_directory, message_subdir, id); - if (preserve_message_logs) + + if (message_logs) { - sprintf(CS big_buffer, "%s/msglog.OLD/%s", spool_directory, id); - if ((rc = Urename(spoolname, big_buffer)) < 0) + sprintf(CS spoolname, "%s/msglog/%s/%s", spool_directory, message_subdir, id); + if (preserve_message_logs) { - (void)directory_make(spool_directory, US"msglog.OLD", MSGLOG_DIRECTORY_MODE, - TRUE); - rc = Urename(spoolname, big_buffer); - } - if (rc < 0) - log_write(0, LOG_MAIN|LOG_PANIC_DIE, "failed to move %s to the msglog.OLD " - "directory", spoolname); - } - else - { - if (Uunlink(spoolname) < 0) - log_write(0, LOG_MAIN|LOG_PANIC_DIE, "failed to unlink %s", spoolname); + sprintf(CS big_buffer, "%s/msglog.OLD/%s", spool_directory, id); + if ((rc = Urename(spoolname, big_buffer)) < 0) + { + (void)directory_make(spool_directory, US"msglog.OLD", MSGLOG_DIRECTORY_MODE, + TRUE); + rc = Urename(spoolname, big_buffer); + } + if (rc < 0) + log_write(0, LOG_MAIN|LOG_PANIC_DIE, "failed to move %s to the msglog.OLD " + "directory", spoolname); + } + else + { + if (Uunlink(spoolname) < 0) + log_write(0, LOG_MAIN|LOG_PANIC_DIE, "failed to unlink %s", spoolname); + } } /* Remove the two message files. */ @@ -6064,8 +6092,7 @@ /* Log freezing just before we update the -H file, to minimize the chance of a race problem. */ - fprintf(message_log, "*** Frozen%s\n", frozen_info); - fflush(message_log); + message_log_write("*** Frozen%s\n", frozen_info); log_write(0, LOG_MAIN, "Frozen%s", frozen_info); } @@ -6087,7 +6114,7 @@ /* Finished with the message log. If the message is complete, it will have been unlinked or renamed above. */ -fclose(message_log); +if (message_logs) fclose(message_log); /* Now we can close and remove the journal file. Its only purpose is to record successfully completed deliveries asap so that this information doesn't get