commit 62dc691279a0d0ee0a46a7b0bea6f65257e81f03 Author: Michael Krayer Date: Fri May 22 09:47:41 2020 +0200 forked from git://archivemail.git.sourceforge.net/gitroot/archivemail/archivemail diff --git a/CHANGELOG b/CHANGELOG new file mode 100644 index 0000000..d0d4424 --- /dev/null +++ b/CHANGELOG @@ -0,0 +1,333 @@ +version 0.9.0 - 9 July 2011 + + * Fixed manpage installation path to be FHS compliant + * Speed up IMAP archiving with the --quiet option + * Ported the manpage from SGML to XML + * Fix test suite failures with Python 2.7. Closes: #3314293. + * IMAP: support international mailbox names containing non-ASCII characters. + * IMAP: handle broken servers sending no untagged SEARCH response. + Closes: #879716, #3213272. + * IMAP: support servers listening on non-standard ports. Closes: #3168416. + +version 0.8.2 - 16 October 2010 + + * IMAP: don't prepend NAMESPACE prefix to INBOX and its children. + Closes: #3083236. + +version 0.8.1 - 30 September 2010 + + * IMAP: fixed handling of LIST replies by the server where the mailbox name + is not a quoted string. (Thanks Karsten Müller) + +Version 0.8.0 - 9 August 2010 + + * Fixed date header parsing to be precise with timezone information. Also, + when writing From_ line timestamps, don't use UTC but local time, without + timezone information. + * To determine the delivery date of a message, archivemail now looks for the + timestamp of the latest 'Received' header before resorting to + 'Resent-Date' or 'Date'. This should give much better results when there + is no 'Delivery-date' header, which is still checked first. + (Thanks Andrew Ferrier & Christian Brabandt) + Closes: #1481316, #1764855, Debian bug #272666. + * If present, the 'Resent-date' header now takes precedence over 'Date'. + * IMAP: recognize when a server advertises LOGINDISABLED. + * New option --debug-imap; this just sets imaplib.Debug, thereby dumping + archivemail's chat with the server to stdout. + * Fixed crash with Python 2.5 when archiving an empty maildir. + (Thanks "Aidant") Closes: #1878940. + * New option --all to archive all messages in a mailbox. Closes: #1764846. + * Fixed a crash when archiving maildirs with --days=0. (Thanks John Goerzen) + * IMAP: automatically add NAMESPACE prefix to a mailbox path if necessary. + * Removed the feature to setuid to the mailbox owners when run as root. + This was a bad idea; it's really hard to do safely, if at all possible. + Obsoletes: patch #2783134. + * Replaced some simple minded file operation security checks with more + decent ones. This means we can safely operate in /tmp, for example. The + price is that we no longer accept symlinked files. Obsoletes: patch + #1874868. + * Don't use rename() to update mbox files and the archive, but write the + files directly. This is more fragile, but required for correct mbox + locking, and also for mboxes in mail spool directories where we don't have + permission to create files. It also means that if selinux is enabled, + archivemail now preserves the selinux security context of an mbox. + Closes: #2210732. + * Fixed the test suite to deal with nanosecond file timestamps. These are + provided by ext4 and XFS, for example. Closes: #2043900. + * Cleaned up the test suite, replacing a lot of duplicated code and avoiding + a lot of redundand testing. This speeds up the test suite by a factor of + 15 or so. + * mbox locking got completely rewritten. Switched from flock to lockf + locking, which is NFS-safe and portable, and we now lock with lockf first, + then with a dotlock, instead of the other way around. (This is makes + archivemail compatible with Debian systems. ;) + * We now omit the dotlock if we don't have sufficient permissions to create + it in the mbox directory. (The file is still locked with lockf.) + Since we also no longer use rename() to commit changes to an mbox, (see + above) this means archivemail can now operate on mbox files in the system + mail spool. Closes: #855269. + * Refactoring of the mbox classes; much of the code got rewritten. + * The archive now also gets locked while archivemail updates it. + * Various Python language fixes (for example don't use "0" and "1" as + boolean constants). + * Added a lot of test cases for maildir archiving to the test suite. + Maildir testing should now be roughly on par with mbox testing. + * IMAP servers (Dovecot and UW-IMAP at least) may store mailbox meta data + for mboxes in a pseudo message. Such messages are now detected and never + archived. Obsoletes: patch #2210707. (Thanks, "tlhackque") + * New option --prefix, or short -p, to specify an archive name prefix. Like + a suffix specified with the --suffix option, the prefix is expanded with + strftime(). Specifying this option disables the default archive name + suffix. Obsoletes: feature request #604281. (Thanks Serafeim Zanikolas + for an initial patch) + * When archiving a mailbox with a leading dot in the name and with no archive + name prefix specified, archivemail no longer creates hidden archives, but + strips the dot off the archive name. In particular, this makes working + with Maildir++ subfolders more convenient. Closes: feature request + #604281. + * New option --archive-name, or short -a, to hard-code an archive filename. + Like the --suffix and --prefix options, it is expanded with strftime(). + This option conflicts with archiving multiple mailboxes. Closes: feature + request #1306538. + * archivemail now expands wildcards in IMAP mailbox names. For example, the + url imaps://user@server/foo/* will expand to all subfolders of foo. + Closes: feature request #1978540. Obsoletes: patch #1918937. + +Version 0.7.2 - 9 November 2007 + + * IMAP: fixed crash by working around python bug #1277098, which is still pending + in python << 2.5. + +Version 0.7.1 - 7 November 2007 + + * Fixed incompatibility with Python 2.5 which broke Maildir handling. + Closes: #1670422 + * Username and password in IMAP URLs can now be double-quoted, so it should be + no longer a problem if they contain delimiters like the '@' character. + Closes: #1640878 + * Invalid messages/files in Maildirs caused archivemail to silently stop + processing mails and claim it's all done. Now skip these and go ahead. + (Thanks Elan Ruusamäe for tracking this down.) Closes: #1783369. + (The Debian package has a different fix for this problem since 0.6.1-4, + closing Debian bugs #255944 and #305902.) + * Fixed IMAP message flag conversion which was completely broken. (Thanks + Christian Brabandt) Closes: Debian bug #434807 + * New option --copy: archive mail, but don't delete it from the mailbox. + This is mainly useful for testing purposes, and complements --delete. + Closes: #981865, #988803, #1764851, Debian bug #434798 + * If running as root, only switch the effective uid and gid back if we have + actually switched them before. Closes: #1762907 + * The automatic seteuid feature of archivemail is insecure and thus + deprecated; it will be removed from later versions. + * Expand tilde in argument of long option --pwfile. (Thanks Christian + Brabandt) Closes: Debian bug #434813 + * archivemail now accepts --days=0 + * Fixed crash if --warn-duplicate is used with IMAP (Thanks Christian + Brabandt) Closes: Debian bug #434786 (the Debian package already has a fix) + * When converting from other formats to mbox, archivemail used to preserve + existing 'Status' and 'X-Status' mbox headers; these are now silently + overwritten to ensure they have correct values. + * IMAP: if selecting the mailbox fails, archivemail detects the server's + mailbox hierarchy delimiter, replaces slashes in the mailbox name with the + delimiter and tries again. Closes: #1826757, Debian bug #368112 + +Version 0.7.0 - 2 November 2006 + * Fixed long options --filter-append and --pwfile to accept their arguments. + Closes: #1555935 + * Fixed From_ line generation to actually look for the 'Return-path' and + 'From' headers. Closes: #1555797 + * Fixed IMAP authentication/URL parsing, which wasn't working at all in + v0.6.2. Require username encoded in URL, but be flexible with the password: + handle both --pwfile and URL-encoded password, and fallback to querying the + user if neither is present. Closes: #1555951 + * Convert on-the-wire CRLF to native EOL when saving messages from an IMAP + folder. Closes: #1555803 + * Updated man page. This also addresses #1555951 + * Fixed unittest failure by updating --suffix testcase. Based on analysis by + Peter Poeml. Thanks, Peter. + * Fixed invalid IMAP filter string for large messages (--size option). + (Thanks to the anonymous bug reporter) Closes: #863813 + * Fixed IMAP --dry-run so it doesn't download all messages that would be + archived. + * Fixed IMAP --delete which didn't work at all. (Thanks Anand) + Closes: Debian bug #203282 + * Terminate each message in newly written mbox with an empty line if the + message source is not an mbox-format folder. (Thanks Chung-chieh Shan) + Closes: Debian bug #250410 + * Mangle From_ in message body if the message source is not an mbox-format + folder. (Thanks Chung-chieh Shan) Closes: Debian bug #250402 + * Added new option --dont-mangle to turn off From_ mangling. + * Bumped Python dependency to version 2.3. + * Fixed unittest TestMboxExclusiveLock which failed on Solaris. (Thanks Paul + Rodger) Closes: #904652 + * Fixed unsafe creation of temporary files in the test suite. + This addresses Debian bug #385253, and reading the BTS log, it seems this + issue was assigned CVE-2006-4245, although I cannot find any further + reference to that CVE. Note that the bug was initially reported to affect + archivemail itself, too. This is not correct. (Thanks Joey Hess) + Closes: Debian bug #385253 + * Fixed cleanup of temporary files after test suite failures. + * Fixed dotlocking race condition. + * Stats are now working with IMAP. + * Stats now report the total size of a mailbox and of the archived messages + therefrom. + * Always barf if the archive destination directory is world-writable. + * Distributing man page with the tarball again and fixed distutils setup. + Closes: #1574720 (Thanks Grant Goodyear) + * Improved IMAP error reporting + +Version 0.6.2 - 27 June 2006 + * add -F/--filter-append option to append an arbitrary string to the IMAP + filter string + * don't delete more than a certain number of messages at a time. The max + command len is limited. Fixes bug 942403 (Archiving large IMAP folders fails) + * IMAP: try CRAM-MD5 login first, if that fails fallback to plain login + * add SSL support per imaps URL (after patch by Tobias Gruetzmacher) + * add -P/--pwfile option to supply the IMAP password, so it does not end up in + the shell history + * Fix SyntaxWarning: assignment to None (bug #843890) + * Use the archive cut date rather than the actual time with the --suffix + option. (Thanks Manuel Estrada Sainz) + +Version 0.6.1 - 31 October 2002 + * Removed a test rule that we could archive messages older than the + Unix epoch. Newer versions of python now give an overflow error calling + mktime() on dates < 1970 instead of returning a negative number. + +Version 0.6.0 - 3 October 2002 + * Added IMAP mailbox support. (Thanks Mark Roach) + +Version 0.5.1 - 18 September 2002 + * Fixed a bug where when running archivemail as root on a non-root mailbox, + the temporary container directory would be created as root before the + seteuid() took place. (Thanks Jay Hesselberth) + +Version 0.5.0 - 15 September 2002 + * Fixed a bug where mailbox locking would fail under Solaris. (Thanks Mark + Sheller) + * Fixed a bug where archiving maildir mailboxes without a 'Received Date' or + 'From' header would fail. (Thanks Hugo van der Merwe) + * Removed yet another bug where messages dated on the Unix epoch would fail. + +Version 0.4.9 - 18 August 2002 + * Fixed a bug where an exception was thrown if a message was dated exactly + on the Unix epoch. + * Fixed a bug where trailing slashes on the end of maildir/MH mailbox + arguments were being used in the archive name. + +Version 0.4.8 - 20 May 2002 + * Call mkdir() to create a container directory in which we can place any + created tempfiles + +Version 0.4.7 - 9 May 2002 + * Fixed a bug where archivemail would abort if it received a date header + with a completely blank value. + +Version 0.4.6 - 6 May 2002 + * Fixed a bug where the os.rename() calls could fail if we were moving + temporary files across different filesystems/partitions. + +Version 0.4.5 - 29 April 2002 + * Fixed a bug where if you used the '--delete' option to completely clean + an mbox mailbox you would get a python error. + * Added a lot more testing to test_archivemail.py (test_archivemail.py is + now 37% bigger than archivemail -- scary) + * Added a new '--size' option which allows you to only archive messages + over a certain size. + +Version 0.4.4 - 27 April 2002 + * Fixed a bug where the long --suffix option was not working (although the + short option, '-s' was). + * Added time-based format directives to the --suffix option, so that you + can do things like specify --suffix='%B%Y' to create archives named + after the current month and year + * Added some more tests to test_archivemail.py + +Version 0.4.3 - 26 April 2002 + * Fixed a couple of bugs where I was using python version 2.2 syntax that + was giving errors in python v2.0 and v2.1. + * Changed the python requirements for the test script from python version + 2.0 to version 2.1. This is because the unittest module is only available + in version 2.1 and above. + +Version 0.4.2 - 24 April 2002 + * Added the ability to archive messages older than a given absolute date + with the new option '--date'. + * Fixed a bug where archivemail would complain about messages older than + 1970. Yes, someone had a 'Date' header with 1967 :) + * Complain if the mailbox to be read does not look like a valid mbox-format + mailbox. + * Added a few more tests to test_archivemail.py + +Version 0.4.1 - 21 April 2002 + * Don't archive messages that are flagged important unless we are given the + --include-flagged option. + * Fixed a bug where when archiving messages from maildir mailboxes, we were + not preserving the status information contained in the filename suffix to + Status and X-Status mbox headers. This means we forgot if we had read or + replied to the message. + * We now complain if an mbox-format mailbox that is being read changes in + size -- this should not happen, since we have locked these files, but it + is a good sanity check. + * Changed from using the mailbox.PortableUnixMailbox class to read mbox + mailboxes to the mailbox.UnixMailbox class. This fixes bugs where unquoted + 'From ' lines in the body of messages were confusing archivemail. + +Version 0.4.0 - 17 April 2002 + * Added an option --no-compress to make archives but not compress them with + gzip. + * Added an option --preserve-unread to not archive unread messages. + * Added a few more unittests. + +Version 0.3.2 - 13 April 2002 + * Added a lot more information to the manpage, including examples and + notes. + * Fixed up the README file and archivemail usage message. + * Added an example archivemail shell script that I run from crontab. + +Version 0.3.1 - 12 April 2002 + * Stopped calling 'gzip' externally and started using the gzip library + so that we can append to a copy of the gzip archive directly. + * Removed 'bzip2' and 'compress' options since they are increasing + complexity without adding much, and needed to be called externally. + Maybe when python gets a bzip2 library I will add back an option to + compress archives using bzip2. + * Included a man page & sgml docbook source. + +Version 0.3.0 - 11 April 2002 + * We now preserve the last-accessed and last-modified timestamps correctly + * We now preserve the correct permissions on the original mailbox instead + of always mode 600 + * Fixed a bug where lockfiles were being created that were not + world-readable + * Made archivemail work better when used as a python module so it can + integrate better with unittest. (... although I still distribute it + without the .py extension - dodgy?) + * Bundled a unit-testing script for archivemail + * Started using a distutils 'setup.py' script for installation. + +Version 0.2.1 - 4 April 2002 + * Since we might not have a parse-able 'Date-Received' or 'Date' field, + use 5 different ways to guess the date of a message. + * Removed the '--use-mtime' option since it is obsolete -- we will always + use the file modification time for the message if other date-parsing + methods fail. + * Check to see if we are running as root -- if so, change our + effective userid and groupid to that of the original mailbox. This will + make sure any archives or tempfiles we write have the same ownership and + will allow the root user to do "archivemail /var/spool/mail/*" + * Fixed a bug where if you ran 'archivemail.py foo/mbox' it would create + the archive file in the current directory instead of the directory 'foo'. + +Version 0.2.0 - 3 April 2002 + * Added support for reading from MH mailboxes + * Refuse to proceed if we would be making tempfiles in world-writable + directories + * Clamped down with lots of assert()s checking function parameters + * Complain earlier if we do not have permission to write to the output + directory + * Use the 'Date' field of a message when constructing the 'From_' line + from a maildir/MH mailbox if there is no 'Delivery-Date' field. + +Version 0.1.0 - 31 March 2002 + * Initial release diff --git a/COPYING b/COPYING new file mode 100644 index 0000000..86fd703 --- /dev/null +++ b/COPYING @@ -0,0 +1,341 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc. + 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Library General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Library General +Public License instead of this License. + diff --git a/FAQ b/FAQ new file mode 100644 index 0000000..9fa2f29 --- /dev/null +++ b/FAQ @@ -0,0 +1,15 @@ + +1. Why doesn't archivemail support bzip2 compression in addition to gzip? +------------------------------------------------------------------------- + +The bzip2 module in Python 2.x is not fully compatible with the gzip module, +and cannot be used with the current implementation of compressed mailbox +support in archivemail. See Python feature request #5863 for details. + +2. Can you add a switch to archive mailboxes greater than a certain size? +------------------------------------------------------------------------- + +If you are using mbox format mailboxes instead, use the find(1) command instead, it is more flexible: + + find $HOME/Mail -type f ! -name '*archive*' + diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..886a863 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,11 @@ +include CHANGELOG +include COPYING +include FAQ +include MANIFEST +include TODO +include NEWS +include archivemail.1 +include archivemail.xml +include db2man.xsl +graft examples +include test_archivemail diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..57bad9c --- /dev/null +++ b/Makefile @@ -0,0 +1,43 @@ +VERSION=$(shell python setup.py --version) +VERSION_TAG=v$(subst .,_,$(VERSION)) +TARFILE=archivemail-$(VERSION).tar.gz +HTDOCS=htdocs-$(VERSION) + +default: + @echo "no default target" + +clean: + rm -rf $(HTDOCS) + +test: + python test_archivemail + +clobber: clean + rm -rf build dist + rm -f $(HTDOCS).tgz + + +sdist: clobber doc + python setup.py sdist + +tag: + git tag -a $(VERSION_TAG) + +doc: archivemail.1 archivemail.html + +htdocs: $(HTDOCS).tgz +$(HTDOCS).tgz: index.html archivemail.html RELNOTES style.css manpage.css + install -d -m 775 $(HTDOCS) + install -m 664 $^ $(HTDOCS) + cd $(HTDOCS) && mv archivemail.html manpage.html + tar czf $(HTDOCS).tgz $(HTDOCS) + +archivemail.1: archivemail.xml db2man.xsl + xsltproc db2man.xsl archivemail.xml + +archivemail.html: archivemail.xml db2html.xsl + xsltproc --output archivemail.html \ + db2html.xsl archivemail.xml + tidy -modify -indent -f /dev/null archivemail.html || true + +.PHONY: default clean test clobber sdist tag doc htdocs diff --git a/NEWS b/NEWS new file mode 100644 index 0000000..8c84fdd --- /dev/null +++ b/NEWS @@ -0,0 +1,44 @@ +Notable changes in archivemail 0.9.0: + + * IMAP: support for international mailbox names containing non-ASCII + characters. + +Notable changes in archivemail 0.8.0: + + * Removed the feature to setuid to the mailbox owners when run as root. + This was a bad idea; it's really hard to do safely, if at all possible. + Obsoletes: patch #2783134. + * New option --archive-name, or short -a, to hard-code an archive filename. + Like the --suffix and --prefix options, it is expanded with strftime(). + This option conflicts with archiving multiple mailboxes. Closes: feature + request #1306538. + * New option --prefix, or short -p, to specify an archive name prefix. Like + a suffix specified with the --suffix option, the prefix is expanded with + strftime(). Specifying this option disables the default archive name + suffix. Obsoletes: feature request #604281. (Thanks Serafeim Zanikolas + for an initial patch) + * New option --all to archive all messages in a mailbox. Closes: #1764846. + * archivemail now expands wildcards in IMAP mailbox names. For example, the + url imaps://user@server/foo/* will expand to all subfolders of foo. + Closes: feature request #1978540. Obsoletes: patch #1918937. + * To determine the delivery date of a message, archivemail now looks for the + timestamp of the latest 'Received' header before resorting to + 'Resent-Date' or 'Date'. This should give much better results when there + is no 'Delivery-date' header, which is still checked first. + (Thanks Andrew Ferrier & Christian Brabandt) + Closes: #1481316, #1764855, Debian bug #272666. + * We now omit the dotlock if we don't have sufficient permissions to create + it in the mbox directory. (The file is still locked with lockf.) + Together with more changes under the hood (see below) this means + archivemail can now operate on mbox files in the system mail spool. + Closes: #855269. + * Replaced some simple minded file operation security checks with more + decent ones. This means we can safely operate in /tmp, for example. The + price is that we no longer accept symlinked files. Obsoletes: patch + #1874868. + * The archive now also gets locked while archivemail updates it. + * mbox locking got completely rewritten. Switched from flock to lockf + locking, which is NFS-safe and portable, and we now lock with lockf first, + then with a dotlock, instead of the other way around. (This is makes + archivemail compatible with Debian systems. ;) + diff --git a/README b/README new file mode 100644 index 0000000..cbb3fb4 --- /dev/null +++ b/README @@ -0,0 +1,63 @@ + +----------------------------------------------------------- +archivemail - archive and compress old mail in your mailbox +----------------------------------------------------------- + +OVERVIEW: + +archivemail is a tool written in python(1) for archiving and compressing old +email in mailboxes. + +It can move messages older than the specified number of days to a separate +'archive' mbox-format mailbox that is compressed with 'gzip'. + +For example, have you been subscribing to the 'linux-kernel' mailing list +for the last 6 years and ended up with an 160-meg mailbox that 'mutt' is +taking a long time to load? 'archivemail' can move all messages that are +older than 6 months to a separate compressed mailbox, and leave you with +just the most recent messages. + +'archivemail' can save a lot of disk space and will significantly reduce +overhead on your mail reader. The number of days before mail is considered +'old' is up to you, but the default is 180 days. You can also archive messages +by an absolute date or only archive unread messages. + + +REQUIREMENTS: + +archivemail requires python version 2.3. It also uses some optional python +modules, but these should be pretty much standard; if you get an ImportError +nonetheless, please report it, thanks. (For contact addresses see below.) + +Python is available from http://www.python.org/ + + +INSTALLATION: + +If you want to test archivemail: + python test_archivemail + +To install archivemail, run: + python setup.py install + + +USE: + +For more detailed information and examples, look at the archivemail man page. + +The best way to run archivemail is from cron, giving the '-q' option to +archivemail to make it quiet, only printing messages if something went wrong. +Check out the 'examples' directory for an example shell script to be run from +cron. + +The archivemail website is at: http://archivemail.sourceforge.net/ + +If you have any feedback or bug reports about archivemail, you are very +welcome to email the maintainers; as of November 2006, these are: + +Nikolaus Schulz +Peter Poeml + + +-- Paul Rodger , archivemail author + Updated by: Nikolaus Schulz, maintainer diff --git a/TODO b/TODO new file mode 100644 index 0000000..e6950ed --- /dev/null +++ b/TODO @@ -0,0 +1,103 @@ +Integrate --debug-imap option into yet-to-be-implemented -vv switch? +I had the idea to provide separate debugging info levels anyway, see --debug +below. + +Gracefully close IMAP connection upon unexptected error (currently archivemail +just terminates). + +LOCKING & Co: +* Block signals while writing changed mailbox back. +* Double-check the entire locking code. + +Seems like existing archives are not read or validated in any way. New archive +data is blindly appended... Probably okay, but should be documented. + +IMAP SEARCH BEFORE disregards time and timezone information. This should at +least be documented. E.g. I've found that '-d 0' didn't match all messages in +an IMAP mailbox. This is because the SEARCH key is (BEFORE 14-Nov-2007) on 14 +November, not matching messages that arrived today. (This problem is probably +fixed for most use cases by the --all option.) + +Document mbox format issues: link to +http://homepages.tesco.net/~J.deBoynePollard/FGA/mail-mbox-formats.html, +qmail mbox manpage, Debian manpage, RFC 4155. Document what mbox format we can +read, and what we write. +FIXME: we cannot yet parse rfc 2822 addr-spec stuff like quoted local-parts in +return-path addresses. + +Minor annoyance: when a From_ line is generated, guess_delivery_time() reports +the used date header a second time. + +Check sf.net and Debian BTS for new bugs. Again. + +IMAP: ensure mailbox archives are properly named. Currently imap folder names +are mapped like this: + + IMAP URL | resulting mbox_archive + ------------+------------------------ + test.box | test.box_archive.gz + test/box | box_archive.gz + + +Implement --include-draft. But before, think about it again. (This is feature +request #1569305.) + +Implement a fallback if an IMAP server doesn't support SEARCH. (Ouch!) + +Add IMAP tests to the testsuite (upload test messages with IMAP "APPEND +date-string"). This should be done without any real network I/O. + +Try to port archivemail to email.message and the new mailboxes in Python 2.5. +Is these flexible enough for our needs? + +Line out what we want with respect to multiple selection criteria. +Some make sense, but this easily gets too complex, and if only it's a hassle +with adding all the options. Hm. + +Reject patch #1036022 "Added option to inverse date compare" after cooling down +because the patch is both stupid (copy+paste code) and broken. Don't see why +anyone should want this/we should support it. +If this is reasonable *at all*, I think we'd better go for all the complexity +to honour _two_ cut off dates (see Debian bug "#184124: archivemail: -D and -d +should not be incompatible", which is a comparably half-baken thought). + +Add --debug or -vv switch, and move the printing of diagnostic info for each +message to --debug. + +Perhaps add some more nice stuff like printing of subject, sender... +See tracker #868714 "added stats option to archivemail", which has a point. +Message-Ids are useful for diagnosis, but not very nice to read for humans. + +Be a nicer citizen with respect to mailbox locking. + +Perhaps prune/shorten IMAP mailbox URLs in messages? +They may be quite long and may contain the sensitive password. +Also shows up in the process list... +Perhaps find a clean, lean replacement for all that clutter in the IMAP urls. + +Require --output-dir for IMAP archiving? Otherwise we just drop the archive in +in the current working directory. + +Check all items below, which are from the original author. :-) + +.archivemailrc support + +When you get a file-not-found in the 6th mailbox of 10, it aborts the whole +run. Better to fail gracefully and keep going. + +Add more tests (see top of test_archivemail) + +We need some better checking to see if we are really looking at a valid +mbox-format mailbox. + +Add an option to not cut threads. + +Add MMDF mailbox support + +Add Babyl mailbox support + +Add option to archive depending on mailbox size threshold +- is this a good idea? + +Add option to archive depending on number of messages +- is this a good idea? diff --git a/archivemail b/archivemail new file mode 100755 index 0000000..26b9aca --- /dev/null +++ b/archivemail @@ -0,0 +1,1951 @@ +#! /usr/bin/env python +############################################################################ +# Copyright (C) 2002 Paul Rodger , +# (C) 2006 Peter Poeml , +# (C) 2006-2010 Nikolaus Schulz +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################ +""" +Archive and compress old mail in mbox, MH or maildir-format mailboxes. +Website: http://archivemail.sourceforge.net/ +""" + +# global administrivia +__version__ = "archivemail v0.9.0" +__copyright__ = """\ +Copyright (C) 2002 Paul Rodger + (C) 2006 Peter Poeml , + (C) 2006-2011 Nikolaus Schulz +This is free software; see the source for copying conditions. There is NO +warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.""" + +import sys + +def check_python_version(): + """Abort if we are running on python < v2.3""" + too_old_error = "This program requires python v2.3 or greater. " + \ + "Your version of python is:\n%s""" % sys.version + try: + version = sys.version_info # we might not even have this function! :) + if (version[0] < 2) or (version[0] == 2 and version[1] < 3): + print too_old_error + sys.exit(1) + except AttributeError: + print too_old_error + sys.exit(1) + +# define & run this early +# (IMAP over SSL requires Python >= 2.3) +check_python_version() + +import fcntl +import getopt +import gzip +import mailbox +import os +import pwd +import re +import rfc822 +import shutil +import signal +import stat +import string +import tempfile +import time +import urlparse +import errno +import socket +import locale + +# From_ mangling regex. +from_re = re.compile(r'^From ', re.MULTILINE) +imapsize_re = re.compile(r'^(?P[0-9]+) \(RFC822\.SIZE (?P[0-9]+)\)') + +userencoding = locale.getpreferredencoding() + +############## class definitions ############### + +class ArchivemailException(Exception): + pass +class UserError(ArchivemailException): + pass +class UnexpectedError(ArchivemailException): + pass +class LockUnavailable(ArchivemailException): + pass + +class Stats: + """Class to collect and print statistics about mailbox archival""" + __archived = 0 + __archived_size = 0 + __mailbox_name = None + __archive_name = None + __start_time = 0 + __total = 0 + __total_size = 0 + + def __init__(self, mailbox_name, final_archive_name): + """Constructor for a new set of statistics. + + Arguments: + mailbox_name -- filename/dirname of the original mailbox + final_archive_name -- filename for the final 'mbox' archive, without + compression extension (eg .gz) + + """ + assert mailbox_name + assert final_archive_name + self.__start_time = time.time() + self.__mailbox_name = mailbox_name + self.__archive_name = final_archive_name + ".gz" + + def another_message(self, size): + """Add one to the internal count of total messages processed + and record message size.""" + self.__total = self.__total + 1 + self.__total_size = self.__total_size + size + + def another_archived(self, size): + """Add one to the internal count of messages archived + and record message size.""" + self.__archived = self.__archived + 1 + self.__archived_size = self.__archived_size + size + + def display(self): + """Print statistics about how many messages were archived""" + end_time = time.time() + time_seconds = end_time - self.__start_time + action = "archived" + if options.delete_old_mail: + action = "deleted" + if options.dry_run: + action = "I would have " + action + print "%s:\n %s %d of %d message(s) (%s of %s) in %.1f seconds" % \ + (self.__mailbox_name, action, self.__archived, self.__total, + nice_size_str(self.__archived_size), + nice_size_str(self.__total_size), time_seconds) + + +class StaleFiles: + """Class to keep track of files to be deleted on abnormal exit""" + dotlock_files = [] # dotlock files for source mbox and final archive + temp_mboxes = [] # temporary retain and archive mboxes + temp_dir = None # our tempfile directory container + + def clean(self): + """Delete any temporary files or lockfiles that exist""" + while self.dotlock_files: + dotlock = self.dotlock_files.pop() + vprint("removing stale dotlock file '%s'" % dotlock) + try: + os.remove(dotlock) + except (IOError, OSError): pass + while self.temp_mboxes: + mbox = self.temp_mboxes.pop() + vprint("removing stale temporary mbox '%s'" % mbox) + try: + os.remove(mbox) + except (IOError, OSError): pass + if self.temp_dir: + vprint("removing stale tempfile directory '%s'" % self.temp_dir) + try: + os.rmdir(self.temp_dir) + except OSError, e: + if e.errno == errno.ENOTEMPTY: # Probably a bug + user_warning("cannot remove temporary directory '%s', " + "directory not empty" % self.temp_dir) + except IOError: pass + else: self.temp_dir = None + + + +class Options: + """Class to store runtime options, including defaults""" + archive_prefix = None + archive_suffix = None + archive_default_suffix = "_archive" + archive_name = None + days_old_max = 180 + date_old_max = None + delete_old_mail = False + dry_run = False + filter_append = None + include_flagged = False + locking_attempts = 5 + lockfile_extension = ".lock" + lock_sleep = True + no_compress = False + only_archive_read = False + output_dir = None + pwfile = None + preserve_unread = False + mangle_from = True + quiet = False + read_buffer_size = 8192 + script_name = os.path.basename(sys.argv[0]) + min_size = None + verbose = False + debug_imap = 0 + warn_duplicates = False + copy_old_mail = False + archive_all = False + + def parse_args(self, args, usage): + """Set our runtime options from the command-line arguments. + + Arguments: + args -- this is sys.argv[1:] + usage -- a usage message to display on '--help' or bad arguments + + Returns the remaining command-line arguments that have not yet been + parsed as a string. + + """ + try: + opts, args = getopt.getopt(args, '?D:S:Vd:hno:F:P:qs:p:a:uv', + ["date=", "days=", "delete", "dry-run", "help", + "include-flagged", "no-compress", "output-dir=", + "filter-append=", "pwfile=", "dont-mangle", + "preserve-unread", "quiet", "size=", "suffix=", + "prefix=", "archive-name=", "verbose", + "debug-imap=", "version", "warn-duplicate", + "copy", "all"]) + except getopt.error, msg: + user_error(msg) + + archive_by = None + + for o, a in opts: + if o == '--delete': + if self.copy_old_mail: + user_error("found conflicting options --copy and --delete") + self.delete_old_mail = True + if o == '--include-flagged': + self.include_flagged = True + if o == '--no-compress': + self.no_compress = True + if o == '--warn-duplicate': + self.warn_duplicates = True + if o in ('-D', '--date'): + if archive_by: + user_error("you cannot specify both -d and -D options") + archive_by = "date" + self.date_old_max = self.date_argument(a) + if o in ('-d', '--days'): + if archive_by: + user_error("you cannot specify both -d and -D options") + archive_by = "days" + self.days_old_max = string.atoi(a) + if o in ('-o', '--output-dir'): + self.output_dir = os.path.expanduser(a) + if o in ('-P', '--pwfile'): + self.pwfile = os.path.expanduser(a) + if o in ('-F', '--filter-append'): + self.filter_append = a + if o in ('-h', '-?', '--help'): + print usage + sys.exit(0) + if o in ('-n', '--dry-run'): + self.dry_run = True + if o in ('-q', '--quiet'): + self.quiet = True + if o in ('-s', '--suffix'): + self.archive_suffix = a + if o in ('-p', '--prefix'): + self.archive_prefix = a + if o in ('-a', '--archive-name'): + self.archive_name = os.path.expanduser(a) + if o in ('-S', '--size'): + self.min_size = string.atoi(a) + if o in ('-u', '--preserve-unread'): + self.preserve_unread = True + if o == '--dont-mangle': + self.mangle_from = False + if o in ('-v', '--verbose'): + self.verbose = True + if o == '--debug-imap': + self.debug_imap = int(a) + if o == '--copy': + if self.delete_old_mail: + user_error("found conflicting options --copy and --delete") + self.copy_old_mail = True + if o == '--all': + self.archive_all = True + if o in ('-V', '--version'): + print __version__ + "\n\n" + __copyright__ + sys.exit(0) + return args + + def sanity_check(self, args): + """Complain bitterly about our options now rather than later""" + if self.output_dir: + check_sane_destdir(self.output_dir) + if self.days_old_max < 0: + user_error("--days argument must be positive") + if self.days_old_max >= 10000: + user_error("--days argument must be less than 10000") + if self.min_size is not None and self.min_size < 1: + user_error("--size argument must be greater than zero") + if self.quiet and self.verbose: + user_error("you cannot use both the --quiet and --verbose options") + if self.pwfile: + if not os.path.isfile(self.pwfile): + user_error("pwfile %s does not exist" % self.pwfile) + if self.archive_name and len(args) > 1: + user_error("the --archive-name cannot be used with multiple " \ + "mailboxes") + + def date_argument(self, string): + """Converts a date argument string into seconds since the epoch""" + date_formats = ( + "%Y-%m-%d", # ISO format + "%d %b %Y" , # Internet format + "%d %B %Y" , # Internet format with full month names + ) + time.accept2dyear = False # I'm not going to support 2-digit years + for format in date_formats: + try: + date = time.strptime(string, format) + seconds = time.mktime(date) + return seconds + except (ValueError, OverflowError): + pass + user_error("cannot parse the date argument '%s'\n" + "The date should be in ISO format (eg '2002-04-23'),\n" + "Internet format (eg '23 Apr 2002') or\n" + "Internet format with full month names (eg '23 April 2002')" % + string) + + +class LockableMboxMixin: + """Locking methods for mbox files.""" + + def __init__(self, mbox_file, mbox_file_name): + self.mbox_file = mbox_file + self.mbox_file_name = mbox_file_name + self._locked = False + self._use_dotlock = True + + def lock(self): + """Lock this mbox with both a dotlock and a posix lock.""" + assert not self._locked + attempt = 1 + while True: + try: + self._posix_lock() + self._dotlock_lock() + break + except LockUnavailable, e: + self._posix_unlock() + attempt += 1 + if (attempt > options.locking_attempts): + unexpected_error(str(e)) + vprint("%s - sleeping..." % e) + time.sleep(options.lock_sleep) + except: + self._posix_unlock() + raise + self._locked = True + + def unlock(self): + """Unlock this mbox.""" + assert self._locked + self._dotlock_unlock() + self._posix_unlock() + self._locked = False + + def _posix_lock(self): + """Set an exclusive posix lock on the 'mbox' mailbox""" + vprint("trying to acquire posix lock on file '%s'" % self.mbox_file_name) + try: + fcntl.lockf(self.mbox_file, fcntl.LOCK_EX|fcntl.LOCK_NB) + except IOError, e: + if e.errno in (errno.EAGAIN, errno.EACCES): + raise LockUnavailable("posix lock for '%s' unavailable" % \ + self.mbox_file_name) + else: + raise + vprint("acquired posix lock on file '%s'" % self.mbox_file_name) + + def _posix_unlock(self): + """Unset any posix lock on the 'mbox' mailbox""" + vprint("dropping posix lock on file '%s'" % self.mbox_file_name) + fcntl.lockf(self.mbox_file, fcntl.LOCK_UN) + + def _dotlock_lock(self): + """Create a dotlock file for the 'mbox' mailbox""" + hostname = socket.gethostname() + pid = os.getpid() + box_dir, prelock_prefix = os.path.split(self.mbox_file_name) + prelock_suffix = ".%s.%s%s" % (hostname, pid, options.lockfile_extension) + lock_name = self.mbox_file_name + options.lockfile_extension + vprint("trying to create dotlock file '%s'" % lock_name) + try: + plfd, prelock_name = tempfile.mkstemp(prelock_suffix, prelock_prefix, + dir=box_dir) + except OSError, e: + if e.errno == errno.EACCES: + if not options.quiet: + user_warning("no write permissions: omitting dotlock for '%s'" % \ + self.mbox_file_name) + self._use_dotlock = False + return + raise + try: + try: + os.link(prelock_name, lock_name) + # We've got the lock. + except OSError, e: + if os.fstat(plfd)[stat.ST_NLINK] == 2: + # The Linux man page for open(2) claims that in this + # case we have actually succeeded to create the link, + # and this assumption seems to be folklore. + # So we've got the lock. + pass + elif e.errno == errno.EEXIST: + raise LockUnavailable("Dotlock for '%s' unavailable" % self.mbox_file_name) + else: + raise + _stale.dotlock_files.append(lock_name) + finally: + os.close(plfd) + os.unlink(prelock_name) + vprint("acquired lockfile '%s'" % lock_name) + + def _dotlock_unlock(self): + """Delete the dotlock file for the 'mbox' mailbox.""" + if not self._use_dotlock: + return + lock_name = self.mbox_file_name + options.lockfile_extension + vprint("removing lockfile '%s'" % lock_name) + os.remove(lock_name) + _stale.dotlock_files.remove(lock_name) + + def commit(self): + """Sync the mbox file to disk.""" + self.mbox_file.flush() + os.fsync(self.mbox_file.fileno()) + + def close(self): + """Close the mbox file""" + vprint("closing file '%s'" % self.mbox_file_name) + assert not self._locked + self.mbox_file.close() + + +class Mbox(mailbox.UnixMailbox, LockableMboxMixin): + """A mostly-read-only mbox with locking. The mbox content can only be + modified by overwriting the entire underlying file.""" + + def __init__(self, path): + """Constructor for opening an existing 'mbox' mailbox. + Extends constructor for mailbox.UnixMailbox() + + Named Arguments: + path -- file name of the 'mbox' file to be opened + """ + assert path + fd = safe_open_existing(path) + st = os.fstat(fd) + self.original_atime = st.st_atime + self.original_mtime = st.st_mtime + self.starting_size = st.st_size + self.mbox_file = os.fdopen(fd, "r+") + self.mbox_file_name = path + LockableMboxMixin.__init__(self, self.mbox_file, path) + mailbox.UnixMailbox.__init__(self, self.mbox_file) + + def reset_timestamps(self): + """Set the file timestamps to the original values""" + assert self.original_atime + assert self.original_mtime + assert self.mbox_file_name + os.utime(self.mbox_file_name, (self.original_atime, \ + self.original_mtime)) + + def get_size(self): + """Return the current size of the mbox file on disk""" + return os.path.getsize(self.mbox_file_name) + + def overwrite_with(self, mbox_filename): + """Overwrite the mbox content with the content of the given mbox file.""" + fin = open(mbox_filename, "r") + self.mbox_file.seek(0) + shutil.copyfileobj(fin, self.mbox_file) + self.mbox_file.truncate() + + +class ArchiveMbox(LockableMboxMixin): + """Simple append-only access to the archive mbox. Entirely content-agnostic.""" + + def __init__(self, path): + fd = safe_open(path) + self.mbox_file = os.fdopen(fd, "a") + LockableMboxMixin.__init__(self, self.mbox_file, path) + + def append(self, filename): + """Append the content of the given file to the mbox.""" + assert self._locked + fin = open(filename, "r") + oldsize = os.fstat(self.mbox_file.fileno()).st_size + try: + shutil.copyfileobj(fin, self.mbox_file) + except: + # We can safely abort here without data loss, because + # we have not yet changed the original mailbox + self.mbox_file.truncate(oldsize) + raise + fin.close() + + +class TempMbox: + """A write-only temporary mbox. No locking methods.""" + + def __init__(self, prefix=tempfile.template): + """Creates a temporary mbox file.""" + fd, filename = tempfile.mkstemp(prefix=prefix) + self.mbox_file_name = filename + _stale.temp_mboxes.append(filename) + self.mbox_file = os.fdopen(fd, "w") + # an empty gzip file is not really empty (it contains the gzip header + # and trailer), so we need to track manually if this mbox is empty + self.empty = True + + def write(self, msg): + """Write a rfc822 message object to the 'mbox' mailbox. + If the rfc822 has no Unix 'From_' line, then one is constructed + from other headers in the message. + + Arguments: + msg -- rfc822 message object to be written + + """ + assert msg + assert self.mbox_file + + self.empty = False + vprint("saving message to file '%s'" % self.mbox_file_name) + unix_from = msg.unixfrom + if unix_from: + msg_has_mbox_format = True + else: + msg_has_mbox_format = False + unix_from = make_mbox_from(msg) + self.mbox_file.write(unix_from) + assert msg.headers + self.mbox_file.writelines(msg.headers) + self.mbox_file.write(os.linesep) + + # The following while loop is about twice as fast in + # practice to 'self.mbox_file.writelines(msg.fp.readlines())' + assert options.read_buffer_size > 0 + linebuf = "" + while True: + body = msg.fp.read(options.read_buffer_size) + if (not msg_has_mbox_format) and options.mangle_from: + # Be careful not to break pattern matching + splitindex = body.rfind(os.linesep) + nicebody = linebuf + body[:splitindex] + linebuf = body[splitindex:] + body = from_re.sub('>From ', nicebody) + if not body: + break + self.mbox_file.write(body) + if not msg_has_mbox_format: + self.mbox_file.write(os.linesep) + + def commit(self): + """Sync the mbox file to disk.""" + self.mbox_file.flush() + os.fsync(self.mbox_file.fileno()) + + def close(self): + """Close the mbox file""" + vprint("closing file '%s'" % self.mbox_file_name) + self.mbox_file.close() + + def saveas(self, filename): + """Rename this temporary mbox file to the given name, making it + permanent. Emergency use only.""" + os.rename(self.mbox_file_name, filename) + _stale.temp_mboxes.remove(self.mbox_file_name) + + def remove(self): + """Delete the temporary mbox file.""" + os.remove(self.mbox_file_name) + _stale.temp_mboxes.remove(self.mbox_file_name) + + +class CompressedTempMbox(TempMbox): + """A compressed version of a TempMbox.""" + + def __init__(self, prefix=tempfile.template): + TempMbox.__init__(self, prefix) + self.raw_file = self.mbox_file + self.mbox_file = gzip.GzipFile(mode="a", fileobj=self.mbox_file) + # Workaround that GzipFile.close() isn't idempotent in Python < 2.6 + # (python issue #2959). There is no GzipFile.closed, so we need a + # replacement. + self.gzipfile_closed = False + + def commit(self): + """Finish gzip file and sync it to disk.""" + # This method is currently not used + self.mbox_file.close() # close GzipFile, writing gzip trailer + self.gzipfile_closed = True + self.raw_file.flush() + os.fsync(self.raw_file.fileno()) + + def close(self): + """Close the gzip file.""" + if not self.gzipfile_closed: + self.mbox_file.close() + self.raw_file.close() + + +class IdentityCache: + """Class used to remember Message-IDs and warn if they are seen twice""" + seen_ids = {} + mailbox_name = None + + def __init__(self, mailbox_name): + """Constructor: takes the mailbox name as an argument""" + assert mailbox_name + self.mailbox_name = mailbox_name + + def warn_if_dupe(self, msg): + """Print a warning message if the message has already appeared""" + assert msg + message_id = msg.get('Message-ID') + assert message_id + if self.seen_ids.has_key(message_id): + user_warning("duplicate message id: '%s' in mailbox '%s'" % + (message_id, self.mailbox_name)) + self.seen_ids[message_id] = True + + +# global class instances +options = Options() # the run-time options object +_stale = StaleFiles() # remember what we have to delete on abnormal exit + + +def main(args = sys.argv[1:]): + global _stale + + # this usage message is longer than 24 lines -- bad idea? + usage = """Usage: %s [options] mailbox [mailbox...] +Moves old mail in IMAP, mbox, MH or maildir-format mailboxes to an mbox-format +mailbox compressed with gzip. + +Options are as follows: + -d, --days=NUM archive messages older than NUM days (default: %d) + -D, --date=DATE archive messages older than DATE + -o, --output-dir=DIR directory to store archives (default: same as original) + -P, --pwfile=FILE file to read imap password from (default: None) + -F, --filter-append=STRING append arbitrary string to the IMAP filter string + -p, --prefix=NAME prefix for archive filename (default: none) + -s, --suffix=NAME suffix for archive filename (default: '%s') + -a, --archive-name=NAME specify complete archive filename + -S, --size=NUM only archive messages NUM bytes or larger + -n, --dry-run don't write to anything - just show what would be done + -u, --preserve-unread never archive unread messages + --dont-mangle do not mangle From_ in message bodies + --delete delete rather than archive old mail (use with caution!) + --copy copy rather than archive old mail + --include-flagged messages flagged important can also be archived + --all archive all messages + --no-compress do not compress archives with gzip + --warn-duplicate warn about duplicate Message-IDs in the same mailbox + -v, --verbose report lots of extra debugging information + --debug-imap=NUM set IMAP debugging output level (0 is none) + -q, --quiet quiet mode - print no statistics (suitable for crontab) + -V, --version display version information + -h, --help display this message + +Example: %s linux-kernel + This will move all messages older than %s days to a 'mbox' mailbox called + 'linux-kernel_archive.gz', deleting them from the original 'linux-kernel' + mailbox. If the 'linux-kernel_archive.gz' mailbox already exists, the + newly archived messages are appended. + +To archive IMAP mailboxes, format your mailbox argument like this: + imap://username:password@server/mailbox + (substitute 'imap' with 'imaps' for an SSL connection) + +Website: http://archivemail.sourceforge.net/ """ % \ + (options.script_name, options.days_old_max, options.archive_suffix, + options.script_name, options.days_old_max) + + args = options.parse_args(args, usage) + if len(args) == 0: + print usage + sys.exit(1) + + options.sanity_check(args) + + for mailbox_path in args: + archive(mailbox_path) + + +######## errors and debug ########## + +def vprint(string): + """Print the string argument if we are in verbose mode""" + if options.verbose: + print string + + +def unexpected_error(string): + """Print the string argument, a 'shutting down' message and abort. Raise + UnexpectedErrors if archivemail is run as a module. This function never + returns.""" + if not __name__ == '__main__': + raise UnexpectedError(string) + sys.stderr.write("%s: %s\n" % (options.script_name, string)) + sys.stderr.write("%s: unexpected error encountered - shutting down\n" % + options.script_name) + sys.exit(1) + + +def user_error(string): + """Print the string argument and abort. Raise UserError if archivemail is + run as a module. This function never returns.""" + if not __name__ == '__main__': + raise UserError(string) + sys.stderr.write("%s: %s\n" % (options.script_name, string)) + sys.exit(1) + + +def user_warning(string): + """Print the string argument""" + sys.stderr.write("%s: Warning - %s\n" % (options.script_name, string)) + +########### operations on a message ############ + +def make_mbox_from(message): + """Return a string suitable for use as a 'From_' mbox header for the + message. + + Arguments: + message -- the rfc822 message object + + """ + assert message + address = guess_return_path(message) + time_message = guess_delivery_time(message) + date = time.localtime(time_message) + assert date + date_string = time.asctime(date) + mbox_from = "From %s %s\n" % (address, date_string) + return mbox_from + + +def guess_return_path(message): + """Return a guess at the Return Path address of an rfc822 message""" + assert message + + for header in ('Return-path', 'From'): + address_header = message.get(header) + if address_header: + (name, address) = rfc822.parseaddr(address_header) + if address: + return address + # argh, we can't find any valid 'Return-path' guesses - just + # just use the current unix username like mutt does + login = pwd.getpwuid(os.getuid())[0] + assert login + return login + + +def guess_delivery_time(message): + """Return a guess at the delivery date of an rfc822 message""" + assert message + # try to guess the delivery date from various headers + # get more desparate as we go through the array + for header in 'Delivery-date', 'Received', 'Resent-Date', 'Date': + try: + if header == 'Received': + # This should be good enough for almost all headers in the wild; + # if we're guessing wrong, parsedate_tz() will fail graciously. + token = message.getrawheader(header).rsplit(';', 1)[-1] + else: + token = message.get(header) + date = rfc822.parsedate_tz(token) + if date: + time_message = rfc822.mktime_tz(date) + vprint("using valid time found from '%s' header" % header) + return time_message + except (AttributeError, IndexError, ValueError, OverflowError): pass + # as a second-last resort, try the date from the 'From_' line (ugly) + # this will only work from a mbox-format mailbox + if (message.unixfrom): + # Hmm. This will break with full-blown RFC 2822 addr-spec's. + header = message.unixfrom.split(None, 2)[-1] + # Interpret no timezone as localtime + date = rfc822.parsedate_tz(header) + if date: + try: + time_message = rfc822.mktime_tz(date) + vprint("using valid time found from unix 'From_' header") + return time_message + except (ValueError, OverflowError): pass + # the headers have no valid dates -- last resort, try the file timestamp + # this will not work for mbox mailboxes + try: + file_name = get_filename(message) + except AttributeError: + # we are looking at a 'mbox' mailbox - argh! + # Just return the current time - this will never get archived :( + vprint("no valid times found at all -- using current time!") + return time.time() + if not os.path.isfile(file_name): + unexpected_error("mailbox file name '%s' has gone missing" % \ + file_name) + time_message = os.path.getmtime(file_name) + vprint("using valid time found from '%s' last-modification time" % \ + file_name) + return time_message + + +def add_status_headers(message): + """ + Add Status and X-Status headers to a message from a maildir mailbox. + + Maildir messages store their information about being read/replied/etc in + the suffix of the filename rather than in Status and X-Status headers in + the message. In order to archive maildir messages into mbox format, it is + nice to preserve this information by putting it into the status headers. + + """ + status = "" + x_status = "" + file_name = get_filename(message) + match = re.search(":2,(.+)$", file_name) + if match: + flags = match.group(1) + for flag in flags: + if flag == "D": # (draft): the user considers this message a draft + pass # does this make any sense in mbox? + elif flag == "F": # (flagged): user-defined 'important' flag + x_status = x_status + "F" + elif flag == "R": # (replied): the user has replied to this message + x_status = x_status + "A" + elif flag == "S": # (seen): the user has viewed this message + status = status + "R" + elif flag == "T": # (trashed): user has moved this message to trash + pass # is this Status: D ? + else: + pass # no whingeing here, although it could be a good experiment + + # files in the maildir 'cur' directory are no longer new, + # they are the same as messages with 'Status: O' headers in mbox + last_dir = os.path.basename(os.path.dirname(file_name)) + if last_dir == "cur": + status = status + "O" + + # Overwrite existing 'Status' and 'X-Status' headers. They add no value in + # maildirs, and we better don't listen to them. + if status: + vprint("converting maildir status into Status header '%s'" % status) + message['Status'] = status + else: + del message['Status'] + if x_status: + vprint("converting maildir status into X-Status header '%s'" % x_status) + message['X-Status'] = x_status + else: + del message['X-Status'] + +def add_status_headers_imap(message, flags): + """Add Status and X-Status headers to a message from an imap mailbox.""" + status = "" + x_status = "" + for flag in flags: + if flag == "\\Draft": # (draft): the user considers this message a draft + pass # does this make any sense in mbox? + elif flag == "\\Flagged": # (flagged): user-defined 'important' flag + x_status = x_status + "F" + elif flag == "\\Answered": # (replied): the user has replied to this message + x_status = x_status + "A" + elif flag == "\\Seen": # (seen): the user has viewed this message + status = status + "R" + elif flag == "\\Deleted": # (trashed): user has moved this message to trash + pass # is this Status: D ? + else: + pass # no whingeing here, although it could be a good experiment + if not "\\Recent" in flags: + status = status + "O" + + # As with maildir folders, overwrite Status and X-Status headers + # if they exist. + vprint("converting imap status (%s)..." % " ".join(flags)) + if status: + vprint("generating Status header '%s'" % status) + message['Status'] = status + else: + vprint("not generating Status header") + del message['Status'] + if x_status: + vprint("generating X-Status header '%s'" % x_status) + message['X-Status'] = x_status + else: + vprint("not generating X-Status header") + del message['X-Status'] + +def is_flagged(message): + """return true if the message is flagged important, false otherwise""" + # MH and mbox mailboxes use the 'X-Status' header to indicate importance + x_status = message.get('X-Status') + if x_status and re.search('F', x_status): + vprint("message is important (X-Status header='%s')" % x_status) + return True + file_name = None + try: + file_name = get_filename(message) + except AttributeError: + pass + # maildir mailboxes use the filename suffix to indicate flagged status + if file_name and re.search(":2,.*F.*$", file_name): + vprint("message is important (filename info has 'F')") + return True + vprint("message is not flagged important") + return False + + +def is_unread(message): + """return true if the message is unread, false otherwise""" + # MH and mbox mailboxes use the 'Status' header to indicate read status + status = message.get('Status') + if status and re.search('R', status): + vprint("message has been read (status header='%s')" % status) + return False + file_name = None + try: + file_name = get_filename(message) + except AttributeError: + pass + # maildir mailboxes use the filename suffix to indicate read status + if file_name and re.search(":2,.*S.*$", file_name): + vprint("message has been read (filename info has 'S')") + return False + vprint("message is unread") + return True + + +def sizeof_message(message): + """Return size of message in bytes (octets).""" + assert message + file_name = None + message_size = None + try: + file_name = get_filename(message) + except AttributeError: + pass + if file_name: + # with maildir and MH mailboxes, we can just use the file size + message_size = os.path.getsize(file_name) + else: + # with mbox mailboxes, not so easy + message_size = 0 + if message.unixfrom: + message_size = message_size + len(message.unixfrom) + for header in message.headers: + message_size = message_size + len(header) + message_size = message_size + 1 # the blank line after the headers + start_offset = message.fp.tell() + message.fp.seek(0, 2) # seek to the end of the message + end_offset = message.fp.tell() + message.rewindbody() + message_size = message_size + (end_offset - start_offset) + return message_size + +def is_smaller(message, size): + """Return true if the message is smaller than size bytes, false otherwise""" + assert message + assert size > 0 + message_size = sizeof_message(message) + if message_size < size: + vprint("message is too small (%d bytes), minimum bytes : %d" % \ + (message_size, size)) + return True + else: + vprint("message is not too small (%d bytes), minimum bytes: %d" % \ + (message_size, size)) + return False + + +def should_archive(message): + """Return true if we should archive the message, false otherwise""" + if options.archive_all: + return True + old = False + time_message = guess_delivery_time(message) + if options.date_old_max == None: + old = is_older_than_days(time_message, options.days_old_max) + else: + old = is_older_than_time(time_message, options.date_old_max) + + # I could probably do this in one if statement, but then I wouldn't + # understand it. + if not old: + return False + if not options.include_flagged and is_flagged(message): + return False + if options.min_size and is_smaller(message, options.min_size): + return False + if options.preserve_unread and is_unread(message): + return False + return True + + +def is_older_than_time(time_message, max_time): + """Return true if a message is older than the specified time, + false otherwise. + + Arguments: + time_message -- the delivery date of the message measured in seconds + since the epoch + max_time -- maximum time allowed for message + + """ + days_old = (max_time - time_message) / 24 / 60 / 60 + if time_message < max_time: + vprint("message is %.2f days older than the specified date" % days_old) + return True + vprint("message is %.2f days younger than the specified date" % \ + abs(days_old)) + return False + + +def is_older_than_days(time_message, max_days): + """Return true if a message is older than the specified number of days, + false otherwise. + + Arguments: + time_message -- the delivery date of the message measured in seconds + since the epoch + max_days -- maximum number of days before message is considered old + """ + time_now = time.time() + if time_message > time_now: + vprint("warning: message has date in the future") + return False + secs_old_max = (max_days * 24 * 60 * 60) + days_old = (time_now - time_message) / 24 / 60 / 60 + vprint("message is %.2f days old" % days_old) + if ((time_message + secs_old_max) < time_now): + return True + return False + +def build_imap_filter(): + """Return an imap filter string""" + + imap_filter = [] + if options.date_old_max == None: + time_now = time.time() + secs_old_max = (options.days_old_max * 24 * 60 * 60) + time_old = time.gmtime(time_now - secs_old_max) + else: + time_old = time.gmtime(options.date_old_max) + time_str = time.strftime('%d-%b-%Y', time_old) + imap_filter.append("BEFORE %s" % time_str) + + if not options.include_flagged: + imap_filter.append("UNFLAGGED") + if options.min_size: + imap_filter.append("LARGER %d" % options.min_size) + if options.preserve_unread: + imap_filter.append("SEEN") + if options.filter_append: + imap_filter.append(options.filter_append) + + return '(' + string.join(imap_filter, ' ') + ')' + +############### mailbox operations ############### + +def archive(mailbox_name): + """Archives a mailbox. + + Arguments: + mailbox_name -- the filename/dirname/url of the mailbox to be archived + """ + assert mailbox_name + + # strip any trailing slash (we could be archiving a maildir or MH format + # mailbox and somebody was pressing in bash) - we don't want to use + # the trailing slash in the archive name + mailbox_name = mailbox_name.rstrip("/") + assert mailbox_name + + set_signal_handlers() + os.umask(077) # saves setting permissions on mailboxes/tempfiles + + vprint("processing '%s'" % mailbox_name) + is_imap = urlparse.urlparse(mailbox_name)[0] in ('imap', 'imaps') + if not is_imap: + # Check if the mailbox exists, and refuse to mess with other people's + # stuff + try: + fuid = os.stat(mailbox_name).st_uid + except OSError, e: + user_error(str(e)) + else: + if fuid != os.getuid(): + user_error("'%s' is owned by someone else!" % mailbox_name) + + old_temp_dir = tempfile.tempdir + try: + # create a temporary directory for us to work in securely + tempfile.tempdir = None + new_temp_dir = tempfile.mkdtemp('archivemail') + assert new_temp_dir + _stale.temp_dir = new_temp_dir + tempfile.tempdir = new_temp_dir + vprint("set tempfile directory to '%s'" % new_temp_dir) + + if is_imap: + vprint("guessing mailbox is of type: imap(s)") + _archive_imap(mailbox_name) + elif os.path.isfile(mailbox_name): + vprint("guessing mailbox is of type: mbox") + _archive_mbox(mailbox_name) + elif os.path.isdir(mailbox_name): + cur_path = os.path.join(mailbox_name, "cur") + new_path = os.path.join(mailbox_name, "new") + if os.path.isdir(cur_path) and os.path.isdir(new_path): + vprint("guessing mailbox is of type: maildir") + _archive_dir(mailbox_name, "maildir") + else: + vprint("guessing mailbox is of type: MH") + _archive_dir(mailbox_name, "mh") + else: + user_error("'%s' is not a normal file or directory" % mailbox_name) + + # remove our special temp directory - hopefully empty + os.rmdir(new_temp_dir) + _stale.temp_dir = None + + finally: + tempfile.tempdir = old_temp_dir + clean_up() + +def _archive_mbox(mailbox_name): + """Archive a 'mbox' style mailbox - used by archive_mailbox()""" + assert mailbox_name + final_archive_name = make_archive_name(mailbox_name) + vprint("archiving '%s' to '%s' ..." % (mailbox_name, final_archive_name)) + check_archive(final_archive_name) + stats = Stats(mailbox_name, final_archive_name) + cache = IdentityCache(mailbox_name) + original = Mbox(path=mailbox_name) + if options.dry_run or options.copy_old_mail: + retain = None + else: + retain = TempMbox(prefix="retain") + archive = prepare_temp_archive() + + original.lock() + msg = original.next() + if not msg and (original.starting_size > 0): + user_error("'%s' is not a valid mbox-format mailbox" % mailbox_name) + if msg and 'X-IMAP' in msg: + # Dovecot and UW-IMAP pseudo message for mailbox meta data + vprint("detected IMAP pseudo message") + if retain: + retain.write(msg) + msg = original.next() + while (msg): + msg_size = sizeof_message(msg) + stats.another_message(msg_size) + vprint("processing message '%s'" % msg.get('Message-ID')) + if options.warn_duplicates: + cache.warn_if_dupe(msg) + if should_archive(msg): + stats.another_archived(msg_size) + if options.delete_old_mail: + vprint("decision: delete message") + else: + vprint("decision: archive message") + if archive: + archive.write(msg) + else: + vprint("decision: retain message") + if retain: + retain.write(msg) + msg = original.next() + vprint("finished reading messages") + if original.starting_size != original.get_size(): + unexpected_error("the mailbox '%s' changed size during reading!" % \ + mailbox_name) + # Write the new archive before modifying the mailbox, to prevent + # losing data if something goes wrong + commit_archive(archive, final_archive_name) + if retain: + pending_changes = original.mbox_file.tell() != retain.mbox_file.tell() + if pending_changes: + retain.commit() + retain.close() + vprint("writing back changed mailbox '%s'..." % \ + original.mbox_file_name) + # Prepare for recovery on error. + # FIXME: tempfile.tempdir is our nested dir. + saved_name = "%s/%s.%s.%s-%s-%s" % \ + (tempfile.tempdir, options.script_name, + os.path.basename(original.mbox_file_name), + socket.gethostname(), os.getuid(), + os.getpid()) + try: + original.overwrite_with(retain.mbox_file_name) + original.commit() + except: + retain.saveas(saved_name) + print "Error writing back changed mailbox; saved good copy to " \ + "%s" % saved_name + raise + else: + retain.close() + vprint("no changes to mbox '%s'" % original.mbox_file_name) + retain.remove() + original.unlock() + original.close() + original.reset_timestamps() # Minor race here; mutt has this too. + if not options.quiet: + stats.display() + + +def _archive_dir(mailbox_name, type): + """Archive a 'maildir' or 'MH' style mailbox - used by archive_mailbox()""" + assert mailbox_name + assert type + final_archive_name = make_archive_name(mailbox_name) + vprint("archiving '%s' to '%s' ..." % (mailbox_name, final_archive_name)) + check_archive(final_archive_name) + stats = Stats(mailbox_name, final_archive_name) + delete_queue = [] + + if type == "maildir": + original = mailbox.Maildir(mailbox_name) + elif type == "mh": + original = mailbox.MHMailbox(mailbox_name) + else: + unexpected_error("unknown type: %s" % type) + cache = IdentityCache(mailbox_name) + archive = prepare_temp_archive() + + for msg in original: + if not msg: + vprint("ignoring invalid message '%s'" % get_filename(msg)) + continue + msg_size = sizeof_message(msg) + stats.another_message(msg_size) + vprint("processing message '%s'" % msg.get('Message-ID')) + if options.warn_duplicates: + cache.warn_if_dupe(msg) + if should_archive(msg): + stats.another_archived(msg_size) + if options.delete_old_mail: + vprint("decision: delete message") + else: + vprint("decision: archive message") + if archive: + if type == "maildir": + add_status_headers(msg) + archive.write(msg) + if not options.dry_run and not options.copy_old_mail: + delete_queue.append(get_filename(msg)) + else: + vprint("decision: retain message") + vprint("finished reading messages") + # Write the new archive before modifying the mailbox, to prevent + # losing data if something goes wrong + commit_archive(archive, final_archive_name) + for file_name in delete_queue: + vprint("removing original message: '%s'" % file_name) + try: os.remove(file_name) + except OSError, e: + if e.errno != errno.ENOENT: raise + if not options.quiet: + stats.display() + +def _archive_imap(mailbox_name): + """Archive an imap mailbox - used by archive_mailbox()""" + assert mailbox_name + import imaplib + import cStringIO + import getpass + + vprint("Setting imaplib.Debug = %d" % options.debug_imap) + imaplib.Debug = options.debug_imap + archive = None + imap_username, imap_password, \ + imap_server, imap_server_port, \ + imap_folder_pattern = parse_imap_url(mailbox_name) + if not imap_password: + if options.pwfile: + imap_password = open(options.pwfile).read().rstrip() + else: + if (not os.isatty(sys.stdin.fileno())) or options.quiet: + unexpected_error("No imap password specified") + imap_password = getpass.getpass('IMAP password: ') + + is_ssl = mailbox_name[:5].lower() == 'imaps' + if is_ssl: + vprint("establishing secure connection to server %s, port %s" % + (imap_server, imap_server_port)) + imap_srv = imaplib.IMAP4_SSL(imap_server, imap_server_port) + else: + vprint("establishing connection to server %s, port %s" % + (imap_server, imap_server_port)) + imap_srv = imaplib.IMAP4(imap_server, imap_server_port) + if "AUTH=CRAM-MD5" in imap_srv.capabilities: + vprint("authenticating (cram-md5) to server as %s" % imap_username) + result, response = imap_srv.login_cram_md5(imap_username, imap_password) + elif not "LOGINDISABLED" in imap_srv.capabilities: + vprint("logging in to server as %s" % imap_username) + result, response = imap_srv.login(imap_username, imap_password) + else: + user_error("imap server %s has login disabled (hint: " + "try ssl/imaps)" % imap_server) + + mailboxes = imap_find_mailboxes(imap_srv, imap_folder_pattern) + for imap_folder in mailboxes: + final_archive_name = make_archive_name(imap_folder) + vprint("archiving mailbox '%s' on IMAP server '%s' to '%s' ..." % + (imap_folder, imap_server, final_archive_name)) + check_archive(final_archive_name) + cur_mailbox = mailbox_name[:-len(imap_folder_pattern)] + imap_folder + stats = Stats(cur_mailbox, final_archive_name) + cache = IdentityCache(cur_mailbox) + + imap_smart_select(imap_srv, imap_folder) + total_msg_count = int(imap_srv.response("EXISTS")[1][0]) + vprint("folder has %d message(s)" % total_msg_count) + + # IIUIC the message sequence numbers are stable for the whole session, since + # we just send SEARCH, FETCH and STORE commands, which should prevent the + # server from sending untagged EXPUNGE responses -- see RFC 3501 (IMAP4rev1) + # 7.4.1 and RFC 2180 (Multi-Accessed Mailbox Practice). + # Worst thing should be that we bail out FETCHing a message that has been + # deleted. + + if options.archive_all: + message_list = [str(n) for n in range(1, total_msg_count+1)] + else: + imap_filter = build_imap_filter() + vprint("imap filter: '%s'" % imap_filter) + vprint("searching messages matching criteria") + result, response = imap_srv.search(None, imap_filter) + if result != 'OK': unexpected_error("imap search failed; server says '%s'" % + response[0]) + if response[0] is not None: + # response is a list with a single item, listing message + # sequence numbers like ['1 2 3 1016'] + message_list = response[0].split() + else: + # Broken server has sent no untagged response; assume empty result set. + message_list = [] + vprint("%d messages are matching filter" % len(message_list)) + + # First, gather data for the statistics. + if total_msg_count > 0 and not options.quiet: + vprint("fetching size of messages...") + result, response = imap_srv.fetch('1:*', '(RFC822.SIZE)') + if result != 'OK': unexpected_error("Failed to fetch message sizes; " + "server says '%s'" % response[0]) + # response is a list with entries like '1016 (RFC822.SIZE 3118)', + # where the first number is the message sequence number, the second is + # the size. + for x in response: + m = imapsize_re.match(x) + msn, msg_size = m.group('msn'), int(m.group('size')) + stats.another_message(msg_size) + if msn in message_list: + stats.another_archived(msg_size) + + if not options.dry_run: + if not options.delete_old_mail: + archive = prepare_temp_archive() + vprint("fetching messages...") + for msn in message_list: + # Fetching message flags and body together always finds \Seen + # set. To check \Seen, we must fetch the flags first. + result, response = imap_srv.fetch(msn, '(FLAGS)') + if result != 'OK': unexpected_error("Failed to fetch message " + "flags; server says '%s'" % response[0]) + msg_flags = imaplib.ParseFlags(response[0]) + result, response = imap_srv.fetch(msn, '(RFC822)') + if result != 'OK': unexpected_error("Failed to fetch message; " + "server says '%s'" % response[0]) + msg_str = response[0][1].replace("\r\n", os.linesep) + msg = rfc822.Message(cStringIO.StringIO(msg_str)) + vprint("processing message '%s'" % msg.get('Message-ID')) + add_status_headers_imap(msg, msg_flags) + if options.warn_duplicates: + cache.warn_if_dupe(msg) + archive.write(msg) + commit_archive(archive, final_archive_name) + if not options.copy_old_mail: + vprint("Deleting %s messages" % len(message_list)) + # do not delete more than a certain number of messages at a time, + # because the command length is limited. This avoids that servers + # terminate the connection with EOF or TCP RST. + max_delete = 100 + for i in range(0, len(message_list), max_delete): + result, response = imap_srv.store( \ + string.join(message_list[i:i+max_delete], ','), + '+FLAGS.SILENT', '\\Deleted') + if result != 'OK': unexpected_error("Error while deleting " + "messages; server says '%s'" % response[0]) + vprint("Closing mailbox.") + imap_srv.close() + if not options.quiet: + stats.display() + vprint("Terminating connection.") + imap_srv.logout() + + +############### IMAP functions ############### + + +# First, some IMAP modified UTF-7 support functions. + +# The modified BASE64 alphabet. 64 characters, each one encodes 6 Bit. +mb64alpha = string.ascii_uppercase + string.ascii_lowercase + string.digits + '+,' + +def isprint_ascii(char): + """Test for an ASCII printable character.""" + return 0x20 <= ord(char) and ord(char) <= 0x7e + +def mod_utf7_encode(ustr): + """Encode unicode string object in modified UTF-7.""" + + def mb64_encode(tomb64): + """Encode unicode string object as a modified UTF-7 shifted sequence + in modified BASE64.""" + u16be = tomb64.encode('utf_16_be') + mb64 = "" + # Process 24-bit blocks, encoding them in 6-bit steps. + for block in [u16be[i:i+3] for i in range(0, len(u16be), 3)]: + idx = 0 + shift = 2 + for octet in block: + mb64 += mb64alpha[idx | (ord(octet) >> shift)] + idx = (ord(octet) << (6-shift)) & 0x3f + shift += 2 + mb64 += mb64alpha[idx] + return mb64 + + mu7 = "" + tomb64 = u"" + for c in ustr: + if not isprint_ascii(c): + tomb64 += c + continue + if tomb64: + mu7 += '&' + mb64_encode(tomb64) + '-' + tomb64 = u"" + if c == '&': + mu7 += '&-' + else: + mu7 += str(c) + if tomb64: + mu7 += '&' + mb64_encode(tomb64) + '-' + return mu7 + +def mod_utf7_decode(mu7): + """Decode a modified UTF-7 encoded string to an unicode string object.""" + + def mb64_decode(mb64): + """Decode a modified UTF-7 shifted sequence from modified BASE64 to an + unicode string object.""" + if not mb64: + # A null shift '&-' decodes to '&'. + return u"&" + u16be = "" + # Process blocks of 4 BASE64 characters, decoding each char to 6 bits. + for block in [mb64[i:i+4] for i in range(0, len(mb64), 4)]: + carrybits = mb64alpha.index(block[0]) << 2 + shift = 4 + for char in block[1:]: + bits = mb64alpha.index(char) + u16be += chr(carrybits | (bits >> shift)) + carrybits = (bits << (8-shift)) & 0xff + shift -= 2 + if carrybits: + raise ValueError("Ill-formed modified UTF-7 string: " + "trailing bits in shifted sequence") + return u16be.decode('utf_16_be') + + ustr = u"" + mb64 = "" + inmb64 = False + for octet in mu7: + if not isprint_ascii(octet): + raise ValueError("Ill-formed modified UTF-7 string: " + "contains non-printable ASCII" % ord(octet)) + if not inmb64: + if octet == '&': + inmb64 = True + else: + ustr += octet + continue + + if octet in mb64alpha: + mb64 += octet + continue + + if octet == '-': + inmb64 = False + ustr += mb64_decode(mb64) + mb64 = "" + else: + break # This triggers the exception below. + + if inmb64: + raise ValueError("Ill-formed modified UTF-7 string: " + "unterminated BASE64 sequence") + return ustr + + +def imap_quote(astring): + """Quote an IMAP `astring' string (see RFC 3501, section "Formal Syntax").""" + if astring.startswith('"') and astring.endswith('"'): + quoted = astring + else: + quoted = '"' + astring.replace('\\', '\\\\').replace('"', '\\"') + '"' + return quoted + +def imap_unquote(quoted): + """Un-quote a `quoted' IMAP string (see RFC 3501, section "Formal Syntax").""" + if not (quoted.startswith('"') and quoted.endswith('"')): + unquoted = quoted + else: + unquoted = re.sub(r'\\(\\|")', r'\1', quoted[1:-1]) + return unquoted + +def parse_imap_url(url): + """Parse IMAP URL and return username, password (if appliciable), servername + and foldername.""" + + def split_qstr(string, delim): + """Split string once at delim, keeping quoted substring intact. + Strip and unescape quotes where necessary.""" + rm = re.match(r'"(.+?(?"."|NIL)', response[0]) + if not m: + unexpected_error("imap_getdelim(): cannot parse '%s'" % response[0]) + delim = m.group('delim').strip('"') + vprint("Found mailbox hierarchy delimiter: '%s'" % delim) + if delim == "NIL": + return None + return delim + + +def imap_get_namespace(srv): + """Return the IMAP namespace prefixes and hierarchy delimiters.""" + assert 'NAMESPACE' in srv.capabilities + result, response = srv.namespace() + if result != 'OK': + unexpected_error("Cannot retrieve IMAP namespace; server says: '%s'" + % response[0]) + vprint("NAMESPACE response: %s" % repr(response[0])) + # Typical response is e.g. + # ['(("INBOX." ".")) NIL (("#shared." ".")("shared." "."))'] or + # ['(("" ".")) NIL NIL'], see RFC 2342. + # Make a reasonable guess parsing this beast. + try: + m = re.match(r'\(\("([^"]*)" (?:"(.)"|NIL)', response[0]) + nsprefix, hdelim = m.groups() + except: + print "Cannot parse IMAP NAMESPACE response %s" % repr(response) + raise + return nsprefix, hdelim + + +def imap_smart_select(srv, mailbox): + """Select the given mailbox on the IMAP server.""" + roflag = options.dry_run or options.copy_old_mail + # Work around python bug #1277098 (still pending in python << 2.5) + if not roflag: + roflag = None + if roflag: + vprint("examining imap folder '%s' read-only" % mailbox) + else: + vprint("selecting imap folder '%s'" % mailbox) + imap_mailbox = mod_utf7_encode(mailbox.decode(userencoding)) + result, response = srv.select(imap_quote(imap_mailbox), roflag) + if result != 'OK': + unexpected_error("selecting '%s' failed; server says: '%s'." \ + % (mailbox, response[0])) + if not roflag: + # Sanity check that we don't silently fail to delete messages. + # As to the following indices: IMAP4.response(key) returns + # a tuple (key, ['']) if the key is found, (key, [None]) + # otherwise. Imaplib just *loves* to nest trivial lists! + permflags = srv.response("PERMANENTFLAGS")[1][0] + if permflags: + permflags = permflags.strip('()').lower().split() + if not '\\deleted' in permflags: + unexpected_error("Server doesn't allow deleting messages in " \ + "'%s'." % mailbox) + elif "IMAP4REV1" in srv.capabilities: + vprint("Suspect IMAP4rev1 server, doesn't send PERMANENTFLAGS " \ + "upon SELECT") + + +def imap_find_mailboxes(srv, mailbox): + """Find matching mailboxes on the IMAP server, correcting an invalid + mailbox path if possible.""" + for curbox in imap_guess_mailboxnames(srv, mailbox): + if '%' in curbox or '*' in curbox: + vprint("Looking for mailboxes matching '%s'..." % curbox) + else: + vprint("Looking for mailbox '%s'..." % curbox) + curbox = mod_utf7_encode(curbox.decode(userencoding)) + result, response = srv.list(pattern=imap_quote(curbox)) + if result != 'OK': + unexpected_error("LIST command failed; " \ + "server says: '%s'" % response[0]) + # Say we queried for the mailbox "foo". + # Upon success, response is e.g. ['(\\HasChildren) "." foo']. + # Upon failure, response is [None]. Funky imaplib! + if response[0] != None: + break + else: + user_error("Cannot find mailbox '%s' on server." % mailbox) + mailboxes = [] + for mailbox_data in response: + if not mailbox_data: # imaplib sometimes returns an empty string + continue + try: + m = re.match(r'\((.*?)\) (?:"."|NIL) (.+)', mailbox_data) + except TypeError: + # May be a literal. For literals, imaplib returns a tuple like + # ('(\\HasNoChildren) "." {12}', 'with "quote"'). + m = re.match(r'\((.*?)\) (?:"."|NIL) \{\d+\}$', mailbox_data[0]) + if m is None: + unexpected_error("cannot parse LIST reply %s" % + (mailbox_data,)) + attrs = m.group(1) + name = mailbox_data[1] + else: + attrs, name = m.groups() + name = imap_unquote(name) + try: + name = mod_utf7_decode(name) + except ValueError: + vprint("Mailbox name '%s' returned by server doesn't look like " + "modified UTF-7" % name) + name = name.decode('utf-8') + name = name.encode(userencoding) + if '\\noselect' in attrs.lower().split(): + vprint("skipping not selectable mailbox '%s'" % name) + continue + vprint("Found mailbox '%s'" % name) + mailboxes.append(name) + if not mailboxes: + user_error("No matching folder is selectable") + return mailboxes + + +def imap_guess_mailboxnames(srv, mailbox): + """Return a list of possible real IMAP mailbox names in descending order + of preference, compiled by prepending an IMAP namespace prefix if necessary, + and by translating hierarchy delimiters.""" + if 'NAMESPACE' in srv.capabilities: + nsprefix, hdelim = imap_get_namespace(srv) + else: + vprint("Server doesn't support NAMESPACE command.") + nsprefix = "" + hdelim = imap_getdelim(srv) + vprint("IMAP namespace prefix: '%s', hierarchy delimiter: '%s'" % \ + (nsprefix, hdelim)) + if mailbox.upper() == "INBOX" or \ + (hdelim is not None and mailbox.upper().startswith("INBOX" + hdelim)): + # INBOX is not a real mailbox name, so namespace prefixes do not apply + # to INBOX and its children + boxnames = [mailbox] + elif mailbox.startswith(nsprefix): + boxnames = [mailbox] + else: + boxnames = [nsprefix + mailbox] + if os.path.sep in mailbox and hdelim is not None: + mailbox = mailbox.replace(os.path.sep, hdelim) + if mailbox.upper().startswith("INBOX" + hdelim): + boxnames.append(mailbox) + else: + if mailbox.startswith(nsprefix): + boxnames.append(mailbox) + if nsprefix: + boxnames.append(nsprefix + mailbox) + return boxnames + + +############### misc functions ############### + + +def set_signal_handlers(): + """set signal handlers to clean up temporary files on unexpected exit""" + # Make sure we clean up nicely - we don't want to leave stale dotlock + # files about if something bad happens to us. This is quite + # important, even though procmail will delete stale files after a while. + signal.signal(signal.SIGHUP, clean_up_signal) # signal 1 + # SIGINT (signal 2) is handled as a python exception + signal.signal(signal.SIGQUIT, clean_up_signal) # signal 3 + signal.signal(signal.SIGTERM, clean_up_signal) # signal 15 + + +def clean_up(): + """Delete stale files""" + vprint("cleaning up ...") + _stale.clean() + + +def clean_up_signal(signal_number, stack_frame): + """Delete stale files -- to be registered as a signal handler. + + Arguments: + signal_number -- signal number of the terminating signal + stack_frame -- the current stack frame + + """ + # this will run the above clean_up(), since unexpected_error() + # will abort with sys.exit() and clean_up will be registered + # at this stage + unexpected_error("received signal %s" % signal_number) + +def prepare_temp_archive(): + """Create temporary archive mbox.""" + if options.dry_run or options.delete_old_mail: + return None + if options.no_compress: + return TempMbox() + else: + return CompressedTempMbox() + +def commit_archive(archive, final_archive_name): + """Finalize temporary archive and write it to its final destination.""" + if not options.no_compress: + final_archive_name = final_archive_name + '.gz' + if archive: + archive.close() + if not archive.empty: + final_archive = ArchiveMbox(final_archive_name) + final_archive.lock() + try: + final_archive.append(archive.mbox_file_name) + final_archive.commit() + finally: + final_archive.unlock() + final_archive.close() + archive.remove() + +def make_archive_name(mailbox_name): + """Derive archive name and (relative) path from the mailbox name.""" + # allow the user to embed time formats such as '%B' in the archive name + if options.date_old_max == None: + tm = time.localtime(time.time() - options.days_old_max*24*60*60) + else: + tm = time.localtime(options.date_old_max) + prefix = suffix = "" + if options.archive_name: + archive_head = "" + archive_tail = time.strftime(options.archive_name, tm) + else: + if options.archive_prefix is None and options.archive_suffix is None: + suffix = options.archive_default_suffix + else: + if options.archive_prefix: + prefix = time.strftime(options.archive_prefix, tm) + if options.archive_suffix: + suffix = time.strftime(options.archive_suffix, tm) + archive_head, archive_tail = os.path.split(mailbox_name) + if not prefix: + # Don't create hidden archives, e.g. when processing Maildir++ + # subfolders + archive_tail = archive_tail.lstrip('.') + if options.output_dir: + archive_head = options.output_dir + archive_name = os.path.join(archive_head, prefix + archive_tail + suffix) + return archive_name + +def check_sane_destdir(dir): + """Do a very primitive check if the given directory looks like a reasonable + destination directory and bail out if it doesn't.""" + assert dir + if not os.path.isdir(dir): + user_error("output directory does not exist: '%s'" % dir) + if not os.access(dir, os.W_OK): + user_error("no write permission on output directory: '%s'" % dir) + +def check_archive(archive_name): + """Check if existing archive files are (not) compressed as expected and + check if we can work with the destination directory.""" + compressed_archive = archive_name + ".gz" + if options.no_compress: + if os.path.isfile(compressed_archive): + user_error("There is already a file named '%s'!\n" + "Have you been previously compressing this archive?\n" + "You probably should uncompress it manually, and try running me " + "again." % compressed_archive) + elif os.path.isfile(archive_name): + user_error("There is already a file named '%s'!\n" + "Have you been reading this archive?\n" + "You probably should re-compress it manually, and try running me " + "again." % archive_name) + dest_dir = os.path.dirname(archive_name) + if not dest_dir: + dest_dir = os.getcwd() + check_sane_destdir(dest_dir) + +def nice_size_str(size): + """Return given size in bytes as '12kB', '1.2MB'""" + kb = size / 1024.0 + mb = kb / 1024.0 + if mb >= 1.0: return str(round(mb, 1)) + 'MB' + if kb >= 1.0: return str(round(kb)) + 'kB' + return str(size) + 'B' + + +def get_filename(msg): + """If the given rfc822.Message can be identified with a file (no mbox), + return the filename, otherwise raise AttributeError.""" + try: + return msg.fp.name + except AttributeError: + # Ugh, that's ugly. msg.fp is not a plain file, it may be an + # instance of + # a. mailbox._Subfile + # (msg from mailbox.UnixMailbox, Python <= 2.4) + # File object is msg.fp.fp, we don't want that + # b. mailbox._PartialFile, subclass of mailbox._ProxyFile + # (msg from mailbox.UnixMailbox, Python >= 2.5) + # File object is msg.fp._file, we don't want that + # c. mailbox._ProxyFile + # (msg from mailbox.Maildir, Python >= 2.5) + # File object is msg.fp._file, we do want that. + if msg.fp.__class__ == mailbox._ProxyFile: + assert hasattr(mailbox, "_PartialFile") + return msg.fp._file.name + raise + +def safe_open_create(filename): + """Create and open a file in a NFSv2-safe way, and return a r/w file descriptor. + The new file is created with mode 600.""" + # This is essentially a simplified version of the dotlocking function. + vprint("Creating file '%s'" % filename) + dir, basename = os.path.split(filename) + # We rely on tempfile.mkstemp to create files safely and with 600 mode. + fd, pre_name = tempfile.mkstemp(prefix=basename+".pre-", dir=dir) + try: + try: + os.link(pre_name, filename) + except OSError, e: + if os.fstat(fd).st_nlink == 2: + pass + else: + raise + finally: + os.unlink(pre_name) + return fd + +def safe_open_existing(filename): + """Safely open an existing file, and return a r/w file descriptor.""" + lst = os.lstat(filename) + if stat.S_ISLNK(lst.st_mode): + unexpected_error("file '%s' is a symlink." % filename) + fd = os.open(filename, os.O_RDWR) + fst = os.fstat(fd) + if fst.st_nlink != 1: + unexpected_error("file '%s' has %d hard links." % \ + (filename, fst.st_nlink)) + if stat.S_ISDIR(fst.st_mode): + unexpected_error("file '%s' is a directory." % filename) + for i in stat.ST_DEV, stat.ST_INO, stat.ST_UID, stat.ST_GID, stat.ST_MODE, stat.ST_NLINK: + if fst[i] != lst[i]: + unexpected_error("file status changed unexpectedly") + return fd + +def safe_open(filename): + """Safely open a file, creating it if it doesn't exist, and return a + r/w file descriptor.""" + # This borrows from postfix code. + vprint("Opening archive...") + try: + fd = safe_open_existing(filename) + except OSError, e: + if e.errno != errno.ENOENT: raise + fd = safe_open_create(filename) + return fd + +# this is where it all happens, folks +if __name__ == '__main__': + main() diff --git a/archivemail.xml b/archivemail.xml new file mode 100644 index 0000000..9bc5642 --- /dev/null +++ b/archivemail.xml @@ -0,0 +1,794 @@ + + +lockf +2'> + + +gzip +1'> + + +procmail +1'> + + +python +1'> + + +crontab +5'> + + +mbox +5'> +]> + + + +5 July 2011 + + +archivemail +1 +archivemail user manual +archivemail 0.9.0 + + + +archivemail +archive and compress your old email + + + + + +archivemail + +MAILBOX + + + + + +Description + + +archivemail is a tool for archiving and compressing old +email in mailboxes. +By default it will read the mailbox MAILBOX, moving +messages that are older than the specified number of days (180 by default) to +a &mbox;-format mailbox in the same directory that is compressed with &gzip;. +It can also just delete old email rather than archive it. + + + +By default, archivemail derives the archive filename from +the mailbox name by appending an _archive suffix to the +mailbox name. For example, if you run archivemail on a +mailbox called exsouthrock, the archive will be created +with the filename exsouthrock_archive.gz. +This default behavior can be overridden with command line options, choosing +a custom suffix, a prefix, or a completely custom name for the archive. + + + +archivemail supports reading IMAP, +Maildir, MH and +mbox-format mailboxes, but always writes +mbox-format archives. + + + +Messages that are flagged important are not archived or deleted unless +explicitly requested with the option. +Also, archivemail can be configured not to archive unread +mail, or to only archive messages larger than a specified size. + + + +To archive an IMAP-format mailbox, use the format +imap://username:password@server/mailbox to specify +the mailbox. +archivemail will expand wildcards in +IMAP mailbox names according to +RFC 3501, which says: The +character "*" is a wildcard, and matches zero or more characters at this +position. The character "%" is similar to "*", but it does not match a +hierarchy delimiter. +You can omit the password from the URL; use the + option to make archivemail read +the password from a file, or alternatively just enter it upon request. +If the option is set, archivemail +does not look for a password in the URL, and the colon is +not considered a delimiter. +Substitute imap with +imaps, and archivemail will +establish a secure SSL connection. +See below for more IMAP peculiarities. + + + + +Options + + + + + + +Archive messages older than NUM +days. The default is 180. This option is incompatible with the + option below. + + + + + + +Archive messages older than DATE. +DATE can be a date string in ISO format (eg +2002-04-23), Internet format (eg 23 Apr +2002) or Internet format with full month names (eg +23 April 2002). Two-digit years are not supported. +This option is incompatible with the option above. + + + + + + +Use the directory name PATH to +store the mailbox archives. The default is the same directory as the mailbox +to be read. + + + + + + +Read IMAP password from file +FILE instead of from the command line. Note +that this will probably not work if you are archiving folders from +more than one IMAP account. + + + + + + +Append STRING to the +IMAP filter string. +For IMAP wizards. + + + + + + +Prefix NAME to the archive name. +NAME is expanded by the &python; function +time.strftime(), which means that you can specify special +directives in NAME to make an archive named after +the archive cut-off date. +See the discussion of the option for a list of valid +strftime() directives. +The default is not to add a prefix. + + + + + + + +Use the suffix NAME to create the filename used for +archives. The default is _archive, unless a prefix is +specified. + + +Like a prefix, the suffix NAME is expanded by the +&python; function time.strftime() with the archive +cut-off date. time.strftime() understands the following +directives: + + %a + + Locale's abbreviated weekday name. + + + %A + + Locale's full weekday name. + + + %b + + Locale's abbreviated month name. + + + %B + + Locale's full month name. + + + %c + + Locale's appropriate date and time representation. + + + %d + + Day of the month as a decimal number [01,31]. + + + %H + + Hour (24-hour clock) as a decimal number [00,23]. + + + %I + + Hour (12-hour clock) as a decimal number [01,12]. + + + %j + + Day of the year as a decimal number [001,366]. + + + %m + + Month as a decimal number [01,12]. + + + %M + + Minute as a decimal number [00,59]. + + + %p + + Locale's equivalent of either AM or PM. + + + %S + + Second as a decimal number [00,61]. (1) + + + %U + + Week number of the year (Sunday as the first day of the week) + as a decimal number [00,53]. All days in a new year preceding + the first Sunday are considered to be in week 0. + + + %w + + Weekday as a decimal number [0(Sunday),6]. + + + %W + + Week number of the year (Monday as the first day of the week) + as a decimal number [00,53]. All days in a new year preceding + the first Sunday are considered to be in week 0. + + + %x + + Locale's appropriate date representation. + + + %X + + Locale's appropriate time representation. + + + %y + + Year without century as a decimal number [00,99]. + + + %Y + + Year with century as a decimal number. + + + %Z + + Time zone name (or by no characters if no time zone exists). + + + %% + + A literal % character. + + + + + + + + + +Use NAME as the archive name, +ignoring the name of the mailbox that is archived. +Like prefixes and suffixes, NAME is expanded by +time.strftime() with the archive cut-off date. +Because it hard-codes the archive name, this option cannot be used when +archiving multiple mailboxes. + + + + + + +Only archive messages that are NUM +bytes or greater. + + + + + + + +Don't write to any files -- just show what would have been done. This is +useful for testing to see how many messages would have been archived. + + + + + + + +Do not archive any messages that have not yet been read. +archivemail determines if a message in a +mbox-format or MH-format mailbox has +been read by looking at the Status header (if it exists). +If the status header is equal to RO or +OR then archivemail +assumes the message has been read. +archivemail determines if a maildir +message has been read by looking at the filename. +If the filename contains an S after +:2, then it assumes the message has been read. + + + + + + + + +Do not mangle lines in message bodies beginning with +From . +When archiving a message from a mailbox not in mbox +format, by default archivemail mangles such lines by +prepending a > to them, since mail user +agents might otherwise interpret these lines as message separators. +Messages from mbox folders are never mangled. See &mbox; +for more information. + + + + + + + + +Delete rather than archive old mail. Use this option with caution! + + + + + + + + +Copy rather than archive old mail. +Creates an archive, but the archived messages are not deleted from the +originating mailbox, which is left unchanged. +This is a complement to the option, and mainly +useful for testing purposes. +Note that multiple passes will create duplicates, since messages are blindly +appended to an existing archive. + + + + + + + + +Archive all messages, without distinction. + + + + + + + + +Normally messages that are flagged important are not archived or deleted. If +you specify this option, these messages can be archived or deleted just like +any other message. + + + + + + + + +Do not compress any archives. + + + + + + + + +Warn about duplicate Message-IDs that appear in the input +mailbox. + + + + + + +Reports lots of extra debugging information about what is going on. + + + + + + + + +Set IMAP debugging level. This makes +archivemail dump its conversation with the +IMAP server and some internal IMAP +processing to stdout. Higher values for +NUM give more elaborate output. Set +NUM to 4 to see all exchanged +IMAP commands. (Actually, NUM +is just passed literally to imaplib.Debug.) + + + + + + + +Turns on quiet mode. Do not print any statistics about how many messages were +archived. This should be used if you are running +archivemail from cron. + + + + + + + +Display the version of archivemail and exit. + + + + + + + +Display brief summary information about how to run +archivemail. + + + + + + + +Notes + + +archivemail requires &python; version 2.3 or later. +When reading an mbox-format mailbox, +archivemail will create a lockfile with the extension +.lock so that &procmail; will not +deliver to the mailbox while it is being processed. It will also create an +advisory lock on the mailbox using &lockf;. The archive is locked in the same +way when it is updated. +archivemail will also complain and abort if a 3rd-party +modifies the mailbox while it is being read. + + + +archivemail will always attempt to preserve the last-access +and last-modify times of the input mailbox. Archive mailboxes are always +created with a mode of 0600. +If archivemail finds a pre-existing archive mailbox it will +append rather than overwrite that archive. +archivemail will refuse to operate on mailboxes that are +symbolic links. + + + +archivemail attempts to find the delivery date of a message +by looking for valid dates in the following headers, in order of precedence: +Delivery-date, +Received, +Resent-Date and +Date. +If it cannot find any valid date in these headers, it will use the +last-modified file timestamp on MH and +Maildir format mailboxes, or the date on the +From_ line on mbox-format mailboxes. + + + +When archiving mailboxes with leading dots in the name, +archivemail will strip the dots off the archive name, so +that the resulting archive file is not hidden. +This is not done if the or + option is used. +Should there really be mailboxes distinguished only by leading dots in the +name, they will thus be archived to the same archive file by default. + + + +A conversion from other formats to &mbox; will silently overwrite existing +Status and X-Status message headers. + + + +<acronym>IMAP</acronym> + +When archivemail processes an IMAP +folder, all messages in that folder will have their \Recent +flag unset, and they will probably not show up as new in your +user agent later on. +There is no way around this, it's just how IMAP works. +This does not apply, however, if you run archivemail with +the options or . + + +archivemail relies on server-side searches to determine the +messages that should be archived. +When matching message dates, IMAP servers refer to server +internal message dates, and these may differ from both delivery time of a +message and its Date header. +Also, there exist broken servers which do not implement server side searches. + +<acronym>IMAP</acronym> <acronym>URL</acronym>s + +archivemail's IMAP +URL parser was written with the RFC 2882 +(Internet Message Format) rules for the +local-part of email addresses in mind. +So, rather than enforcing an URL-style encoding of +non-ascii and reserved characters, it allows you to +double-quote the username and password. +If your username or password contains the delimiter characters +@ or :, just quote it like this: +imap://"username@bogus.com":"password"@imap.bogus.com/mailbox. +You can use a backslash to escape double-quotes that are part of a quoted +username or password. +Note that quoting only a substring will not work, and be aware that your shell +will probably remove unprotected quotes or backslashes. + + +Similarly, there is no need to percent-encode non-ascii +characters in IMAP mailbox names. +As long as your locale is configured properly, archivemail +should handle these without problems. +Note, however, that due to limitations of the IMAP +protocol, non-ascii characters do not mix well with +wildcards in mailbox names. + + +archivemail tries to be smart when handling mailbox paths. +In particular, it will automatically add an IMAP +NAMESPACE prefix to the mailbox path if necessary; and if +you are archiving a subfolder, you can use the slash as a path separator +instead of the IMAP server's internal representation. + + + + + + +Examples + + + +To archive all messages in the mailbox debian-user that +are older than 180 days to a compressed mailbox called +debian-user_archive.gz in the current directory: + +bash$ archivemail debian-user + + + + + + +To archive all messages in the mailbox debian-user that +are older than 180 days to a compressed mailbox called +debian-user_October_2001.gz (where the current month and +year is April, 2002) in the current directory: + +bash$ archivemail --suffix '_%B_%Y' debian-user + + + + + + +To archive all messages in the mailbox cm-melb that +are older than the first of January 2002 to a compressed mailbox called +cm-melb_archive.gz in the current directory: + +bash$ archivemail --date='1 Jan 2002' cm-melb + + + + + + +Exactly the same as the above example, using an ISO date +format instead: + +bash$ archivemail --date=2002-01-01 cm-melb + + + + + + +To delete all messages in the mailbox spam that +are older than 30 days: + +bash$ archivemail --delete --days=30 spam + + + + + + +To archive all read messages in the mailbox incoming that +are older than 180 days to a compressed mailbox called +incoming_archive.gz in the current directory: + +bash$ archivemail --preserve-unread incoming + + + + + + +To archive all messages in the mailbox received that +are older than 180 days to an uncompressed mailbox called +received_archive in the current directory: + +bash$ archivemail --no-compress received + + + + + + +To archive all mailboxes in the directory $HOME/Mail +that are older than 90 days to compressed mailboxes in the +$HOME/Mail/Archive directory: + +bash$ archivemail -d90 -o $HOME/Mail/Archive $HOME/Mail/* + + + + + + +To archive all mails older than 180 days from the given +IMAP INBOX to a compressed mailbox +INBOX_archive.gz in the +$HOME/Mail/Archive directory, quoting the password and +reading it from the environment variable PASSWORD: + + + +bash$ archivemail -o $HOME/Mail/Archive imaps://user:'"'$PASSWORD'"'@example.org/INBOX + + +Note the protected quotes. + + + + + +To archive all mails older than 180 days in subfolders of foo on the given IMAP +server to corresponding archives in the current working directory, reading the +password from the file ~/imap-pass.txt: + + +bash$ archivemail --pwfile=~/imap-pass.txt imaps://user@example.org/foo/* + + + + + +Tips + +Probably the best way to run archivemail is from your +&crontab; file, using the option. +Don't forget to try the and perhaps the + option for non-destructive testing. + + + + +Exit Status +Normally the exit status is 0. Nonzero indicates an unexpected error. + + + + +Bugs + +If an IMAP mailbox path contains slashes, the archive +filename will be derived from the basename of the mailbox. +If the server's folder separator differs from the Unix slash and is used in +the IMAP URL, however, the whole path +will be considered the basename of the mailbox. +E.g. the two URLs +imap://user@example.com/folder/subfolder and +imap://user@example.com/folder.subfolder will be +archived in subfolder_archive.gz and +folder.subfolder_archive.gz, respectively, although they +might refer to the same IMAP mailbox. + + +archivemail does not support reading +MMDF or Babyl-format mailboxes. In fact, +it will probably think it is reading an mbox-format mailbox +and cause all sorts of problems. + + + +archivemail is still too slow, but if you are running from +&crontab; you won't care. Archiving maildir-format +mailboxes should be a lot quicker than mbox-format +mailboxes since it is less painful for the original mailbox to be +reconstructed after selective message removal. + + + + +See Also + + &mbox; + &crontab; + &python; + &procmail; + + + + +<acronym>Url</acronym> +The archivemail home page is currently hosted at +sourceforge + + + + +Author + This manual page was written by Paul Rodger <paul at paulrodger +dot com>. Updated and supplemented by Nikolaus Schulz +microschulz@web.de + + + diff --git a/db2html.xsl b/db2html.xsl new file mode 100644 index 0000000..d3be5f6 --- /dev/null +++ b/db2html.xsl @@ -0,0 +1,10 @@ + + + + manpage.css + +

archivemail

+
+ + diff --git a/db2man.xsl b/db2man.xsl new file mode 100644 index 0000000..bcc3f61 --- /dev/null +++ b/db2man.xsl @@ -0,0 +1,29 @@ + + + + + + + + + .TP + + + + + + + + + + + + + + + + + + + diff --git a/examples/archivemail_all b/examples/archivemail_all new file mode 100644 index 0000000..30cb05e --- /dev/null +++ b/examples/archivemail_all @@ -0,0 +1,31 @@ +#!/bin/sh +# +# This is an example shell script I use from my crontab(5) file to selectively +# archive some of my mailboxes. Most of these mailboxes come directly from +# procmail and are in maildir-format -- not that it should really matter. +# +# I probably could have done them all as: +# $ARCMAIL $HOME/Mail/* +# ...if I had enough disk space to keep mail uncompressed for 180 days :) +# +set -e + +ARCMAIL="/usr/local/bin/archivemail --quiet --output-dir=$HOME/Mail/Archive/ " + +$ARCMAIL --days 14 $HOME/Mail/debian-user \ + $HOME/Mail/linux-kernel \ + $HOME/Mail/python-list \ + $HOME/Mail/spam-l + +$ARCMAIL --days 30 --delete $HOME/Mail/duplicates + +$ARCMAIL --days 90 $HOME/Mail/bugtraq \ + $HOME/Mail/debian-devel \ + $HOME/Mail/debian-mentors \ + $HOME/Mail/spam + +$ARCMAIL $HOME/Mail/cm-melb \ + $HOME/Mail/exsouthrock \ + $HOME/Mail/received \ + $HOME/Mail/sent \ + $HOME/Mail/vim diff --git a/index.html b/index.html new file mode 100644 index 0000000..90bef89 --- /dev/null +++ b/index.html @@ -0,0 +1,192 @@ + + + + + archivemail – a tool for archiving and compressing old email + + + + + +

archivemail

+ +
+
+ Latest version: 0.9.0
+ Released on 2011-07-09 +
+

What is it?

+

+archivemail is a tool for archiving and compressing +old email in mailboxes. +It moves messages older than the specified number of +days to a separate mbox format mailbox that is +compressed with gzip. +It can also just delete old email rather than archive it. + +

What can it do for me?

+

+Maybe some of your mailboxes are quite large (eg, over 10,000 messages) and +they are taking a while to load in your mail reader. Perhaps they are taking +up too much space on your disk. Archiving old messages to a separate, +compressed mailbox will mean: +

    +
  1. Your mail reader will get a huge performance boost loading and reading + your mail. +
  2. You will be taking up less disk space, since old mail will be compressed. + (Mail usually compresses quite nicely.) +
  3. You won't be confronted with semi-obsolete mail all the time. +
+

+You can also use archivemail as a simple backup +tool. + +

Features overview

+
    +
  • Supports archiving + IMAP, + mbox, + MH and + Maildir format + mailboxes. +
  • Old mail can be either archived or just deleted. +
  • The age in days that is considered old is configurable – it defaults + to 180 days. + You can also set an absolute limit date. +
  • Messages that are flagged important are not archived or deleted unless + explicitely requested. + +
  • Can be configured to preserve unread mail. +
  • Can be configured to only archive messages over a given byte size. +
  • Stores the compressed archive mailboxes in a directory of + your choice, with an extension of your choice. +
  • Easy read-only testing, not touching your valuable data. +
  • Supports IMAPS/SSL. +
  • When archiving IMAP mailboxes, the message selection can be refined by + extending the underlying IMAP SEARCH command with arbitrary + search keys (you will have to cope with the raw IMAP protocol, though). +
+ +

Documentation

+

+The archivemail manpage is the primary documentation +for archivemail. +To see what has changed in the latest version, check the release notes. +If you want to have a closer look at the current development status, here's the + +CHANGELOG and the +TODO list, fresh from the repository. + +

Where can I get it?

+

+You can grab the latest version of archivemail +directly from the archivemail +download area at Sourceforge. +There should be up-to-date binary RPM packages at the +OpenSUSE build +service for SUSE Linux and Fedora Core. +Also, many Linux distributions provide packages; e.g. there is a +Debian +package. + +

+archivemail is written in Python, and hacking it is +easy and fun; you can check out the source from the git repository with +the following command: +

+git clone git://archivemail.git.sourceforge.net/gitroot/archivemail/archivemail +
+

+See also the short +introduction to git access at sourceforge. + + + +

Getting involved

+ + +

Requirements

+

+archivemail requires Python 2.3 or newer. +It also uses some optional python modules, but these should be pretty much +standard; if you get an ImportError nonetheless, please +report it, thanks. + +

License

+

+This software is licensed under the terms of the +GNU GPL, either +version 2 of the license, or any later version. + +

Credits

+

+archivemail was written by Paul Rodger +<paul at paulrodger dot + com> +and is currently maintained by Peter Poeml <poeml + at suse dot de>, Nikolaus Schulz <microschulz@web.de> +and Brandon Knitter. + +

+

+ + + + SourceForge.net Logo + + + + + diff --git a/manpage.css b/manpage.css new file mode 100644 index 0000000..915cea2 --- /dev/null +++ b/manpage.css @@ -0,0 +1,15 @@ +@import "style.css"; +h2 { + font-variant: small-caps; + font-size: 170%; +} +.informalexample { + margin-bottom: 1.2em; +} +div.informalexample .screen { + margin-left: 2ex; +} + +a#strftime + dl dt { float: left; margin: 0.3ex 0; width: 1.5em; } +a#strftime + dl dd { float: left; margin: 0.3ex 0; margin-left: 1.2em; width: 90%; } +dt { clear: left; } diff --git a/setup.py b/setup.py new file mode 100755 index 0000000..e4521c9 --- /dev/null +++ b/setup.py @@ -0,0 +1,33 @@ +#! /usr/bin/env python + +import sys + +def check_python_version(): + """Abort if we are running on python < v2.0""" + too_old_error = """This program requires python v2.0 or greater. +Your version of python is: %s""" % sys.version + try: + version = sys.version_info # we might not even have this function! :) + if (version[0] < 2): + print too_old_error + sys.exit(1) + except AttributeError: + print too_old_error + sys.exit(1) + +# define & run this early - 'distutils.core' requires Python >= 2.0 +check_python_version() +from distutils.core import setup + +setup(name="archivemail", + version="0.9.0", + description="archive and compress old email", + license="GNU GPL", + url="http://archivemail.sourceforge.net/", + author="Paul Rodger", + author_email="paul@paulrodger.com", + maintainer="Nikolaus Schulz, Peter Poeml", + maintainer_email="nikosch@users.sourceforge.net, poeml@users.sourceforge.net", + scripts=["archivemail"], + data_files=[("share/man/man1", ["archivemail.1"])], + ) diff --git a/style.css b/style.css new file mode 100644 index 0000000..7893786 --- /dev/null +++ b/style.css @@ -0,0 +1,49 @@ +body { + padding: 2%; + line-height: 130%; + margin: 0; + /*color: #036;*/ +} +h1 { + font-size: 220%; + font-weight: bold; + padding: 0 0 0.4em 0.1em; + /*margin: 0 0 0.5em 0; */ + margin: 0; + /*border-bottom: 2px solid black;*/ +} +hr { + border: 1px #b8b8b8 solid; +} +h1 + hr { + margin-top: 0; + margin-bottom: 1.7em; +} +h2 { + margin: 1em 0 0.8em 0; + padding: 0; + font-size: 150%; +} +img {border: none;} +a { + text-decoration: underline; +} +a:link { + /*color: #0073c7;*/ + color: blue; + background-color: inherit; +} + +a:visited { + /*color: #5A88B5;*/ + color: #844084; + background-color: inherit; +} + +/* +a:hover, +a:active { + color: #0073c7; + background-color: #f0f0f0; +} +*/ diff --git a/test_archivemail b/test_archivemail new file mode 100755 index 0000000..ed033e2 --- /dev/null +++ b/test_archivemail @@ -0,0 +1,1771 @@ +#! /usr/bin/env python +############################################################################ +# Copyright (C) 2002 Paul Rodger +# (C) 2006-2011 Nikolaus Schulz +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################ +""" +Unit-test archivemail using 'PyUnit'. + +TODO: add tests for: + * dotlock locks already existing + * archiving MH-format mailboxes + * a 3rd party process changing the mbox file being read + +""" + +import sys + +def check_python_version(): + """Abort if we are running on python < v2.4""" + too_old_error = "This test script requires python version 2.4 or later. " + \ + "Your version of python is:\n%s" % sys.version + try: + version = sys.version_info # we might not even have this function! :) + if (version[0] < 2) or (version[0] == 2 and version[1] < 4): + print too_old_error + sys.exit(1) + except AttributeError: + print too_old_error + sys.exit(1) + +# define & run this early because 'unittest' requires Python >= 2.1 +check_python_version() + +import copy +import fcntl +import filecmp +import os +import re +import shutil +import stat +import tempfile +import time +import unittest +import gzip +import cStringIO +import rfc822 +import mailbox + +from types import ModuleType +archivemail = ModuleType("archivemail") +try: + module_fp = open("archivemail", "r") +except IOError: + print "The archivemail script should be in the current directory in order" + print "to be imported and tested. Sorry." + sys.exit(1) +exec module_fp in archivemail.__dict__ + +# We want to iterate over messages in a compressed archive mbox and verify +# them. This involves seeking in the mbox. The gzip.Gzipfile.seek() in +# Python 2.5 doesn't understand whence; this is Python bug #1355023, triggered +# by mailbox._PartialFile.seek(). The bug is still pending as of Python +# 2.5.2. To work around it, we subclass gzip.GzipFile. +# +# It should be noted that seeking backwards in a GzipFile is emulated by +# re-reading the entire file from the beginning, which is extremely +# inefficient and won't work with large files; but our test archives are all +# small, so it's okay. + +class FixedGzipFile(gzip.GzipFile): + """GzipFile with seek method accepting whence parameter.""" + def seek(self, offset, whence=0): + try: + # Try calling gzip.GzipFile.seek with the whence parameter. + # For Python >= 2.7, it returns the new offset; pass that on. + return gzip.GzipFile.seek(self, offset, whence) + except TypeError: + if whence: + if whence == 1: + offset = self.offset + offset + else: + raise ValueError('Seek from end not supported') + return gzip.GzipFile.seek(self, offset) + +# precision of os.utime() when restoring mbox timestamps +utimes_precision = 5 + +class MessageIdFactory: + """Factory to create `uniqe' message-ids.""" + def __init__(self): + self.seq = 0 + def __call__(self): + self.seq += 1 + return "" % self.seq + +make_msgid = MessageIdFactory() + +class IndexedMailboxDir: + """An indexed mailbox directory, providing random message access by + message-id. Intended as a base class for a maildir and an mh subclass.""" + + def __init__(self, mdir_name): + assert tempfile.tempdir + self.root = tempfile.mkdtemp(prefix=mdir_name) + self.msg_id_dict = {} + self.deliveries = 0 + + def _add_to_index(self, msg_text, fpath): + """Add the given message to the index, for later random access.""" + # Extract the message-id as index key + msg_id = None + fp = cStringIO.StringIO(msg_text) + while True: + line = fp.readline() + # line empty means we didn't find a message-id + assert line + if line.lower().startswith("message-id:"): + msg_id = line.split(":", 1)[-1].strip() + assert msg_id + break + assert not self.msg_id_dict.has_key(msg_id) + self.msg_id_dict[msg_id] = fpath + + def get_all_filenames(self): + """Return all relative pathnames of files in this mailbox.""" + return self.msg_id_dict.values() + +class SimpleMaildir(IndexedMailboxDir): + """Primitive Maildir class, just good enough for generating short-lived + test maildirs.""" + + def __init__(self, mdir_name='maildir'): + IndexedMailboxDir.__init__(self, mdir_name) + for d in "cur", "tmp", "new": + os.mkdir(os.path.join(self.root, d)) + + def write(self, msg_str, new=True, flags=[]): + """Store a message with the given flags.""" + assert not (new and flags) + if new: + subdir = "new" + else: + subdir = "cur" + fname = self._mkname(new, flags) + relpath = os.path.join(subdir, fname) + path = os.path.join(self.root, relpath) + assert not os.path.exists(path) + f = open(path, "w") + f.write(msg_str) + f.close() + self._add_to_index(msg_str, relpath) + + def _mkname(self, new, flags): + """Generate a unique filename for a new message.""" + validflags = 'DFPRST' + for f in flags: + assert f in validflags + # This 'unique' name should be good enough, since nobody else + # will ever write messages to this maildir folder. + uniq = str(self.deliveries) + self.deliveries += 1 + if new: + return uniq + if not flags: + return uniq + ':2,' + finfo = "".join(sorted(flags)) + return uniq + ':2,' + finfo + + def get_message_and_mbox_status(self, msgid): + """For the Message-Id msgid, return the matching message in text + format and its status, expressed as a set of mbox flags.""" + fpath = self.msg_id_dict[msgid] # Barfs if not found + mdir_flags = fpath.rsplit('2,', 1)[-1] + flagmap = { + 'F': 'F', + 'R': 'A', + 'S': 'R' + } + mbox_flags = set([flagmap[x] for x in mdir_flags]) + if fpath.startswith("cur/"): + mbox_flags.add('O') + fp = open(os.path.join(self.root, fpath), "r") + msg = fp.read() + fp.close() + return msg, mbox_flags + + +class TestCaseInTempdir(unittest.TestCase): + """Base class for testcases that need to create temporary files. + All testcases that create temporary files should be derived from this + class, not directly from unittest.TestCase. + TestCaseInTempdir provides these methods: + + setUp() Creates a safe temporary directory and sets tempfile.tempdir. + + tearDown() Recursively removes the temporary directory and unsets + tempfile.tempdir. + + Overriding methods should call the ones above.""" + temproot = None + + def setUp(self): + if not self.temproot: + assert not tempfile.tempdir + self.temproot = tempfile.tempdir = \ + tempfile.mkdtemp(prefix="test-archivemail") + + def tearDown(self): + assert tempfile.tempdir == self.temproot + if self.temproot: + shutil.rmtree(self.temproot) + tempfile.tempdir = self.temproot = None + + +############ Mbox Class testing ############## + +class TestMboxDotlock(TestCaseInTempdir): + def setUp(self): + super(TestMboxDotlock, self).setUp() + self.mbox_name = make_mbox() + self.mbox_mode = os.stat(self.mbox_name)[stat.ST_MODE] + self.mbox = archivemail.Mbox(self.mbox_name) + + def testDotlock(self): + """dotlock_lock/unlock should create/delete a lockfile""" + lock = self.mbox_name + ".lock" + self.mbox._dotlock_lock() + assert os.path.isfile(lock) + self.mbox._dotlock_unlock() + assert not os.path.isfile(lock) + + def testDotlockingSucceedsUponEACCES(self): + """A dotlock should silently be omitted upon EACCES.""" + archivemail.options.quiet = True + mbox_dir = os.path.dirname(self.mbox_name) + os.chmod(mbox_dir, 0500) + try: + self.mbox._dotlock_lock() + self.mbox._dotlock_unlock() + finally: + os.chmod(mbox_dir, 0700) + archivemail.options.quiet = False + +class TestMboxPosixLock(TestCaseInTempdir): + def setUp(self): + super(TestMboxPosixLock, self).setUp() + self.mbox_name = make_mbox() + self.mbox = archivemail.Mbox(self.mbox_name) + + def testPosixLock(self): + """posix_lock/unlock should create/delete an advisory lock""" + + # The following code snippet heavily lends from the Python 2.5 mailbox + # unittest. + # BEGIN robbery: + + # Fork off a subprocess that will lock the file for 2 seconds, + # unlock it, and then exit. + pid = os.fork() + if pid == 0: + # In the child, lock the mailbox. + self.mbox._posix_lock() + time.sleep(2) + self.mbox._posix_unlock() + os._exit(0) + + # In the parent, sleep a bit to give the child time to acquire + # the lock. + time.sleep(0.5) + # The parent's file self.mbox.mbox_file shares fcntl locks with the + # duplicated FD in the child; reopen it so we get a different file + # table entry. + file = open(self.mbox_name, "r+") + lock_nb = fcntl.LOCK_EX | fcntl.LOCK_NB + fd = file.fileno() + try: + self.assertRaises(IOError, fcntl.lockf, fd, lock_nb) + + finally: + # Wait for child to exit. Locking should now succeed. + exited_pid, status = os.waitpid(pid, 0) + + fcntl.lockf(fd, lock_nb) + fcntl.lockf(fd, fcntl.LOCK_UN) + # END robbery + + +class TestMboxNext(TestCaseInTempdir): + def setUp(self): + super(TestMboxNext, self).setUp() + self.not_empty_name = make_mbox(messages=18) + self.empty_name = make_mbox(messages=0) + + def testNextEmpty(self): + """mbox.next() should return None on an empty mailbox""" + mbox = archivemail.Mbox(self.empty_name) + msg = mbox.next() + self.assertEqual(msg, None) + + def testNextNotEmpty(self): + """mbox.next() should a message on a populated mailbox""" + mbox = archivemail.Mbox(self.not_empty_name) + for count in range(18): + msg = mbox.next() + assert msg + msg = mbox.next() + self.assertEqual(msg, None) + + +############ TempMbox Class testing ############## + +class TestTempMboxWrite(TestCaseInTempdir): + def setUp(self): + super(TestTempMboxWrite, self).setUp() + + def testWrite(self): + """mbox.write() should append messages to a mbox mailbox""" + read_file = make_mbox(messages=3) + mbox_read = archivemail.Mbox(read_file) + mbox_write = archivemail.TempMbox() + write_file = mbox_write.mbox_file_name + for count in range(3): + msg = mbox_read.next() + mbox_write.write(msg) + mbox_read.close() + mbox_write.close() + assert filecmp.cmp(read_file, write_file, shallow=0) + + def testWriteNone(self): + """calling mbox.write() with no message should raise AssertionError""" + write = archivemail.TempMbox() + self.assertRaises(AssertionError, write.write, None) + +class TestTempMboxRemove(TestCaseInTempdir): + def setUp(self): + super(TestTempMboxRemove, self).setUp() + self.mbox = archivemail.TempMbox() + self.mbox_name = self.mbox.mbox_file_name + + def testMboxRemove(self): + """remove() should delete a mbox mailbox""" + assert os.path.exists(self.mbox_name) + self.mbox.remove() + assert not os.path.exists(self.mbox_name) + + + +########## options class testing ################# + +class TestOptionDefaults(unittest.TestCase): + def testVerbose(self): + """verbose should be off by default""" + self.assertEqual(archivemail.options.verbose, False) + + def testDaysOldMax(self): + """default archival time should be 180 days""" + self.assertEqual(archivemail.options.days_old_max, 180) + + def testQuiet(self): + """quiet should be off by default""" + self.assertEqual(archivemail.options.quiet, False) + + def testDeleteOldMail(self): + """we should not delete old mail by default""" + self.assertEqual(archivemail.options.delete_old_mail, False) + + def testNoCompress(self): + """no-compression should be off by default""" + self.assertEqual(archivemail.options.no_compress, False) + + def testIncludeFlagged(self): + """we should not archive flagged messages by default""" + self.assertEqual(archivemail.options.include_flagged, False) + + def testPreserveUnread(self): + """we should not preserve unread messages by default""" + self.assertEqual(archivemail.options.preserve_unread, False) + +class TestOptionParser(unittest.TestCase): + def setUp(self): + self.oldopts = copy.copy(archivemail.options) + + def testOptionDate(self): + """--date and -D options are parsed correctly""" + date_formats = ( + "%Y-%m-%d", # ISO format + "%d %b %Y" , # Internet format + "%d %B %Y" , # Internet format with full month names + ) + date = time.strptime("2000-07-29", "%Y-%m-%d") + unixdate = time.mktime(date) + for df in date_formats: + d = time.strftime(df, date) + for opt in '-D', '--date=': + archivemail.options.date_old_max = None + archivemail.options.parse_args([opt+d], "") + self.assertEqual(unixdate, archivemail.options.date_old_max) + + def testOptionPreserveUnread(self): + """--preserve-unread option is parsed correctly""" + archivemail.options.parse_args(["--preserve-unread"], "") + assert archivemail.options.preserve_unread + archivemail.options.preserve_unread = False + archivemail.options.parse_args(["-u"], "") + assert archivemail.options.preserve_unread + + def testOptionSuffix(self): + """--suffix and -s options are parsed correctly""" + for suffix in ("_static_", "_%B_%Y", "-%Y-%m-%d"): + archivemail.options.parse_args(["--suffix="+suffix], "") + self.assertEqual(archivemail.options.archive_suffix, suffix) + archivemail.options.archive_suffix = None + archivemail.options.parse_args(["-s", suffix], "") + self.assertEqual(archivemail.options.archive_suffix, suffix) + + def testOptionPrefix(self): + """--prefix and -p options are parsed correctly""" + for prefix in ("_static_", "_%B_%Y", "-%Y-%m-%d"): + archivemail.options.parse_args(["--prefix="+prefix], "") + self.assertEqual(archivemail.options.archive_prefix, prefix) + archivemail.options.archive_prefix = None + archivemail.options.parse_args(["-p", prefix], "") + self.assertEqual(archivemail.options.archive_prefix, prefix) + + def testOptionArchivename(self): + """--archive-name and -a options are parsed correctly""" + for name in ("custom", ".withdot", "custom_%Y", "%Y/joe"): + archivemail.options.parse_args(["--archive-name="+name], "") + self.assertEqual(archivemail.options.archive_name, name) + archivemail.options.archive_name = None + archivemail.options.parse_args(["-a", name], "") + self.assertEqual(archivemail.options.archive_name, name) + + def testOptionDryrun(self): + """--dry-run option is parsed correctly""" + archivemail.options.parse_args(["--dry-run"], "") + assert archivemail.options.dry_run + archivemail.options.preserve_unread = False + archivemail.options.parse_args(["-n"], "") + assert archivemail.options.dry_run + + def testOptionDays(self): + """--days and -d options are parsed correctly""" + archivemail.options.parse_args(["--days=11"], "") + self.assertEqual(archivemail.options.days_old_max, 11) + archivemail.options.days_old_max = None + archivemail.options.parse_args(["-d11"], "") + self.assertEqual(archivemail.options.days_old_max, 11) + + def testOptionDelete(self): + """--delete option is parsed correctly""" + archivemail.options.parse_args(["--delete"], "") + assert archivemail.options.delete_old_mail + + def testOptionCopy(self): + """--copy option is parsed correctly""" + archivemail.options.parse_args(["--copy"], "") + assert archivemail.options.copy_old_mail + + def testOptionOutputdir(self): + """--output-dir and -o options are parsed correctly""" + for path in "/just/some/path", "relative/path": + archivemail.options.parse_args(["--output-dir=%s" % path], "") + self.assertEqual(archivemail.options.output_dir, path) + archivemail.options.output_dir = None + archivemail.options.parse_args(["-o%s" % path], "") + self.assertEqual(archivemail.options.output_dir, path) + + def testOptionNocompress(self): + """--no-compress option is parsed correctly""" + archivemail.options.parse_args(["--no-compress"], "") + assert archivemail.options.no_compress + + def testOptionSize(self): + """--size and -S options are parsed correctly""" + size = "666" + archivemail.options.parse_args(["--size=%s" % size ], "") + self.assertEqual(archivemail.options.min_size, int(size)) + archivemail.options.parse_args(["-S%s" % size ], "") + self.assertEqual(archivemail.options.min_size, int(size)) + + def tearDown(self): + archivemail.options = self.oldopts + +########## archivemail.is_older_than_days() unit testing ################# + +class TestIsTooOld(unittest.TestCase): + def testVeryOld(self): + """with max_days=360, should be true for these dates > 1 year""" + for years in range(1, 10): + time_msg = time.time() - (years * 365 * 24 * 60 * 60) + assert archivemail.is_older_than_days(time_message=time_msg, + max_days=360) + + def testOld(self): + """with max_days=14, should be true for these dates > 14 days""" + for days in range(14, 360): + time_msg = time.time() - (days * 24 * 60 * 60) + assert archivemail.is_older_than_days(time_message=time_msg, + max_days=14) + + def testJustOld(self): + """with max_days=1, should be true for these dates >= 1 day""" + for minutes in range(0, 61): + time_msg = time.time() - (25 * 60 * 60) + (minutes * 60) + assert archivemail.is_older_than_days(time_message=time_msg, + max_days=1) + + def testNotOld(self): + """with max_days=9, should be false for these dates < 9 days""" + for days in range(0, 9): + time_msg = time.time() - (days * 24 * 60 * 60) + assert not archivemail.is_older_than_days(time_message=time_msg, + max_days=9) + + def testJustNotOld(self): + """with max_days=1, should be false for these hours <= 1 day""" + for minutes in range(0, 60): + time_msg = time.time() - (23 * 60 * 60) - (minutes * 60) + assert not archivemail.is_older_than_days(time_message=time_msg, + max_days=1) + + def testFuture(self): + """with max_days=1, should be false for times in the future""" + for minutes in range(0, 60): + time_msg = time.time() + (minutes * 60) + assert not archivemail.is_older_than_days(time_message=time_msg, + max_days=1) + +########## archivemail.parse_imap_url() unit testing ################# + +class TestParseIMAPUrl(unittest.TestCase): + def setUp(self): + archivemail.options.quiet = True + archivemail.options.verbose = False + archivemail.options.pwfile = None + + urls_withoutpass = [ + ('imap://user@example.org@imap.example.org/upperbox/lowerbox', + ('user', None, 'example.org@imap.example.org', 143, + 'upperbox/lowerbox')), + ('imap://"user@example.org"@imap.example.org/upperbox/lowerbox', + ('user@example.org', None, 'imap.example.org', 143, + 'upperbox/lowerbox')), + ('imap://user@example.org"@imap.example.org/upperbox/lowerbox', + ('user', None, 'example.org"@imap.example.org', 143, + 'upperbox/lowerbox')), + ('imaps://"user@example.org@imap.example.org/upperbox/lowerbox', + ('"user', None, 'example.org@imap.example.org', 993, + 'upperbox/lowerbox')), + ('imaps://"us\\"er@example.org"@imap.example.org/upperbox/lowerbox', + ('us"er@example.org', None, 'imap.example.org', 993, + 'upperbox/lowerbox')), + ('imaps://user\\@example.org@imap.example.org/upperbox/lowerbox', + ('user\\', None, 'example.org@imap.example.org', 993, + 'upperbox/lowerbox')) + ] + urls_withpass = [ + ('imap://user@example.org:passwd@imap.example.org/upperbox/lowerbox', + ('user@example.org', 'passwd', 'imap.example.org', 143, + 'upperbox/lowerbox')), + ('imaps://"user@example.org:passwd@imap.example.org/upperbox/lowerbox', + ('"user@example.org', "passwd", 'imap.example.org', 993, + 'upperbox/lowerbox')), + ('imaps://u\\ser\\@example.org:"p@sswd"@imap.example.org/upperbox/lowerbox', + ('u\\ser\\@example.org', 'p@sswd', 'imap.example.org', 993, + 'upperbox/lowerbox')) + ] + # These are invalid when the password's not stripped. + urls_onlywithpass = [ + ('imaps://"user@example.org":passwd@imap.example.org/upperbox/lowerbox', + ('user@example.org', "passwd", 'imap.example.org', + 'upperbox/lowerbox')) + ] + def testUrlsWithoutPwfile(self): + """Parse test urls with --pwfile option unset. This parses a password in + the URL, if present.""" + archivemail.options.pwfile = None + for mbstr in self.urls_withpass + self.urls_withoutpass: + url = mbstr[0] + result = archivemail.parse_imap_url(url) + self.assertEqual(result, mbstr[1]) + + def testUrlsWithPwfile(self): + """Parse test urls with --pwfile set. In this case the ':' character + loses its meaning as a delimiter.""" + archivemail.options.pwfile = "whocares.txt" + for mbstr in self.urls_onlywithpass: + url = mbstr[0] + self.assertRaises(archivemail.UnexpectedError, + archivemail.parse_imap_url, url) + + def testUrlsDefaultPorts(self): + """If an IMAP URL does not specify a server port, the standard ports + are used.""" + archivemail.options.pwfile = "doesnotexist.txt" + self.assertEqual(143, archivemail.parse_imap_url("imap://user@host/box")[3]) + self.assertEqual(993, archivemail.parse_imap_url("imaps://user@host/box")[3]) + + def testUrlsWithPassAndPortnumber(self): + """IMAP URLs with an embedded password and a server port number are + correctly parsed.""" + self.assertEqual(1234, archivemail.parse_imap_url("imap://user:pass@host:1234/box")[3]) + self.assertEqual(1234, archivemail.parse_imap_url("imap://user:pass@host:1234/box")[3]) + + def tearDown(self): + archivemail.options.quiet = False + archivemail.options.verbose = False + archivemail.options.pwfile = None + +########## quoting and un-quoting of IMAP strings ########## + +class TestIMAPQuoting(unittest.TestCase): + stringlist = ( + ('{braces} and space', '"{braces} and space"'), + ('\\backslash', '"\\\\backslash"'), + ('with "quotes" inbetween', '"with \\"quotes\\" inbetween"'), + ('ending with "quotes"', '"ending with \\"quotes\\""'), + ('\\"backslash before quote', '"\\\\\\"backslash before quote"') + ) + + def testQuote(self): + for unquoted, quoted in self.stringlist: + self.assertEqual(archivemail.imap_quote(unquoted), quoted) + + def testUnquote(self): + for unquoted, quoted in self.stringlist: + self.assertEqual(unquoted, archivemail.imap_unquote(quoted)) + + +########## Modified UTF-7 support functions ########## + +class TestModUTF7(unittest.TestCase): + goodpairs = ( + (u"A\N{NOT IDENTICAL TO}A.", "A&ImI-A."), + (u"Hi Mom -\N{WHITE SMILING FACE}-!", "Hi Mom -&Jjo--!"), + (u"~peter/mail/\u53f0\u5317/\u65e5\u672c\u8a9e", + "~peter/mail/&U,BTFw-/&ZeVnLIqe-") + ) + + def testEncode(self): + """Ensure that encoding text in modified UTF-7 works properly.""" + for text, code in self.goodpairs: + self.assertEqual(archivemail.mod_utf7_encode(text), code) + + def testDecode(self): + """Ensure that decoding modified UTF-7 to text works properly.""" + for text, code in self.goodpairs: + self.assertEqual(archivemail.mod_utf7_decode(code), text) + + +########## acceptance testing ########### + +class TestArchive(TestCaseInTempdir): + """Base class defining helper functions for doing test archiving runs.""" + mbox = None # mbox file that will be processed by archivemail + good_archive = None # Uncompressed reference archive file to verify the + # archive after processing + good_mbox = None # Reference mbox file to verify the mbox after processing + + def verify(self): + assert os.path.exists(self.mbox) + if self.good_mbox is not None: + assertEqualContent(self.mbox, self.good_mbox) + else: + self.assertEqual(os.path.getsize(self.mbox), 0) + archive_name = self.mbox + "_archive" + if not archivemail.options.no_compress: + archive_name += ".gz" + iszipped = True + else: + assert not os.path.exists(archive_name + ".gz") + iszipped = False + if self.good_archive is not None: + assertEqualContent(archive_name, self.good_archive, iszipped) + else: + assert not os.path.exists(archive_name) + + def make_old_mbox(self, body=None, headers=None, messages=1, make_old_archive=False): + """Prepare for a test run with an old mbox by making an old mbox, + optionally an existing archive, and a reference archive to verify the + archive after archivemail has run.""" + self.mbox = make_mbox(body, headers, 181*24, messages) + archive_does_change = not (archivemail.options.dry_run or + archivemail.options.delete_old_mail) + mbox_does_not_change = archivemail.options.dry_run or \ + archivemail.options.copy_old_mail + if make_old_archive: + archive = archivemail.make_archive_name(self.mbox) + self.good_archive = make_archive_and_plain_copy(archive) + if archive_does_change: + append_file(self.mbox, self.good_archive) + elif archive_does_change: + self.good_archive = tempfile.mkstemp()[1] + shutil.copyfile(self.mbox, self.good_archive) + if mbox_does_not_change: + if archive_does_change and not make_old_archive: + self.good_mbox = self.good_archive + else: + self.good_mbox = tempfile.mkstemp()[1] + shutil.copyfile(self.mbox, self.good_mbox) + + def make_mixed_mbox(self, body=None, headers=None, messages=1, make_old_archive=False): + """Prepare for a test run with a mixed mbox by making a mixed mbox, + optionally an existing archive, a reference archive to verify the + archive after archivemail has run, and likewise a reference mbox to + verify the mbox.""" + self.make_old_mbox(body, headers, messages=messages, make_old_archive=make_old_archive) + new_mbox_name = make_mbox(body, headers, 179*24, messages) + append_file(new_mbox_name, self.mbox) + if self.good_mbox is None: + self.good_mbox = new_mbox_name + else: + if self.good_mbox == self.good_archive: + self.good_mbox = tempfile.mkstemp()[1] + shutil.copyfile(self.mbox, self.good_mbox) + else: + append_file(new_mbox_name, self.good_mbox) + + def make_new_mbox(self, body=None, headers=None, messages=1, make_old_archive=False): + """Prepare for a test run with a new mbox by making a new mbox, + optionally an exiting archive, and a reference mbox to verify the mbox + after archivemail has run.""" + self.mbox = make_mbox(body, headers, 179*24, messages) + self.good_mbox = tempfile.mkstemp()[1] + shutil.copyfile(self.mbox, self.good_mbox) + if make_old_archive: + archive = archivemail.make_archive_name(self.mbox) + self.good_archive = make_archive_and_plain_copy(archive) + + +class TestArchiveMbox(TestArchive): + """archiving should work based on the date of messages given""" + + def setUp(self): + self.oldopts = copy.copy(archivemail.options) + archivemail.options.quiet = True + super(TestArchiveMbox, self).setUp() + + def testOld(self): + """archiving an old mailbox""" + self.make_old_mbox(messages=3) + archivemail.archive(self.mbox) + self.verify() + + def testOldFromInBody(self): + """archiving an old mailbox with 'From ' in the body""" + body = """This is a message with ^From at the start of a line +From is on this line +This is after the ^From line""" + self.make_old_mbox(messages=3, body=body) + archivemail.archive(self.mbox) + self.verify() + + def testDateSystem(self): + """test that the --date option works as expected""" + test_headers = ( + { + 'From_' : 'sender@dummy.domain Fri Jul 28 16:11:36 2030', + 'Date' : 'Fri, 28 Jul 2000 16:11:36 +1000', + }, + { + 'From_' : 'sender@dummy.domain Fri Jul 28 16:11:36 2000', + 'Date' : None, + }, + { + 'From_' : 'sender@dummy.domain Fri Jul 28 16:11:36 2030', + 'Date' : None, + 'Delivery-date' : 'Fri, 28 Jul 2000 16:11:36 +1000', + }, + { + 'From_' : 'sender@dummy.domain Fri Jul 28 16:11:36 2030', + 'Date' : None, + 'Resent-Date' : 'Fri, 28 Jul 2000 16:11:36 +1000', + }, + ) + for headers in test_headers: + msg = make_message(default_headers=headers, wantobj=True) + date = time.strptime("2000-07-29", "%Y-%m-%d") + archivemail.options.date_old_max = time.mktime(date) + assert archivemail.should_archive(msg) + date = time.strptime("2000-07-27", "%Y-%m-%d") + archivemail.options.date_old_max = time.mktime(date) + assert not archivemail.should_archive(msg) + + def testMixed(self): + """archiving a mixed mailbox""" + self.make_mixed_mbox(messages=3) + archivemail.archive(self.mbox) + self.verify() + + def testNew(self): + """archiving a new mailbox""" + self.make_new_mbox(messages=3) + archivemail.archive(self.mbox) + self.verify() + + def testOldExisting(self): + """archiving an old mailbox with an existing archive""" + self.make_old_mbox(messages=3, make_old_archive=True) + archivemail.archive(self.mbox) + self.verify() + + def testOldWeirdHeaders(self): + """archiving old mailboxes with weird headers""" + weird_headers = ( + { # we should archive because of the date on the 'From_' line + 'From_' : 'sender@dummy.domain Fri Jul 28 16:11:36 2000', + 'Date' : 'Friskhdfkjkh, 28 Jul 2002 1line noise6:11:36 +1000', + }, + { # we should archive because of the date on the 'From_' line + 'From_' : 'sender@dummy.domain Fri Jul 28 16:11:36 2000', + 'Date' : None, + }, + { # we should archive because of the date in 'Delivery-date' + 'From_' : 'sender@dummy.domain Fri Jul 28 16:11:36 2030', + 'Date' : 'Frcorruptioni, 28 Jul 20line noise00 16:6 +1000', + 'Delivery-date' : 'Fri, 28 Jul 2000 16:11:36 +1000', + }, + { # we should archive because of the date in 'Delivery-date' + 'From_' : 'sender@dummy.domain Fri Jul 28 16:11:36 2030', + 'Date' : None, + 'Delivery-date' : 'Fri, 28 Jul 2000 16:11:36 +1000', + }, + { # we should archive because of the date in 'Resent-Date' + 'From_' : 'sender@dummy.domain Fri Jul 28 16:11:36 2030', + 'Date' : 'Frcorruptioni, 28 Jul 20line noise00 16:6 +1000', + 'Resent-Date' : 'Fri, 28 Jul 2000 16:11:36 +1000', + }, + { # we should archive because of the date in 'Resent-Date' + 'From_' : 'sender@dummy.domain Fri Jul 28 16:11:36 2030', + 'Date' : None, + 'Resent-Date' : 'Fri, 28 Jul 2000 16:11:36 +1000', + }, + { # completely blank dates were crashing < version 0.4.7 + 'From_' : 'sender@dummy.domain Fri Jul 28 16:11:36 2000', + 'Date' : '', + }, + { # completely blank dates were crashing < version 0.4.7 + 'From_' : 'sender@dummy.domain Fri Jul 28 16:11:36 2000', + 'Date' : '', + 'Resent-Date' : '', + }, + ) + fd, self.mbox = tempfile.mkstemp() + fp = os.fdopen(fd, "w") + for headers in weird_headers: + msg_text = make_message(default_headers=headers) + fp.write(msg_text*2) + fp.close() + self.good_archive = tempfile.mkstemp()[1] + shutil.copyfile(self.mbox, self.good_archive) + archivemail.archive(self.mbox) + self.verify() + + def tearDown(self): + archivemail.options = self.oldopts + super(TestArchiveMbox, self).tearDown() + + +class TestArchiveMboxTimestamp(TestCaseInTempdir): + """original mbox timestamps should always be preserved""" + def setUp(self): + super(TestArchiveMboxTimestamp, self).setUp() + archivemail.options.quiet = True + self.mbox_name = make_mbox(messages=3, hours_old=(24 * 180)) + self.mtime = os.path.getmtime(self.mbox_name) - 66 + self.atime = os.path.getatime(self.mbox_name) - 88 + os.utime(self.mbox_name, (self.atime, self.mtime)) + + def testNew(self): + """mbox timestamps should not change after no archival""" + archivemail.options.days_old_max = 181 + archivemail.archive(self.mbox_name) + self.verify() + + def testOld(self): + """mbox timestamps should not change after archival""" + archivemail.options.days_old_max = 179 + archivemail.archive(self.mbox_name) + self.verify() + + def verify(self): + assert os.path.exists(self.mbox_name) + new_atime = os.path.getatime(self.mbox_name) + new_mtime = os.path.getmtime(self.mbox_name) + self.assertAlmostEqual(self.mtime, new_mtime, utimes_precision) + self.assertAlmostEqual(self.atime, new_atime, utimes_precision) + + def tearDown(self): + archivemail.options.quiet = False + archivemail.options.days_old_max = 180 + os.remove(self.mbox_name) + super(TestArchiveMboxTimestamp, self).tearDown() + + +class TestArchiveMboxAll(unittest.TestCase): + def setUp(self): + archivemail.options.quiet = True + archivemail.options.archive_all = True + + def testNew(self): + """new messages should be archived with --all""" + self.msg = make_message(hours_old=24*179, wantobj=True) + assert archivemail.should_archive(self.msg) + + def testOld(self): + """old messages should be archived with --all""" + self.msg = make_message(hours_old=24*181, wantobj=True) + assert archivemail.should_archive(self.msg) + + def tearDown(self): + archivemail.options.quiet = False + archivemail.options.archive_all = False + +class TestArchiveMboxPreserveUnread(unittest.TestCase): + """make sure the 'preserve_unread' option works""" + def setUp(self): + archivemail.options.quiet = True + archivemail.options.preserve_unread = True + self.msg = make_message(hours_old=24*181, wantobj=True) + + def testOldRead(self): + """old read messages should be archived with --preserve-unread""" + self.msg["Status"] = "RO" + assert archivemail.should_archive(self.msg) + + def testOldUnread(self): + """old unread messages should not be archived with --preserve-unread""" + self.msg["Status"] = "O" + assert not archivemail.should_archive(self.msg) + + def tearDown(self): + archivemail.options.quiet = False + archivemail.options.preserve_unread = False + + +class TestArchiveMboxSuffix(unittest.TestCase): + """make sure the 'suffix' option works""" + def setUp(self): + archivemail.options.quiet = True + + def testSuffix(self): + """archiving with specified --suffix arguments""" + for suffix in ("_static_", "_%B_%Y", "-%Y-%m-%d"): + mbox_name = "foobar" + archivemail.options.archive_suffix = suffix + days_old_max = 180 + parsed_suffix_time = time.time() - days_old_max*24*60*60 + parsed_suffix = time.strftime(suffix, + time.localtime(parsed_suffix_time)) + archive_name = mbox_name + parsed_suffix + self.assertEqual(archive_name, + archivemail.make_archive_name(mbox_name)) + + def tearDown(self): + archivemail.options.quiet = False + archivemail.options.archive_suffix = None + +class TestArchiveMboxPrefix(unittest.TestCase): + """make sure the 'prefix' option works""" + def setUp(self): + archivemail.options.quiet = True + + def testPrefix(self): + """archiving with specified --prefix arguments""" + for archive_prefix in ("_static_", "_%B_%Y", "-%Y-%m-%d", "%Y/%m/"): + archivemail.options.archive_prefix = archive_prefix + for mbox_name in "foobar", "/tmp/foobar", "schnorchz/foobar": + archive_dir, archive_base = os.path.split(mbox_name) + days = archivemail.options.days_old_max + tm = time.localtime(time.time() - days*24*60*60) + prefix = time.strftime(archive_prefix, tm) + archive_name = os.path.join(archive_dir, prefix + archive_base) + self.assertEqual(archive_name, + archivemail.make_archive_name(mbox_name)) + + def tearDown(self): + archivemail.options.quiet = False + archivemail.options.archive_prefix = None + +class TestArchiveName(unittest.TestCase): + def setUp(self): + archivemail.options.quiet = True + + def testArchiveName(self): + """test the --archive-name option""" + archive_names = ("custom", ".withdot", "custom_%Y", "%Y/joe") + mbox = "foobar" + for name in archive_names: + archivemail.options.archive_name = name + days = archivemail.options.days_old_max + tm = time.localtime(time.time() - days*24*60*60) + name = time.strftime(name, tm) + self.assertEqual(archivemail.make_archive_name(mbox), name) + + def tearDown(self): + archivemail.options.quiet = False + archivemail.options.archive_name = None + +class TestArchiveAffixes(unittest.TestCase): + def setUp(self): + self.mbox = "harbsch" + self.archive_prefix = "wurbl+" + self.archive_suffix = "+schronk&borsz" + archivemail.options.quiet = True + + def testDefaultPrefix(self): + """if no archive name affix is specified, the default archive suffix is appended""" + self.assertEqual(archivemail.make_archive_name(self.mbox), + self.mbox + archivemail.options.archive_default_suffix) + + def testPrefixKillsDefaultSuffix(self): + """if an archive name prefix is specified, the default archive suffix is not appended""" + archivemail.options.archive_prefix = self.archive_prefix + self.assertEqual(archivemail.make_archive_name(self.mbox), + self.archive_prefix + self.mbox) + + def testPrefixAndSuffix(self): + """specifying both an archive name prefix and suffix works""" + archivemail.options.archive_prefix = self.archive_prefix + archivemail.options.archive_suffix = self.archive_suffix + self.assertEqual(archivemail.make_archive_name(self.mbox), + self.archive_prefix + self.mbox + self.archive_suffix) + + def tearDown(self): + archivemail.options.archive_prefix = None + archivemail.options.archive_suffix = None + archivemail.options.quiet = False + +class TestArchiveHiddenMbox(unittest.TestCase): + def setUp(self): + archivemail.options.quiet = True + self.mbox = ".upper.lower" + + def testHiddenMbox(self): + """leading dots are stripped from the archive name when no prefix is added""" + self.assertEqual(archivemail.make_archive_name(self.mbox), + self.mbox.lstrip('.') + + archivemail.options.archive_default_suffix) + + def testHiddenMboxPrefixedArchive(self): + """no dots are stripped from the archive name when a prefix is added""" + prefix = ".hidden_" + archivemail.options.archive_prefix = prefix + self.assertEqual(archivemail.make_archive_name(self.mbox), + prefix + self.mbox) + + def tearDown(self): + archivemail.options.quiet = False + archivemail.options.archive_prefix = None + +class TestArchiveDryRun(TestArchive): + """make sure the 'dry-run' option works""" + def setUp(self): + super(TestArchiveDryRun, self).setUp() + archivemail.options.quiet = True + archivemail.options.dry_run = True + + def testOld(self): + """archiving an old mailbox with the 'dry-run' option""" + self.make_old_mbox(messages=3) + archivemail.archive(self.mbox) + self.verify() + + def tearDown(self): + archivemail.options.dry_run = False + archivemail.options.quiet = False + super(TestArchiveDryRun, self).tearDown() + + +class TestArchiveDelete(TestArchive): + """make sure the 'delete' option works""" + def setUp(self): + super(TestArchiveDelete, self).setUp() + archivemail.options.quiet = True + archivemail.options.delete_old_mail = True + + def testNew(self): + """archiving a new mailbox with the 'delete' option""" + self.make_new_mbox(messages=3) + archivemail.archive(self.mbox) + self.verify() + + def testMixed(self): + """archiving a mixed mailbox with the 'delete' option""" + self.make_mixed_mbox(messages=3) + archivemail.archive(self.mbox) + self.verify() + + def testOld(self): + """archiving an old mailbox with the 'delete' option""" + self.make_old_mbox(messages=3) + archivemail.archive(self.mbox) + self.verify() + + def tearDown(self): + archivemail.options.delete_old_mail = False + archivemail.options.quiet = False + super(TestArchiveDelete, self).tearDown() + + +class TestArchiveCopy(TestArchive): + """make sure the 'copy' option works""" + def setUp(self): + super(TestArchiveCopy, self).setUp() + archivemail.options.quiet = True + archivemail.options.copy_old_mail = True + + def testNew(self): + """archiving a new mailbox with the 'copy' option""" + self.make_new_mbox(messages=3) + archivemail.archive(self.mbox) + self.verify() + + def testMixed(self): + """archiving a mixed mailbox with the 'copy' option""" + self.make_mixed_mbox(messages=3) + archivemail.archive(self.mbox) + self.verify() + + def testOld(self): + """archiving an old mailbox with the 'copy' option""" + self.make_old_mbox(messages=3) + archivemail.archive(self.mbox) + self.verify() + + def tearDown(self): + archivemail.options.copy_old_mail = False + archivemail.options.quiet = False + super(TestArchiveCopy, self).tearDown() + + +class TestArchiveMboxFlagged(unittest.TestCase): + """make sure the 'include_flagged' option works""" + def setUp(self): + archivemail.options.include_flagged = False + archivemail.options.quiet = True + + def testOld(self): + """by default, old flagged messages should not be archived""" + msg = make_message(default_headers={"X-Status": "F"}, + hours_old=24*181, wantobj=True) + assert not archivemail.should_archive(msg) + + def testIncludeFlaggedNew(self): + """new flagged messages should not be archived with include_flagged""" + msg = make_message(default_headers={"X-Status": "F"}, + hours_old=24*179, wantobj=True) + assert not archivemail.should_archive(msg) + + def testIncludeFlaggedOld(self): + """old flagged messages should be archived with include_flagged""" + archivemail.options.include_flagged = True + msg = make_message(default_headers={"X-Status": "F"}, + hours_old=24*181, wantobj=True) + assert archivemail.should_archive(msg) + + def tearDown(self): + archivemail.options.include_flagged = False + archivemail.options.quiet = False + + +class TestArchiveMboxOutputDir(unittest.TestCase): + """make sure that the 'output-dir' option works""" + def setUp(self): + archivemail.options.quiet = True + + def testOld(self): + """archiving an old mailbox with a sepecified output dir""" + for dir in "/just/a/path", "relative/path": + archivemail.options.output_dir = dir + archive_dir = archivemail.make_archive_name("/tmp/mbox") + self.assertEqual(dir, os.path.dirname(archive_dir)) + + def tearDown(self): + archivemail.options.quiet = False + archivemail.options.output_dir = None + + +class TestArchiveMboxUncompressed(TestArchive): + """make sure that the 'no_compress' option works""" + mbox_name = None + new_mbox = None + old_mbox = None + copy_name = None + + def setUp(self): + archivemail.options.quiet = True + archivemail.options.no_compress = True + super(TestArchiveMboxUncompressed, self).setUp() + + def testOld(self): + """archiving an old mailbox uncompressed""" + self.make_old_mbox(messages=3) + archivemail.archive(self.mbox) + self.verify() + + def testNew(self): + """archiving a new mailbox uncompressed""" + self.make_new_mbox(messages=3) + archivemail.archive(self.mbox) + self.verify() + + def testMixed(self): + """archiving a mixed mailbox uncompressed""" + self.make_mixed_mbox(messages=3) + archivemail.archive(self.mbox) + self.verify() + + def testOldExists(self): + """archiving an old mailbox uncopressed with an existing archive""" + self.make_old_mbox(messages=3, make_old_archive=True) + archivemail.archive(self.mbox) + self.verify() + + def tearDown(self): + archivemail.options.quiet = False + archivemail.options.no_compress = False + super(TestArchiveMboxUncompressed, self).tearDown() + + +class TestArchiveSize(unittest.TestCase): + """check that the 'size' argument works""" + def setUp(self): + archivemail.options.quiet = True + msg_text = make_message(hours_old=24*181) + self.msg_size = len(msg_text) + fp = cStringIO.StringIO(msg_text) + self.msg = rfc822.Message(fp) + + def testSmaller(self): + """giving a size argument smaller than the message""" + archivemail.options.min_size = self.msg_size - 1 + assert archivemail.should_archive(self.msg) + + def testBigger(self): + """giving a size argument bigger than the message""" + archivemail.options.min_size = self.msg_size + 1 + assert not archivemail.should_archive(self.msg) + + def tearDown(self): + archivemail.options.quiet = False + archivemail.options.min_size = None + + +class TestXIMAPMessage(TestArchive): + """Test if IMAP pseudo messages in mboxes are properly handled.""" + def setUp(self): + super(TestXIMAPMessage, self).setUp() + archivemail.options.quiet = True + + def testXIMAPMbox(self): + """IMAP pseudo messages in an mbox are always preserved.""" + self.good_mbox = make_mbox(hours_old=181*24, headers={'X-IMAP': 'dummytext'}, + messages=1) + self.good_archive = make_mbox(hours_old=181*24, messages=3) + self.mbox = tempfile.mkstemp()[-1] + shutil.copyfile(self.good_mbox, self.mbox) + append_file(self.good_archive, self.mbox) + archivemail.archive(self.mbox) + self.verify() + + def tearDown(self): + super(TestXIMAPMessage, self).tearDown() + archivemail.options.quiet = False + + +############# Test archiving maildirs ############### + +class TestArchiveMailboxdir(TestCaseInTempdir): + """Base class defining helper functions for doing test archive runs with + maildirs.""" + maildir = None # Maildir that will be processed by archivemail + orig_maildir_obj = None # A backup copy of the maildir, a SimpleMaildir object + remaining_msg = set() # Filenames of maildir messages that should be preserved + number_archived = 0 # Number of messages that get archived + orig_archive = None # An uncompressed copy of a pre-existing archive, + # if one exists + + def setUp(self): + super(TestArchiveMailboxdir, self).setUp() + self.orig_maildir_obj = SimpleMaildir() + + def verify(self): + self._verify_remaining() + self._verify_archive() + + def _verify_remaining(self): + """Verify that the preserved messages weren't altered.""" + assert self.maildir + # Compare maildir with backup object. + dcmp = filecmp.dircmp(self.maildir, self.orig_maildir_obj.root) + # Top-level has only directories cur, new, tmp and must be unchanged. + self.assertEqual(dcmp.left_list, dcmp.right_list) + found = set() + for d in dcmp.common_dirs: + dcmp2 = dcmp.subdirs[d] + # We need to verify three things. + # 1. directory is a subset of the original... + assert not dcmp2.left_only + # 2. all common files are identical... + self.assertEqual(dcmp2.common_files, dcmp2.same_files) + found = found.union([os.path.join(d, x) for x in dcmp2.common_files]) + # 3. exactly the `new' messages (recorded in self.remaining_msg) + # were preserved. + self.assertEqual(found, self.remaining_msg) + + def _verify_archive(self): + """Verify the archive correctness.""" + # TODO: currently make_archive_name does not include the .gz suffix. + # Is this something that should be fixed? + archive = archivemail.make_archive_name(self.maildir) + if archivemail.options.no_compress: + iszipped = False + else: + archive += '.gz' + iszipped = True + if self.number_archived == 0: + if self.orig_archive: + assertEqualContent(archive, self.orig_archive, iszipped) + else: + assert not os.path.exists(archive) + return + fp_new = fp_archive = tmp_archive_name = None + try: + if self.orig_archive: + new_size = os.path.getsize(archive) + # Brute force: split archive in old and new part and verify the + # parts separately. (Of course this destroys the archive.) + fp_archive = open(archive, "r+") + fp_archive.seek(self.orig_archive_size) + fd, tmp_archive_name = tempfile.mkstemp() + fp_new = os.fdopen(fd, "w") + shutil.copyfileobj(fp_archive, fp_new) + fp_new.close() + fp_archive.truncate(self.orig_archive_size) + fp_archive.close() + assertEqualContent(archive, self.orig_archive, iszipped) + new_archive = tmp_archive_name + else: + new_archive = archive + if archivemail.options.no_compress: + fp_archive = open(new_archive, "r") + else: + fp_archive = FixedGzipFile(new_archive, "r") + mb = mailbox.UnixMailbox(fp_archive) + found = 0 + for msg in mb: + self.verify_maildir_has_msg(self.orig_maildir_obj, msg) + found += 1 + self.assertEqual(found, self.number_archived) + finally: + if tmp_archive_name: + os.remove(tmp_archive_name) + if fp_new is not None: + fp_new.close() + if fp_archive is not None: + fp_archive.close() + + def verify_maildir_has_msg(self, maildir, msg): + """Assert that the given maildir has a copy of the rfc822 message.""" + mid = msg['Message-Id'] # Complains if there is no message-id + mdir_msg_str, mdir_flags = \ + maildir.get_message_and_mbox_status(mid) + mbox_flags = set(msg.get('status', '') + msg.get('x-status', '')) + self.assertEqual(mdir_flags, mbox_flags) + + headers = filter(lambda h: msg.isheader(h) not in ('status', 'x-status'), + msg.headers) + headers = "".join(headers) + msg.rewindbody() + # Discard last mbox LF which is not part of the message. + body = msg.fp.read()[:-1] + msg_str = headers + os.linesep + body + self.assertEqual(mdir_msg_str, msg_str) + + def add_messages(self, body=None, headers=None, hours_old=0, messages=1): + for count in range(messages): + msg = make_message(body, default_headers=headers, mkfrom=False, + hours_old=hours_old) + self.orig_maildir_obj.write(msg, new=False) + + def make_maildir(self, mkold, mknew, body=None, headers=None, messages=1, + make_old_archive=False): + mailbox_does_change = not (archivemail.options.dry_run or + archivemail.options.copy_old_mail) + archive_does_change = not (archivemail.options.dry_run or + archivemail.options.delete_old_mail) + if mknew: + self.add_messages(body, headers, 179*24, messages) + if archive_does_change and archivemail.options.archive_all: + self.number_archived += messages + if mailbox_does_change: + self.remaining_msg = set(self.orig_maildir_obj.get_all_filenames()) + if mkold: + self.add_messages(body, headers, 181*24, messages) + if archive_does_change: + self.number_archived += messages + if not mailbox_does_change: + self.remaining_msg = set(self.orig_maildir_obj.get_all_filenames()) + self.maildir = copy_maildir(self.orig_maildir_obj.root) + if make_old_archive: + archive = archivemail.make_archive_name(self.maildir) + self.orig_archive = make_archive_and_plain_copy(archive) + # FIXME: .gz extension handling is a mess II + if not archivemail.options.no_compress: + archive += '.gz' + self.orig_archive_size = os.path.getsize(archive) + +class TestEmptyMaildir(TestCaseInTempdir): + def setUp(self): + super(TestEmptyMaildir, self).setUp() + archivemail.options.quiet = True + + def testEmpty(self): + """Archiving an empty maildir should not result in an archive.""" + self.mdir = SimpleMaildir() + archivemail.archive(self.mdir.root) + assert not os.path.exists(self.mdir.root + '_archive.gz') + + def tearDown(self): + super(TestEmptyMaildir, self).tearDown() + archivemail.options.quiet = False + +class TestMaildir(TestArchiveMailboxdir): + def setUp(self): + super(TestMaildir, self).setUp() + archivemail.options.quiet = True + + def testOld(self): + self.make_maildir(True, False, messages=3) + archivemail.archive(self.maildir) + self.verify() + + def testNew(self): + self.make_maildir(False, True, messages=3) + archivemail.archive(self.maildir) + self.verify() + + def testMixed(self): + self.make_maildir(True, True, messages=3) + archivemail.archive(self.maildir) + self.verify() + + def testMixedExisting(self): + self.make_maildir(True, True, messages=3, make_old_archive=True) + archivemail.archive(self.maildir) + self.verify() + + def tearDown(self): + archivemail.options.quiet = False + super(TestMaildir, self).tearDown() + + +class TestMaildirPreserveUnread(TestCaseInTempdir): + """Test if the preserve_unread option works with maildirs.""" + def setUp(self): + super(TestMaildirPreserveUnread, self).setUp() + archivemail.options.quiet = True + archivemail.options.preserve_unread = True + + def testOldRead(self): + """--preserve-unread archives old read messages in a maildir.""" + smd = SimpleMaildir("orig") + msg = make_message(hours_old=24*181) + smd.write(msg, new=False, flags='S') + md = mailbox.Maildir(smd.root) + msg_obj = md.next() + assert archivemail.should_archive(msg_obj) + + def testOldUnread(self): + """--preserve-unread preserves old unread messages in a maildir.""" + smd = SimpleMaildir("orig") + msg = make_message(hours_old=24*181) + smd.write(msg, new=False) + md = mailbox.Maildir(smd.root) + msg_obj = md.next() + assert not archivemail.should_archive(msg_obj) + + def tearDown(self): + archivemail.options.quiet = False + archivemail.options.preserve_unread = False + super(TestMaildirPreserveUnread, self).tearDown() + +class TestMaildirAll(TestArchiveMailboxdir): + def setUp(self): + super(TestMaildirAll, self).setUp() + archivemail.options.quiet = True + archivemail.options.archive_all = True + + def testNew(self): + """New maildir messages should be archived with --all""" + self.add_messages(hours_old=24*181) + md = mailbox.Maildir(self.orig_maildir_obj.root) + msg_obj = md.next() + assert archivemail.should_archive(msg_obj) + + def testOld(self): + """Old maildir messages should be archived with --all""" + self.add_messages(hours_old=24*179) + md = mailbox.Maildir(self.orig_maildir_obj.root) + msg_obj = md.next() + assert archivemail.should_archive(msg_obj) + + def tearDown(self): + super(TestMaildirAll, self).tearDown() + archivemail.options.quiet = False + archivemail.options.archive_all = False + +class TestMaildirDryRun(TestArchiveMailboxdir): + def setUp(self): + super(TestMaildirDryRun, self).setUp() + archivemail.options.quiet = True + archivemail.options.dry_run = True + + def testOld(self): + """archiving an old maildir mailbox with the 'dry-run' option""" + self.make_maildir(True, False) + archivemail.archive(self.maildir) + self.verify() + + def tearDown(self): + super(TestMaildirDryRun, self).tearDown() + archivemail.options.quiet = False + archivemail.options.dry_run = False + +class TestMaildirDelete(TestArchiveMailboxdir): + def setUp(self): + super(TestMaildirDelete, self).setUp() + archivemail.options.quiet = True + archivemail.options.delete_old_mail = True + + def testOld(self): + """archiving an old maildir mailbox with the 'delete' option""" + self.make_maildir(True, False) + archivemail.archive(self.maildir) + self.verify() + + def testNew(self): + """archiving a new maildir mailbox with the 'delete' option""" + self.make_maildir(False, True) + archivemail.archive(self.maildir) + self.verify() + + def tearDown(self): + super(TestMaildirDelete, self).tearDown() + archivemail.options.quiet = False + archivemail.options.delete_old_mail = False + +class TestMaildirCopy(TestArchiveMailboxdir): + def setUp(self): + super(TestMaildirCopy, self).setUp() + archivemail.options.quiet = True + archivemail.options.copy_old_mail = True + + def testOld(self): + """archiving an old maildir mailbox with the 'copy' option""" + self.make_maildir(True, False) + archivemail.archive(self.maildir) + self.verify() + + def testNew(self): + """archiving a new maildir mailbox with the 'copy' option""" + self.make_maildir(False, True) + archivemail.archive(self.maildir) + self.verify() + + def tearDown(self): + super(TestMaildirCopy, self).tearDown() + archivemail.options.quiet = False + archivemail.options.copy_old_mail = False + +class TestArchiveMaildirFlagged(TestCaseInTempdir): + """make sure the 'include_flagged' option works with maildir messages""" + def setUp(self): + super(TestArchiveMaildirFlagged, self).setUp() + archivemail.options.include_flagged = False + archivemail.options.quiet = True + + def testOld(self): + """by default, old flagged maildir messages should not be archived""" + smd = SimpleMaildir("orig") + msg = make_message(hours_old=24*181) + smd.write(msg, new=False, flags='F') + md = mailbox.Maildir(smd.root) + msg_obj = md.next() + assert not archivemail.should_archive(msg_obj) + + def testIncludeFlaggedNew(self): + """new flagged maildir messages should not be archived with include_flagged""" + smd = SimpleMaildir("orig") + msg = make_message(hours_old=24*179) + smd.write(msg, new=False, flags='F') + md = mailbox.Maildir(smd.root) + msg_obj = md.next() + assert not archivemail.should_archive(msg_obj) + + def testIncludeFlaggedOld(self): + """old flagged maildir messages should be archived with include_flagged""" + archivemail.options.include_flagged = True + smd = SimpleMaildir("orig") + msg = make_message(hours_old=24*181) + smd.write(msg, new=False, flags='F') + md = mailbox.Maildir(smd.root) + msg_obj = md.next() + assert archivemail.should_archive(msg_obj) + + def tearDown(self): + super(TestArchiveMaildirFlagged, self).tearDown() + archivemail.options.include_flagged = False + archivemail.options.quiet = False + +class TestArchiveMaildirSize(TestCaseInTempdir): + """check that the 'size' argument works with maildir messages""" + def setUp(self): + super(TestArchiveMaildirSize, self).setUp() + archivemail.options.quiet = True + msg = make_message(hours_old=24*181) + self.msg_size = len(msg) + smd = SimpleMaildir("orig") + smd.write(msg, new=False) + md = mailbox.Maildir(smd.root) + self.msg_obj = md.next() + + def testSmaller(self): + """giving a size argument smaller than the maildir message""" + archivemail.options.min_size = self.msg_size - 1 + assert archivemail.should_archive(self.msg_obj) + + def testBigger(self): + """giving a size argument bigger than the maildir message""" + archivemail.options.min_size = self.msg_size + 1 + assert not archivemail.should_archive(self.msg_obj) + + def tearDown(self): + super(TestArchiveMaildirSize, self).tearDown() + archivemail.options.quiet = False + archivemail.options.min_size = None + +########## helper routines ############ + +def make_message(body=None, default_headers={}, hours_old=None, mkfrom=False, wantobj=False): + headers = copy.copy(default_headers) + if not headers: + headers = {} + headers['Message-Id'] = make_msgid() + if not headers.has_key('Date'): + time_message = time.time() - (60 * 60 * hours_old) + headers['Date'] = time.asctime(time.localtime(time_message)) + if not headers.has_key('From'): + headers['From'] = "sender@dummy.domain" + if not headers.has_key('To'): + headers['To'] = "receipient@dummy.domain" + if not headers.has_key('Subject'): + headers['Subject'] = "This is the subject" + if mkfrom and not headers.has_key('From_'): + headers['From_'] = "%s %s" % (headers['From'], headers['Date']) + if not body: + body = "This is the message body" + + msg = "" + if headers.has_key('From_'): + msg = msg + ("From %s\n" % headers['From_']) + del headers['From_'] + for key in headers.keys(): + if headers[key] is not None: + msg = msg + ("%s: %s\n" % (key, headers[key])) + msg = msg + "\n\n" + body + "\n\n" + if not wantobj: + return msg + fp = cStringIO.StringIO(msg) + return rfc822.Message(fp) + +def append_file(source, dest): + """appends the file named 'source' to the file named 'dest'""" + assert os.path.isfile(source) + assert os.path.isfile(dest) + read = open(source, "r") + write = open(dest, "a+") + shutil.copyfileobj(read,write) + read.close() + write.close() + + +def make_mbox(body=None, headers=None, hours_old=0, messages=1): + assert tempfile.tempdir + fd, name = tempfile.mkstemp() + file = os.fdopen(fd, "w") + for count in range(messages): + msg = make_message(body=body, default_headers=headers, + mkfrom=True, hours_old=hours_old) + file.write(msg) + file.close() + return name + +def make_archive_and_plain_copy(archive_name): + """Make an mbox archive of the given name like archivemail may have + created it. Also make an uncompressed copy of this archive and return its + name.""" + copy_fd, copy_name = tempfile.mkstemp() + copy_fp = os.fdopen(copy_fd, "w") + if archivemail.options.no_compress: + fd = os.open(archive_name, os.O_WRONLY|os.O_EXCL|os.O_CREAT) + fp = os.fdopen(fd, "w") + else: + archive_name += ".gz" + fd = os.open(archive_name, os.O_WRONLY|os.O_EXCL|os.O_CREAT) + rawfp = os.fdopen(fd, "w") + fp = gzip.GzipFile(fileobj=rawfp) + for count in range(3): + msg = make_message(hours_old=24*360) + fp.write(msg) + copy_fp.write(msg) + fp.close() + copy_fp.close() + if not archivemail.options.no_compress: + rawfp.close() + return copy_name + +def copy_maildir(maildir, prefix="tmp"): + """Create a copy of the given maildir and return the absolute path of the + new direcory.""" + newdir = tempfile.mkdtemp(prefix=prefix) + for d in "cur", "new", "tmp": + shutil.copytree(os.path.join(maildir, d), os.path.join(newdir, d)) + return newdir + +def assertEqualContent(firstfile, secondfile, zippedfirst=False): + """Verify that the two files exist and have identical content. If zippedfirst + is True, assume that firstfile is gzip-compressed.""" + assert os.path.exists(firstfile) + assert os.path.exists(secondfile) + if zippedfirst: + try: + fp1 = gzip.GzipFile(firstfile, "r") + fp2 = open(secondfile, "r") + assert cmp_fileobj(fp1, fp2) + finally: + fp1.close() + fp2.close() + else: + assert filecmp.cmp(firstfile, secondfile, shallow=0) + +def cmp_fileobj(fp1, fp2): + """Return if reading the fileobjects yields identical content.""" + bufsize = 8192 + while True: + b1 = fp1.read(bufsize) + b2 = fp2.read(bufsize) + if b1 != b2: + return False + if not b1: + return True + +if __name__ == "__main__": + unittest.main()