1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38 """
39 Provides an extension to back up mbox email files.
40
41 Backing up email
42 ================
43
44 Email folders (often stored as mbox flatfiles) are not well-suited being backed
45 up with an incremental backup like the one offered by Cedar Backup. This is
46 because mbox files often change on a daily basis, forcing the incremental
47 backup process to back them up every day in order to avoid losing data. This
48 can result in quite a bit of wasted space when backing up large folders. (Note
49 that the alternative maildir format does not share this problem, since it
50 typically uses one file per message.)
51
52 One solution to this problem is to design a smarter incremental backup process,
53 which backs up baseline content on the first day of the week, and then backs up
54 only new messages added to that folder on every other day of the week. This way,
55 the backup for any single day is only as large as the messages placed into the
56 folder on that day. The backup isn't as "perfect" as the incremental backup
57 process, because it doesn't preserve information about messages deleted from
58 the backed-up folder. However, it should be much more space-efficient, and
59 in a recovery situation, it seems better to restore too much data rather
60 than too little.
61
62 What is this extension?
63 =======================
64
65 This is a Cedar Backup extension used to back up mbox email files via the Cedar
66 Backup command line. Individual mbox files or directories containing mbox
67 files can be backed up using the same collect modes allowed for filesystems in
68 the standard Cedar Backup collect action: weekly, daily, incremental. It
69 implements the "smart" incremental backup process discussed above, using
70 functionality provided by the C{grepmail} utility.
71
72 This extension requires a new configuration section <mbox> and is intended to
73 be run either immediately before or immediately after the standard collect
74 action. Aside from its own configuration, it requires the options and collect
75 configuration sections in the standard Cedar Backup configuration file.
76
77 The mbox action is conceptually similar to the standard collect action,
78 except that mbox directories are not collected recursively. This implies
79 some configuration changes (i.e. there's no need for global exclusions or an
80 ignore file). If you back up a directory, all of the mbox files in that
81 directory are backed up into a single tar file using the indicated
82 compression method.
83
84 @author: Kenneth J. Pronovici <pronovic@ieee.org>
85 """
86
87
88
89
90
91
92 import os
93 import logging
94 import datetime
95 import pickle
96 import tempfile
97 from bz2 import BZ2File
98 from gzip import GzipFile
99 from functools import total_ordering
100
101
102 from CedarBackup3.filesystem import FilesystemList, BackupFileList
103 from CedarBackup3.xmlutil import createInputDom, addContainerNode, addStringNode
104 from CedarBackup3.xmlutil import isElement, readChildren, readFirstChild, readString, readStringList
105 from CedarBackup3.config import VALID_COLLECT_MODES, VALID_COMPRESS_MODES
106 from CedarBackup3.util import isStartOfWeek, buildNormalizedPath
107 from CedarBackup3.util import resolveCommand, executeCommand
108 from CedarBackup3.util import ObjectTypeList, UnorderedList, RegexList, encodePath, changeOwnership
109
110
111
112
113
114
115 logger = logging.getLogger("CedarBackup3.log.extend.mbox")
116
117 GREPMAIL_COMMAND = [ "grepmail", ]
118 REVISION_PATH_EXTENSION = "mboxlast"
119
120
121
122
123
124
125 @total_ordering
126 -class MboxFile(object):
127
128 """
129 Class representing mbox file configuration..
130
131 The following restrictions exist on data in this class:
132
133 - The absolute path must be absolute.
134 - The collect mode must be one of the values in L{VALID_COLLECT_MODES}.
135 - The compress mode must be one of the values in L{VALID_COMPRESS_MODES}.
136
137 @sort: __init__, __repr__, __str__, __cmp__, __eq__, __lt__, __gt__,
138 absolutePath, collectMode, compressMode
139 """
140
141 - def __init__(self, absolutePath=None, collectMode=None, compressMode=None):
142 """
143 Constructor for the C{MboxFile} class.
144
145 You should never directly instantiate this class.
146
147 @param absolutePath: Absolute path to an mbox file on disk.
148 @param collectMode: Overridden collect mode for this directory.
149 @param compressMode: Overridden compression mode for this directory.
150 """
151 self._absolutePath = None
152 self._collectMode = None
153 self._compressMode = None
154 self.absolutePath = absolutePath
155 self.collectMode = collectMode
156 self.compressMode = compressMode
157
163
165 """
166 Informal string representation for class instance.
167 """
168 return self.__repr__()
169
171 """Equals operator, iplemented in terms of original Python 2 compare operator."""
172 return self.__cmp__(other) == 0
173
175 """Less-than operator, iplemented in terms of original Python 2 compare operator."""
176 return self.__cmp__(other) < 0
177
179 """Greater-than operator, iplemented in terms of original Python 2 compare operator."""
180 return self.__cmp__(other) > 0
181
206
208 """
209 Property target used to set the absolute path.
210 The value must be an absolute path if it is not C{None}.
211 It does not have to exist on disk at the time of assignment.
212 @raise ValueError: If the value is not an absolute path.
213 @raise ValueError: If the value cannot be encoded properly.
214 """
215 if value is not None:
216 if not os.path.isabs(value):
217 raise ValueError("Absolute path must be, er, an absolute path.")
218 self._absolutePath = encodePath(value)
219
221 """
222 Property target used to get the absolute path.
223 """
224 return self._absolutePath
225
227 """
228 Property target used to set the collect mode.
229 If not C{None}, the mode must be one of the values in L{VALID_COLLECT_MODES}.
230 @raise ValueError: If the value is not valid.
231 """
232 if value is not None:
233 if value not in VALID_COLLECT_MODES:
234 raise ValueError("Collect mode must be one of %s." % VALID_COLLECT_MODES)
235 self._collectMode = value
236
238 """
239 Property target used to get the collect mode.
240 """
241 return self._collectMode
242
244 """
245 Property target used to set the compress mode.
246 If not C{None}, the mode must be one of the values in L{VALID_COMPRESS_MODES}.
247 @raise ValueError: If the value is not valid.
248 """
249 if value is not None:
250 if value not in VALID_COMPRESS_MODES:
251 raise ValueError("Compress mode must be one of %s." % VALID_COMPRESS_MODES)
252 self._compressMode = value
253
255 """
256 Property target used to get the compress mode.
257 """
258 return self._compressMode
259
260 absolutePath = property(_getAbsolutePath, _setAbsolutePath, None, doc="Absolute path to the mbox file.")
261 collectMode = property(_getCollectMode, _setCollectMode, None, doc="Overridden collect mode for this mbox file.")
262 compressMode = property(_getCompressMode, _setCompressMode, None, doc="Overridden compress mode for this mbox file.")
263
264
265
266
267
268
269 @total_ordering
270 -class MboxDir(object):
271
272 """
273 Class representing mbox directory configuration..
274
275 The following restrictions exist on data in this class:
276
277 - The absolute path must be absolute.
278 - The collect mode must be one of the values in L{VALID_COLLECT_MODES}.
279 - The compress mode must be one of the values in L{VALID_COMPRESS_MODES}.
280
281 Unlike collect directory configuration, this is the only place exclusions
282 are allowed (no global exclusions at the <mbox> configuration level). Also,
283 we only allow relative exclusions and there is no configured ignore file.
284 This is because mbox directory backups are not recursive.
285
286 @sort: __init__, __repr__, __str__, __cmp__, __eq__, __lt__, __gt__,
287 absolutePath, collectMode, compressMode, relativeExcludePaths,
288 excludePatterns
289 """
290
291 - def __init__(self, absolutePath=None, collectMode=None, compressMode=None,
292 relativeExcludePaths=None, excludePatterns=None):
293 """
294 Constructor for the C{MboxDir} class.
295
296 You should never directly instantiate this class.
297
298 @param absolutePath: Absolute path to a mbox file on disk.
299 @param collectMode: Overridden collect mode for this directory.
300 @param compressMode: Overridden compression mode for this directory.
301 @param relativeExcludePaths: List of relative paths to exclude.
302 @param excludePatterns: List of regular expression patterns to exclude
303 """
304 self._absolutePath = None
305 self._collectMode = None
306 self._compressMode = None
307 self._relativeExcludePaths = None
308 self._excludePatterns = None
309 self.absolutePath = absolutePath
310 self.collectMode = collectMode
311 self.compressMode = compressMode
312 self.relativeExcludePaths = relativeExcludePaths
313 self.excludePatterns = excludePatterns
314
321
323 """
324 Informal string representation for class instance.
325 """
326 return self.__repr__()
327
329 """Equals operator, iplemented in terms of original Python 2 compare operator."""
330 return self.__cmp__(other) == 0
331
333 """Less-than operator, iplemented in terms of original Python 2 compare operator."""
334 return self.__cmp__(other) < 0
335
337 """Greater-than operator, iplemented in terms of original Python 2 compare operator."""
338 return self.__cmp__(other) > 0
339
374
376 """
377 Property target used to set the absolute path.
378 The value must be an absolute path if it is not C{None}.
379 It does not have to exist on disk at the time of assignment.
380 @raise ValueError: If the value is not an absolute path.
381 @raise ValueError: If the value cannot be encoded properly.
382 """
383 if value is not None:
384 if not os.path.isabs(value):
385 raise ValueError("Absolute path must be, er, an absolute path.")
386 self._absolutePath = encodePath(value)
387
389 """
390 Property target used to get the absolute path.
391 """
392 return self._absolutePath
393
395 """
396 Property target used to set the collect mode.
397 If not C{None}, the mode must be one of the values in L{VALID_COLLECT_MODES}.
398 @raise ValueError: If the value is not valid.
399 """
400 if value is not None:
401 if value not in VALID_COLLECT_MODES:
402 raise ValueError("Collect mode must be one of %s." % VALID_COLLECT_MODES)
403 self._collectMode = value
404
406 """
407 Property target used to get the collect mode.
408 """
409 return self._collectMode
410
412 """
413 Property target used to set the compress mode.
414 If not C{None}, the mode must be one of the values in L{VALID_COMPRESS_MODES}.
415 @raise ValueError: If the value is not valid.
416 """
417 if value is not None:
418 if value not in VALID_COMPRESS_MODES:
419 raise ValueError("Compress mode must be one of %s." % VALID_COMPRESS_MODES)
420 self._compressMode = value
421
423 """
424 Property target used to get the compress mode.
425 """
426 return self._compressMode
427
429 """
430 Property target used to set the relative exclude paths list.
431 Elements do not have to exist on disk at the time of assignment.
432 """
433 if value is None:
434 self._relativeExcludePaths = None
435 else:
436 try:
437 saved = self._relativeExcludePaths
438 self._relativeExcludePaths = UnorderedList()
439 self._relativeExcludePaths.extend(value)
440 except Exception as e:
441 self._relativeExcludePaths = saved
442 raise e
443
445 """
446 Property target used to get the relative exclude paths list.
447 """
448 return self._relativeExcludePaths
449
451 """
452 Property target used to set the exclude patterns list.
453 """
454 if value is None:
455 self._excludePatterns = None
456 else:
457 try:
458 saved = self._excludePatterns
459 self._excludePatterns = RegexList()
460 self._excludePatterns.extend(value)
461 except Exception as e:
462 self._excludePatterns = saved
463 raise e
464
466 """
467 Property target used to get the exclude patterns list.
468 """
469 return self._excludePatterns
470
471 absolutePath = property(_getAbsolutePath, _setAbsolutePath, None, doc="Absolute path to the mbox directory.")
472 collectMode = property(_getCollectMode, _setCollectMode, None, doc="Overridden collect mode for this mbox directory.")
473 compressMode = property(_getCompressMode, _setCompressMode, None, doc="Overridden compress mode for this mbox directory.")
474 relativeExcludePaths = property(_getRelativeExcludePaths, _setRelativeExcludePaths, None, "List of relative paths to exclude.")
475 excludePatterns = property(_getExcludePatterns, _setExcludePatterns, None, "List of regular expression patterns to exclude.")
476
477
478
479
480
481
482 @total_ordering
483 -class MboxConfig(object):
484
485 """
486 Class representing mbox configuration.
487
488 Mbox configuration is used for backing up mbox email files.
489
490 The following restrictions exist on data in this class:
491
492 - The collect mode must be one of the values in L{VALID_COLLECT_MODES}.
493 - The compress mode must be one of the values in L{VALID_COMPRESS_MODES}.
494 - The C{mboxFiles} list must be a list of C{MboxFile} objects
495 - The C{mboxDirs} list must be a list of C{MboxDir} objects
496
497 For the C{mboxFiles} and C{mboxDirs} lists, validation is accomplished
498 through the L{util.ObjectTypeList} list implementation that overrides common
499 list methods and transparently ensures that each element is of the proper
500 type.
501
502 Unlike collect configuration, no global exclusions are allowed on this
503 level. We only allow relative exclusions at the mbox directory level.
504 Also, there is no configured ignore file. This is because mbox directory
505 backups are not recursive.
506
507 @note: Lists within this class are "unordered" for equality comparisons.
508
509 @sort: __init__, __repr__, __str__, __cmp__, __eq__, __lt__, __gt__,
510 collectMode, compressMode, mboxFiles, mboxDirs
511 """
512
513 - def __init__(self, collectMode=None, compressMode=None, mboxFiles=None, mboxDirs=None):
514 """
515 Constructor for the C{MboxConfig} class.
516
517 @param collectMode: Default collect mode.
518 @param compressMode: Default compress mode.
519 @param mboxFiles: List of mbox files to back up
520 @param mboxDirs: List of mbox directories to back up
521
522 @raise ValueError: If one of the values is invalid.
523 """
524 self._collectMode = None
525 self._compressMode = None
526 self._mboxFiles = None
527 self._mboxDirs = None
528 self.collectMode = collectMode
529 self.compressMode = compressMode
530 self.mboxFiles = mboxFiles
531 self.mboxDirs = mboxDirs
532
538
540 """
541 Informal string representation for class instance.
542 """
543 return self.__repr__()
544
546 """Equals operator, iplemented in terms of original Python 2 compare operator."""
547 return self.__cmp__(other) == 0
548
550 """Less-than operator, iplemented in terms of original Python 2 compare operator."""
551 return self.__cmp__(other) < 0
552
554 """Greater-than operator, iplemented in terms of original Python 2 compare operator."""
555 return self.__cmp__(other) > 0
556
558 """
559 Original Python 2 comparison operator.
560 Lists within this class are "unordered" for equality comparisons.
561 @param other: Other object to compare to.
562 @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other.
563 """
564 if other is None:
565 return 1
566 if self.collectMode != other.collectMode:
567 if str(self.collectMode or "") < str(other.collectMode or ""):
568 return -1
569 else:
570 return 1
571 if self.compressMode != other.compressMode:
572 if str(self.compressMode or "") < str(other.compressMode or ""):
573 return -1
574 else:
575 return 1
576 if self.mboxFiles != other.mboxFiles:
577 if self.mboxFiles < other.mboxFiles:
578 return -1
579 else:
580 return 1
581 if self.mboxDirs != other.mboxDirs:
582 if self.mboxDirs < other.mboxDirs:
583 return -1
584 else:
585 return 1
586 return 0
587
589 """
590 Property target used to set the collect mode.
591 If not C{None}, the mode must be one of the values in L{VALID_COLLECT_MODES}.
592 @raise ValueError: If the value is not valid.
593 """
594 if value is not None:
595 if value not in VALID_COLLECT_MODES:
596 raise ValueError("Collect mode must be one of %s." % VALID_COLLECT_MODES)
597 self._collectMode = value
598
600 """
601 Property target used to get the collect mode.
602 """
603 return self._collectMode
604
606 """
607 Property target used to set the compress mode.
608 If not C{None}, the mode must be one of the values in L{VALID_COMPRESS_MODES}.
609 @raise ValueError: If the value is not valid.
610 """
611 if value is not None:
612 if value not in VALID_COMPRESS_MODES:
613 raise ValueError("Compress mode must be one of %s." % VALID_COMPRESS_MODES)
614 self._compressMode = value
615
617 """
618 Property target used to get the compress mode.
619 """
620 return self._compressMode
621
623 """
624 Property target used to set the mboxFiles list.
625 Either the value must be C{None} or each element must be an C{MboxFile}.
626 @raise ValueError: If the value is not an C{MboxFile}
627 """
628 if value is None:
629 self._mboxFiles = None
630 else:
631 try:
632 saved = self._mboxFiles
633 self._mboxFiles = ObjectTypeList(MboxFile, "MboxFile")
634 self._mboxFiles.extend(value)
635 except Exception as e:
636 self._mboxFiles = saved
637 raise e
638
640 """
641 Property target used to get the mboxFiles list.
642 """
643 return self._mboxFiles
644
646 """
647 Property target used to set the mboxDirs list.
648 Either the value must be C{None} or each element must be an C{MboxDir}.
649 @raise ValueError: If the value is not an C{MboxDir}
650 """
651 if value is None:
652 self._mboxDirs = None
653 else:
654 try:
655 saved = self._mboxDirs
656 self._mboxDirs = ObjectTypeList(MboxDir, "MboxDir")
657 self._mboxDirs.extend(value)
658 except Exception as e:
659 self._mboxDirs = saved
660 raise e
661
663 """
664 Property target used to get the mboxDirs list.
665 """
666 return self._mboxDirs
667
668 collectMode = property(_getCollectMode, _setCollectMode, None, doc="Default collect mode.")
669 compressMode = property(_getCompressMode, _setCompressMode, None, doc="Default compress mode.")
670 mboxFiles = property(_getMboxFiles, _setMboxFiles, None, doc="List of mbox files to back up.")
671 mboxDirs = property(_getMboxDirs, _setMboxDirs, None, doc="List of mbox directories to back up.")
672
673
674
675
676
677
678 @total_ordering
679 -class LocalConfig(object):
680
681 """
682 Class representing this extension's configuration document.
683
684 This is not a general-purpose configuration object like the main Cedar
685 Backup configuration object. Instead, it just knows how to parse and emit
686 Mbox-specific configuration values. Third parties who need to read and
687 write configuration related to this extension should access it through the
688 constructor, C{validate} and C{addConfig} methods.
689
690 @note: Lists within this class are "unordered" for equality comparisons.
691
692 @sort: __init__, __repr__, __str__, __cmp__, __eq__, __lt__, __gt__, mbox,
693 validate, addConfig
694 """
695
696 - def __init__(self, xmlData=None, xmlPath=None, validate=True):
697 """
698 Initializes a configuration object.
699
700 If you initialize the object without passing either C{xmlData} or
701 C{xmlPath} then configuration will be empty and will be invalid until it
702 is filled in properly.
703
704 No reference to the original XML data or original path is saved off by
705 this class. Once the data has been parsed (successfully or not) this
706 original information is discarded.
707
708 Unless the C{validate} argument is C{False}, the L{LocalConfig.validate}
709 method will be called (with its default arguments) against configuration
710 after successfully parsing any passed-in XML. Keep in mind that even if
711 C{validate} is C{False}, it might not be possible to parse the passed-in
712 XML document if lower-level validations fail.
713
714 @note: It is strongly suggested that the C{validate} option always be set
715 to C{True} (the default) unless there is a specific need to read in
716 invalid configuration from disk.
717
718 @param xmlData: XML data representing configuration.
719 @type xmlData: String data.
720
721 @param xmlPath: Path to an XML file on disk.
722 @type xmlPath: Absolute path to a file on disk.
723
724 @param validate: Validate the document after parsing it.
725 @type validate: Boolean true/false.
726
727 @raise ValueError: If both C{xmlData} and C{xmlPath} are passed-in.
728 @raise ValueError: If the XML data in C{xmlData} or C{xmlPath} cannot be parsed.
729 @raise ValueError: If the parsed configuration document is not valid.
730 """
731 self._mbox = None
732 self.mbox = None
733 if xmlData is not None and xmlPath is not None:
734 raise ValueError("Use either xmlData or xmlPath, but not both.")
735 if xmlData is not None:
736 self._parseXmlData(xmlData)
737 if validate:
738 self.validate()
739 elif xmlPath is not None:
740 xmlData = open(xmlPath).read()
741 self._parseXmlData(xmlData)
742 if validate:
743 self.validate()
744
746 """
747 Official string representation for class instance.
748 """
749 return "LocalConfig(%s)" % (self.mbox)
750
752 """
753 Informal string representation for class instance.
754 """
755 return self.__repr__()
756
758 """Equals operator, iplemented in terms of original Python 2 compare operator."""
759 return self.__cmp__(other) == 0
760
762 """Less-than operator, iplemented in terms of original Python 2 compare operator."""
763 return self.__cmp__(other) < 0
764
766 """Greater-than operator, iplemented in terms of original Python 2 compare operator."""
767 return self.__cmp__(other) > 0
768
770 """
771 Original Python 2 comparison operator.
772 Lists within this class are "unordered" for equality comparisons.
773 @param other: Other object to compare to.
774 @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other.
775 """
776 if other is None:
777 return 1
778 if self.mbox != other.mbox:
779 if self.mbox < other.mbox:
780 return -1
781 else:
782 return 1
783 return 0
784
786 """
787 Property target used to set the mbox configuration value.
788 If not C{None}, the value must be a C{MboxConfig} object.
789 @raise ValueError: If the value is not a C{MboxConfig}
790 """
791 if value is None:
792 self._mbox = None
793 else:
794 if not isinstance(value, MboxConfig):
795 raise ValueError("Value must be a C{MboxConfig} object.")
796 self._mbox = value
797
799 """
800 Property target used to get the mbox configuration value.
801 """
802 return self._mbox
803
804 mbox = property(_getMbox, _setMbox, None, "Mbox configuration in terms of a C{MboxConfig} object.")
805
807 """
808 Validates configuration represented by the object.
809
810 Mbox configuration must be filled in. Within that, the collect mode and
811 compress mode are both optional, but the list of repositories must
812 contain at least one entry.
813
814 Each configured file or directory must contain an absolute path, and then
815 must be either able to take collect mode and compress mode configuration
816 from the parent C{MboxConfig} object, or must set each value on its own.
817
818 @raise ValueError: If one of the validations fails.
819 """
820 if self.mbox is None:
821 raise ValueError("Mbox section is required.")
822 if (self.mbox.mboxFiles is None or len(self.mbox.mboxFiles) < 1) and \
823 (self.mbox.mboxDirs is None or len(self.mbox.mboxDirs) < 1):
824 raise ValueError("At least one mbox file or directory must be configured.")
825 if self.mbox.mboxFiles is not None:
826 for mboxFile in self.mbox.mboxFiles:
827 if mboxFile.absolutePath is None:
828 raise ValueError("Each mbox file must set an absolute path.")
829 if self.mbox.collectMode is None and mboxFile.collectMode is None:
830 raise ValueError("Collect mode must either be set in parent mbox section or individual mbox file.")
831 if self.mbox.compressMode is None and mboxFile.compressMode is None:
832 raise ValueError("Compress mode must either be set in parent mbox section or individual mbox file.")
833 if self.mbox.mboxDirs is not None:
834 for mboxDir in self.mbox.mboxDirs:
835 if mboxDir.absolutePath is None:
836 raise ValueError("Each mbox directory must set an absolute path.")
837 if self.mbox.collectMode is None and mboxDir.collectMode is None:
838 raise ValueError("Collect mode must either be set in parent mbox section or individual mbox directory.")
839 if self.mbox.compressMode is None and mboxDir.compressMode is None:
840 raise ValueError("Compress mode must either be set in parent mbox section or individual mbox directory.")
841
843 """
844 Adds an <mbox> configuration section as the next child of a parent.
845
846 Third parties should use this function to write configuration related to
847 this extension.
848
849 We add the following fields to the document::
850
851 collectMode //cb_config/mbox/collectMode
852 compressMode //cb_config/mbox/compressMode
853
854 We also add groups of the following items, one list element per
855 item::
856
857 mboxFiles //cb_config/mbox/file
858 mboxDirs //cb_config/mbox/dir
859
860 The mbox files and mbox directories are added by L{_addMboxFile} and
861 L{_addMboxDir}.
862
863 @param xmlDom: DOM tree as from C{impl.createDocument()}.
864 @param parentNode: Parent that the section should be appended to.
865 """
866 if self.mbox is not None:
867 sectionNode = addContainerNode(xmlDom, parentNode, "mbox")
868 addStringNode(xmlDom, sectionNode, "collect_mode", self.mbox.collectMode)
869 addStringNode(xmlDom, sectionNode, "compress_mode", self.mbox.compressMode)
870 if self.mbox.mboxFiles is not None:
871 for mboxFile in self.mbox.mboxFiles:
872 LocalConfig._addMboxFile(xmlDom, sectionNode, mboxFile)
873 if self.mbox.mboxDirs is not None:
874 for mboxDir in self.mbox.mboxDirs:
875 LocalConfig._addMboxDir(xmlDom, sectionNode, mboxDir)
876
878 """
879 Internal method to parse an XML string into the object.
880
881 This method parses the XML document into a DOM tree (C{xmlDom}) and then
882 calls a static method to parse the mbox configuration section.
883
884 @param xmlData: XML data to be parsed
885 @type xmlData: String data
886
887 @raise ValueError: If the XML cannot be successfully parsed.
888 """
889 (xmlDom, parentNode) = createInputDom(xmlData)
890 self._mbox = LocalConfig._parseMbox(parentNode)
891
892 @staticmethod
894 """
895 Parses an mbox configuration section.
896
897 We read the following individual fields::
898
899 collectMode //cb_config/mbox/collect_mode
900 compressMode //cb_config/mbox/compress_mode
901
902 We also read groups of the following item, one list element per
903 item::
904
905 mboxFiles //cb_config/mbox/file
906 mboxDirs //cb_config/mbox/dir
907
908 The mbox files are parsed by L{_parseMboxFiles} and the mbox
909 directories are parsed by L{_parseMboxDirs}.
910
911 @param parent: Parent node to search beneath.
912
913 @return: C{MboxConfig} object or C{None} if the section does not exist.
914 @raise ValueError: If some filled-in value is invalid.
915 """
916 mbox = None
917 section = readFirstChild(parent, "mbox")
918 if section is not None:
919 mbox = MboxConfig()
920 mbox.collectMode = readString(section, "collect_mode")
921 mbox.compressMode = readString(section, "compress_mode")
922 mbox.mboxFiles = LocalConfig._parseMboxFiles(section)
923 mbox.mboxDirs = LocalConfig._parseMboxDirs(section)
924 return mbox
925
926 @staticmethod
928 """
929 Reads a list of C{MboxFile} objects from immediately beneath the parent.
930
931 We read the following individual fields::
932
933 absolutePath abs_path
934 collectMode collect_mode
935 compressMode compess_mode
936
937 @param parent: Parent node to search beneath.
938
939 @return: List of C{MboxFile} objects or C{None} if none are found.
940 @raise ValueError: If some filled-in value is invalid.
941 """
942 lst = []
943 for entry in readChildren(parent, "file"):
944 if isElement(entry):
945 mboxFile = MboxFile()
946 mboxFile.absolutePath = readString(entry, "abs_path")
947 mboxFile.collectMode = readString(entry, "collect_mode")
948 mboxFile.compressMode = readString(entry, "compress_mode")
949 lst.append(mboxFile)
950 if lst == []:
951 lst = None
952 return lst
953
954 @staticmethod
956 """
957 Reads a list of C{MboxDir} objects from immediately beneath the parent.
958
959 We read the following individual fields::
960
961 absolutePath abs_path
962 collectMode collect_mode
963 compressMode compess_mode
964
965 We also read groups of the following items, one list element per
966 item::
967
968 relativeExcludePaths exclude/rel_path
969 excludePatterns exclude/pattern
970
971 The exclusions are parsed by L{_parseExclusions}.
972
973 @param parent: Parent node to search beneath.
974
975 @return: List of C{MboxDir} objects or C{None} if none are found.
976 @raise ValueError: If some filled-in value is invalid.
977 """
978 lst = []
979 for entry in readChildren(parent, "dir"):
980 if isElement(entry):
981 mboxDir = MboxDir()
982 mboxDir.absolutePath = readString(entry, "abs_path")
983 mboxDir.collectMode = readString(entry, "collect_mode")
984 mboxDir.compressMode = readString(entry, "compress_mode")
985 (mboxDir.relativeExcludePaths, mboxDir.excludePatterns) = LocalConfig._parseExclusions(entry)
986 lst.append(mboxDir)
987 if lst == []:
988 lst = None
989 return lst
990
991 @staticmethod
993 """
994 Reads exclusions data from immediately beneath the parent.
995
996 We read groups of the following items, one list element per item::
997
998 relative exclude/rel_path
999 patterns exclude/pattern
1000
1001 If there are none of some pattern (i.e. no relative path items) then
1002 C{None} will be returned for that item in the tuple.
1003
1004 @param parentNode: Parent node to search beneath.
1005
1006 @return: Tuple of (relative, patterns) exclusions.
1007 """
1008 section = readFirstChild(parentNode, "exclude")
1009 if section is None:
1010 return (None, None)
1011 else:
1012 relative = readStringList(section, "rel_path")
1013 patterns = readStringList(section, "pattern")
1014 return (relative, patterns)
1015
1016 @staticmethod
1018 """
1019 Adds an mbox file container as the next child of a parent.
1020
1021 We add the following fields to the document::
1022
1023 absolutePath file/abs_path
1024 collectMode file/collect_mode
1025 compressMode file/compress_mode
1026
1027 The <file> node itself is created as the next child of the parent node.
1028 This method only adds one mbox file node. The parent must loop for each
1029 mbox file in the C{MboxConfig} object.
1030
1031 If C{mboxFile} is C{None}, this method call will be a no-op.
1032
1033 @param xmlDom: DOM tree as from C{impl.createDocument()}.
1034 @param parentNode: Parent that the section should be appended to.
1035 @param mboxFile: MboxFile to be added to the document.
1036 """
1037 if mboxFile is not None:
1038 sectionNode = addContainerNode(xmlDom, parentNode, "file")
1039 addStringNode(xmlDom, sectionNode, "abs_path", mboxFile.absolutePath)
1040 addStringNode(xmlDom, sectionNode, "collect_mode", mboxFile.collectMode)
1041 addStringNode(xmlDom, sectionNode, "compress_mode", mboxFile.compressMode)
1042
1043 @staticmethod
1045 """
1046 Adds an mbox directory container as the next child of a parent.
1047
1048 We add the following fields to the document::
1049
1050 absolutePath dir/abs_path
1051 collectMode dir/collect_mode
1052 compressMode dir/compress_mode
1053
1054 We also add groups of the following items, one list element per item::
1055
1056 relativeExcludePaths dir/exclude/rel_path
1057 excludePatterns dir/exclude/pattern
1058
1059 The <dir> node itself is created as the next child of the parent node.
1060 This method only adds one mbox directory node. The parent must loop for
1061 each mbox directory in the C{MboxConfig} object.
1062
1063 If C{mboxDir} is C{None}, this method call will be a no-op.
1064
1065 @param xmlDom: DOM tree as from C{impl.createDocument()}.
1066 @param parentNode: Parent that the section should be appended to.
1067 @param mboxDir: MboxDir to be added to the document.
1068 """
1069 if mboxDir is not None:
1070 sectionNode = addContainerNode(xmlDom, parentNode, "dir")
1071 addStringNode(xmlDom, sectionNode, "abs_path", mboxDir.absolutePath)
1072 addStringNode(xmlDom, sectionNode, "collect_mode", mboxDir.collectMode)
1073 addStringNode(xmlDom, sectionNode, "compress_mode", mboxDir.compressMode)
1074 if ((mboxDir.relativeExcludePaths is not None and mboxDir.relativeExcludePaths != []) or
1075 (mboxDir.excludePatterns is not None and mboxDir.excludePatterns != [])):
1076 excludeNode = addContainerNode(xmlDom, sectionNode, "exclude")
1077 if mboxDir.relativeExcludePaths is not None:
1078 for relativePath in mboxDir.relativeExcludePaths:
1079 addStringNode(xmlDom, excludeNode, "rel_path", relativePath)
1080 if mboxDir.excludePatterns is not None:
1081 for pattern in mboxDir.excludePatterns:
1082 addStringNode(xmlDom, excludeNode, "pattern", pattern)
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093 -def executeAction(configPath, options, config):
1094 """
1095 Executes the mbox backup action.
1096
1097 @param configPath: Path to configuration file on disk.
1098 @type configPath: String representing a path on disk.
1099
1100 @param options: Program command-line options.
1101 @type options: Options object.
1102
1103 @param config: Program configuration.
1104 @type config: Config object.
1105
1106 @raise ValueError: Under many generic error conditions
1107 @raise IOError: If a backup could not be written for some reason.
1108 """
1109 logger.debug("Executing mbox extended action.")
1110 newRevision = datetime.datetime.today()
1111 if config.options is None or config.collect is None:
1112 raise ValueError("Cedar Backup configuration is not properly filled in.")
1113 local = LocalConfig(xmlPath=configPath)
1114 todayIsStart = isStartOfWeek(config.options.startingDay)
1115 fullBackup = options.full or todayIsStart
1116 logger.debug("Full backup flag is [%s]", fullBackup)
1117 if local.mbox.mboxFiles is not None:
1118 for mboxFile in local.mbox.mboxFiles:
1119 logger.debug("Working with mbox file [%s]", mboxFile.absolutePath)
1120 collectMode = _getCollectMode(local, mboxFile)
1121 compressMode = _getCompressMode(local, mboxFile)
1122 lastRevision = _loadLastRevision(config, mboxFile, fullBackup, collectMode)
1123 if fullBackup or (collectMode in ['daily', 'incr', ]) or (collectMode == 'weekly' and todayIsStart):
1124 logger.debug("Mbox file meets criteria to be backed up today.")
1125 _backupMboxFile(config, mboxFile.absolutePath, fullBackup,
1126 collectMode, compressMode, lastRevision, newRevision)
1127 else:
1128 logger.debug("Mbox file will not be backed up, per collect mode.")
1129 if collectMode == 'incr':
1130 _writeNewRevision(config, mboxFile, newRevision)
1131 if local.mbox.mboxDirs is not None:
1132 for mboxDir in local.mbox.mboxDirs:
1133 logger.debug("Working with mbox directory [%s]", mboxDir.absolutePath)
1134 collectMode = _getCollectMode(local, mboxDir)
1135 compressMode = _getCompressMode(local, mboxDir)
1136 lastRevision = _loadLastRevision(config, mboxDir, fullBackup, collectMode)
1137 (excludePaths, excludePatterns) = _getExclusions(mboxDir)
1138 if fullBackup or (collectMode in ['daily', 'incr', ]) or (collectMode == 'weekly' and todayIsStart):
1139 logger.debug("Mbox directory meets criteria to be backed up today.")
1140 _backupMboxDir(config, mboxDir.absolutePath,
1141 fullBackup, collectMode, compressMode,
1142 lastRevision, newRevision,
1143 excludePaths, excludePatterns)
1144 else:
1145 logger.debug("Mbox directory will not be backed up, per collect mode.")
1146 if collectMode == 'incr':
1147 _writeNewRevision(config, mboxDir, newRevision)
1148 logger.info("Executed the mbox extended action successfully.")
1149
1151 """
1152 Gets the collect mode that should be used for an mbox file or directory.
1153 Use file- or directory-specific value if possible, otherwise take from mbox section.
1154 @param local: LocalConfig object.
1155 @param item: Mbox file or directory
1156 @return: Collect mode to use.
1157 """
1158 if item.collectMode is None:
1159 collectMode = local.mbox.collectMode
1160 else:
1161 collectMode = item.collectMode
1162 logger.debug("Collect mode is [%s]", collectMode)
1163 return collectMode
1164
1166 """
1167 Gets the compress mode that should be used for an mbox file or directory.
1168 Use file- or directory-specific value if possible, otherwise take from mbox section.
1169 @param local: LocalConfig object.
1170 @param item: Mbox file or directory
1171 @return: Compress mode to use.
1172 """
1173 if item.compressMode is None:
1174 compressMode = local.mbox.compressMode
1175 else:
1176 compressMode = item.compressMode
1177 logger.debug("Compress mode is [%s]", compressMode)
1178 return compressMode
1179
1181 """
1182 Gets the path to the revision file associated with a repository.
1183 @param config: Cedar Backup configuration.
1184 @param item: Mbox file or directory
1185 @return: Absolute path to the revision file associated with the repository.
1186 """
1187 normalized = buildNormalizedPath(item.absolutePath)
1188 filename = "%s.%s" % (normalized, REVISION_PATH_EXTENSION)
1189 revisionPath = os.path.join(config.options.workingDir, filename)
1190 logger.debug("Revision file path is [%s]", revisionPath)
1191 return revisionPath
1192
1194 """
1195 Loads the last revision date for this item from disk and returns it.
1196
1197 If this is a full backup, or if the revision file cannot be loaded for some
1198 reason, then C{None} is returned. This indicates that there is no previous
1199 revision, so the entire mail file or directory should be backed up.
1200
1201 @note: We write the actual revision object to disk via pickle, so we don't
1202 deal with the datetime precision or format at all. Whatever's in the object
1203 is what we write.
1204
1205 @param config: Cedar Backup configuration.
1206 @param item: Mbox file or directory
1207 @param fullBackup: Indicates whether this is a full backup
1208 @param collectMode: Indicates the collect mode for this item
1209
1210 @return: Revision date as a datetime.datetime object or C{None}.
1211 """
1212 revisionPath = _getRevisionPath(config, item)
1213 if fullBackup:
1214 revisionDate = None
1215 logger.debug("Revision file ignored because this is a full backup.")
1216 elif collectMode in ['weekly', 'daily']:
1217 revisionDate = None
1218 logger.debug("No revision file based on collect mode [%s].", collectMode)
1219 else:
1220 logger.debug("Revision file will be used for non-full incremental backup.")
1221 if not os.path.isfile(revisionPath):
1222 revisionDate = None
1223 logger.debug("Revision file [%s] does not exist on disk.", revisionPath)
1224 else:
1225 try:
1226 revisionDate = pickle.load(open(revisionPath, "r"))
1227 logger.debug("Loaded revision file [%s] from disk: [%s]", revisionPath, revisionDate)
1228 except:
1229 revisionDate = None
1230 logger.error("Failed loading revision file [%s] from disk.", revisionPath)
1231 return revisionDate
1232
1234 """
1235 Writes new revision information to disk.
1236
1237 If we can't write the revision file successfully for any reason, we'll log
1238 the condition but won't throw an exception.
1239
1240 @note: We write the actual revision object to disk via pickle, so we don't
1241 deal with the datetime precision or format at all. Whatever's in the object
1242 is what we write.
1243
1244 @param config: Cedar Backup configuration.
1245 @param item: Mbox file or directory
1246 @param newRevision: Revision date as a datetime.datetime object.
1247 """
1248 revisionPath = _getRevisionPath(config, item)
1249 try:
1250 pickle.dump(newRevision, open(revisionPath, "w"))
1251 changeOwnership(revisionPath, config.options.backupUser, config.options.backupGroup)
1252 logger.debug("Wrote new revision file [%s] to disk: [%s]", revisionPath, newRevision)
1253 except:
1254 logger.error("Failed to write revision file [%s] to disk.", revisionPath)
1255
1257 """
1258 Gets exclusions (file and patterns) associated with an mbox directory.
1259
1260 The returned files value is a list of absolute paths to be excluded from the
1261 backup for a given directory. It is derived from the mbox directory's
1262 relative exclude paths.
1263
1264 The returned patterns value is a list of patterns to be excluded from the
1265 backup for a given directory. It is derived from the mbox directory's list
1266 of patterns.
1267
1268 @param mboxDir: Mbox directory object.
1269
1270 @return: Tuple (files, patterns) indicating what to exclude.
1271 """
1272 paths = []
1273 if mboxDir.relativeExcludePaths is not None:
1274 for relativePath in mboxDir.relativeExcludePaths:
1275 paths.append(os.path.join(mboxDir.absolutePath, relativePath))
1276 patterns = []
1277 if mboxDir.excludePatterns is not None:
1278 patterns.extend(mboxDir.excludePatterns)
1279 logger.debug("Exclude paths: %s", paths)
1280 logger.debug("Exclude patterns: %s", patterns)
1281 return(paths, patterns)
1282
1283 -def _getBackupPath(config, mboxPath, compressMode, newRevision, targetDir=None):
1284 """
1285 Gets the backup file path (including correct extension) associated with an mbox path.
1286
1287 We assume that if the target directory is passed in, that we're backing up a
1288 directory. Under these circumstances, we'll just use the basename of the
1289 individual path as the output file.
1290
1291 @note: The backup path only contains the current date in YYYYMMDD format,
1292 but that's OK because the index information (stored elsewhere) is the actual
1293 date object.
1294
1295 @param config: Cedar Backup configuration.
1296 @param mboxPath: Path to the indicated mbox file or directory
1297 @param compressMode: Compress mode to use for this mbox path
1298 @param newRevision: Revision this backup path represents
1299 @param targetDir: Target directory in which the path should exist
1300
1301 @return: Absolute path to the backup file associated with the repository.
1302 """
1303 if targetDir is None:
1304 normalizedPath = buildNormalizedPath(mboxPath)
1305 revisionDate = newRevision.strftime("%Y%m%d")
1306 filename = "mbox-%s-%s" % (revisionDate, normalizedPath)
1307 else:
1308 filename = os.path.basename(mboxPath)
1309 if compressMode == 'gzip':
1310 filename = "%s.gz" % filename
1311 elif compressMode == 'bzip2':
1312 filename = "%s.bz2" % filename
1313 if targetDir is None:
1314 backupPath = os.path.join(config.collect.targetDir, filename)
1315 else:
1316 backupPath = os.path.join(targetDir, filename)
1317 logger.debug("Backup file path is [%s]", backupPath)
1318 return backupPath
1319
1321 """
1322 Gets the tarfile backup file path (including correct extension) associated
1323 with an mbox path.
1324
1325 Along with the path, the tar archive mode is returned in a form that can
1326 be used with L{BackupFileList.generateTarfile}.
1327
1328 @note: The tarfile path only contains the current date in YYYYMMDD format,
1329 but that's OK because the index information (stored elsewhere) is the actual
1330 date object.
1331
1332 @param config: Cedar Backup configuration.
1333 @param mboxPath: Path to the indicated mbox file or directory
1334 @param compressMode: Compress mode to use for this mbox path
1335 @param newRevision: Revision this backup path represents
1336
1337 @return: Tuple of (absolute path to tarfile, tar archive mode)
1338 """
1339 normalizedPath = buildNormalizedPath(mboxPath)
1340 revisionDate = newRevision.strftime("%Y%m%d")
1341 filename = "mbox-%s-%s.tar" % (revisionDate, normalizedPath)
1342 if compressMode == 'gzip':
1343 filename = "%s.gz" % filename
1344 archiveMode = "targz"
1345 elif compressMode == 'bzip2':
1346 filename = "%s.bz2" % filename
1347 archiveMode = "tarbz2"
1348 else:
1349 archiveMode = "tar"
1350 tarfilePath = os.path.join(config.collect.targetDir, filename)
1351 logger.debug("Tarfile path is [%s]", tarfilePath)
1352 return (tarfilePath, archiveMode)
1353
1355 """
1356 Opens the output file used for saving backup information.
1357
1358 If the compress mode is "gzip", we'll open a C{GzipFile}, and if the
1359 compress mode is "bzip2", we'll open a C{BZ2File}. Otherwise, we'll just
1360 return an object from the normal C{open()} method.
1361
1362 @param backupPath: Path to file to open.
1363 @param compressMode: Compress mode of file ("none", "gzip", "bzip").
1364
1365 @return: Output file object.
1366 """
1367 if compressMode == "gzip":
1368 return GzipFile(backupPath, "w")
1369 elif compressMode == "bzip2":
1370 return BZ2File(backupPath, "w")
1371 else:
1372 return open(backupPath, "w")
1373
1374 -def _backupMboxFile(config, absolutePath,
1375 fullBackup, collectMode, compressMode,
1376 lastRevision, newRevision, targetDir=None):
1377 """
1378 Backs up an individual mbox file.
1379
1380 @param config: Cedar Backup configuration.
1381 @param absolutePath: Path to mbox file to back up.
1382 @param fullBackup: Indicates whether this should be a full backup.
1383 @param collectMode: Indicates the collect mode for this item
1384 @param compressMode: Compress mode of file ("none", "gzip", "bzip")
1385 @param lastRevision: Date of last backup as datetime.datetime
1386 @param newRevision: Date of new (current) backup as datetime.datetime
1387 @param targetDir: Target directory to write the backed-up file into
1388
1389 @raise ValueError: If some value is missing or invalid.
1390 @raise IOError: If there is a problem backing up the mbox file.
1391 """
1392 backupPath = _getBackupPath(config, absolutePath, compressMode, newRevision, targetDir=targetDir)
1393 outputFile = _getOutputFile(backupPath, compressMode)
1394 if fullBackup or collectMode != "incr" or lastRevision is None:
1395 args = [ "-a", "-u", absolutePath, ]
1396 else:
1397 revisionDate = lastRevision.strftime("%Y-%m-%dT%H:%M:%S")
1398 args = [ "-a", "-u", "-d", "since %s" % revisionDate, absolutePath, ]
1399 command = resolveCommand(GREPMAIL_COMMAND)
1400 result = executeCommand(command, args, returnOutput=False, ignoreStderr=True, doNotLog=True, outputFile=outputFile)[0]
1401 if result != 0:
1402 raise IOError("Error [%d] executing grepmail on [%s]." % (result, absolutePath))
1403 logger.debug("Completed backing up mailbox [%s].", absolutePath)
1404 return backupPath
1405
1406 -def _backupMboxDir(config, absolutePath,
1407 fullBackup, collectMode, compressMode,
1408 lastRevision, newRevision,
1409 excludePaths, excludePatterns):
1410 """
1411 Backs up a directory containing mbox files.
1412
1413 @param config: Cedar Backup configuration.
1414 @param absolutePath: Path to mbox directory to back up.
1415 @param fullBackup: Indicates whether this should be a full backup.
1416 @param collectMode: Indicates the collect mode for this item
1417 @param compressMode: Compress mode of file ("none", "gzip", "bzip")
1418 @param lastRevision: Date of last backup as datetime.datetime
1419 @param newRevision: Date of new (current) backup as datetime.datetime
1420 @param excludePaths: List of absolute paths to exclude.
1421 @param excludePatterns: List of patterns to exclude.
1422
1423 @raise ValueError: If some value is missing or invalid.
1424 @raise IOError: If there is a problem backing up the mbox file.
1425 """
1426 try:
1427 tmpdir = tempfile.mkdtemp(dir=config.options.workingDir)
1428 mboxList = FilesystemList()
1429 mboxList.excludeDirs = True
1430 mboxList.excludePaths = excludePaths
1431 mboxList.excludePatterns = excludePatterns
1432 mboxList.addDirContents(absolutePath, recursive=False)
1433 tarList = BackupFileList()
1434 for item in mboxList:
1435 backupPath = _backupMboxFile(config, item, fullBackup,
1436 collectMode, "none",
1437 lastRevision, newRevision,
1438 targetDir=tmpdir)
1439 tarList.addFile(backupPath)
1440 (tarfilePath, archiveMode) = _getTarfilePath(config, absolutePath, compressMode, newRevision)
1441 tarList.generateTarfile(tarfilePath, archiveMode, ignore=True, flat=True)
1442 changeOwnership(tarfilePath, config.options.backupUser, config.options.backupGroup)
1443 logger.debug("Completed backing up directory [%s].", absolutePath)
1444 finally:
1445 try:
1446 for cleanitem in tarList:
1447 if os.path.exists(cleanitem):
1448 try:
1449 os.remove(cleanitem)
1450 except: pass
1451 except: pass
1452 try:
1453 os.rmdir(tmpdir)
1454 except: pass
1455