1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38 """
39 Provides an extension to back up mbox email files.
40
41 Backing up email
42 ================
43
44 Email folders (often stored as mbox flatfiles) are not well-suited being backed
45 up with an incremental backup like the one offered by Cedar Backup. This is
46 because mbox files often change on a daily basis, forcing the incremental
47 backup process to back them up every day in order to avoid losing data. This
48 can result in quite a bit of wasted space when backing up large folders. (Note
49 that the alternative maildir format does not share this problem, since it
50 typically uses one file per message.)
51
52 One solution to this problem is to design a smarter incremental backup process,
53 which backs up baseline content on the first day of the week, and then backs up
54 only new messages added to that folder on every other day of the week. This way,
55 the backup for any single day is only as large as the messages placed into the
56 folder on that day. The backup isn't as "perfect" as the incremental backup
57 process, because it doesn't preserve information about messages deleted from
58 the backed-up folder. However, it should be much more space-efficient, and
59 in a recovery situation, it seems better to restore too much data rather
60 than too little.
61
62 What is this extension?
63 =======================
64
65 This is a Cedar Backup extension used to back up mbox email files via the Cedar
66 Backup command line. Individual mbox files or directories containing mbox
67 files can be backed up using the same collect modes allowed for filesystems in
68 the standard Cedar Backup collect action: weekly, daily, incremental. It
69 implements the "smart" incremental backup process discussed above, using
70 functionality provided by the C{grepmail} utility.
71
72 This extension requires a new configuration section <mbox> and is intended to
73 be run either immediately before or immediately after the standard collect
74 action. Aside from its own configuration, it requires the options and collect
75 configuration sections in the standard Cedar Backup configuration file.
76
77 The mbox action is conceptually similar to the standard collect action,
78 except that mbox directories are not collected recursively. This implies
79 some configuration changes (i.e. there's no need for global exclusions or an
80 ignore file). If you back up a directory, all of the mbox files in that
81 directory are backed up into a single tar file using the indicated
82 compression method.
83
84 @author: Kenneth J. Pronovici <pronovic@ieee.org>
85 """
86
87
88
89
90
91
92 import os
93 import logging
94 import datetime
95 import pickle
96 import tempfile
97 from bz2 import BZ2File
98 from gzip import GzipFile
99 from functools import total_ordering
100
101
102 from CedarBackup3.filesystem import FilesystemList, BackupFileList
103 from CedarBackup3.xmlutil import createInputDom, addContainerNode, addStringNode
104 from CedarBackup3.xmlutil import isElement, readChildren, readFirstChild, readString, readStringList
105 from CedarBackup3.config import VALID_COLLECT_MODES, VALID_COMPRESS_MODES
106 from CedarBackup3.util import isStartOfWeek, buildNormalizedPath
107 from CedarBackup3.util import resolveCommand, executeCommand
108 from CedarBackup3.util import ObjectTypeList, UnorderedList, RegexList, encodePath, changeOwnership
109
110
111
112
113
114
115 logger = logging.getLogger("CedarBackup3.log.extend.mbox")
116
117 GREPMAIL_COMMAND = [ "grepmail", ]
118 REVISION_PATH_EXTENSION = "mboxlast"
119
120
121
122
123
124
125 @total_ordering
126 -class MboxFile(object):
127
128 """
129 Class representing mbox file configuration..
130
131 The following restrictions exist on data in this class:
132
133 - The absolute path must be absolute.
134 - The collect mode must be one of the values in L{VALID_COLLECT_MODES}.
135 - The compress mode must be one of the values in L{VALID_COMPRESS_MODES}.
136
137 @sort: __init__, __repr__, __str__, __cmp__, __eq__, __lt__, __gt__,
138 absolutePath, collectMode, compressMode
139 """
140
141 - def __init__(self, absolutePath=None, collectMode=None, compressMode=None):
142 """
143 Constructor for the C{MboxFile} class.
144
145 You should never directly instantiate this class.
146
147 @param absolutePath: Absolute path to an mbox file on disk.
148 @param collectMode: Overridden collect mode for this directory.
149 @param compressMode: Overridden compression mode for this directory.
150 """
151 self._absolutePath = None
152 self._collectMode = None
153 self._compressMode = None
154 self.absolutePath = absolutePath
155 self.collectMode = collectMode
156 self.compressMode = compressMode
157
163
165 """
166 Informal string representation for class instance.
167 """
168 return self.__repr__()
169
171 """Equals operator, iplemented in terms of original Python 2 compare operator."""
172 return self.__cmp__(other) == 0
173
175 """Less-than operator, iplemented in terms of original Python 2 compare operator."""
176 return self.__cmp__(other) < 0
177
179 """Greater-than operator, iplemented in terms of original Python 2 compare operator."""
180 return self.__cmp__(other) > 0
181
206
208 """
209 Property target used to set the absolute path.
210 The value must be an absolute path if it is not C{None}.
211 It does not have to exist on disk at the time of assignment.
212 @raise ValueError: If the value is not an absolute path.
213 @raise ValueError: If the value cannot be encoded properly.
214 """
215 if value is not None:
216 if not os.path.isabs(value):
217 raise ValueError("Absolute path must be, er, an absolute path.")
218 self._absolutePath = encodePath(value)
219
221 """
222 Property target used to get the absolute path.
223 """
224 return self._absolutePath
225
227 """
228 Property target used to set the collect mode.
229 If not C{None}, the mode must be one of the values in L{VALID_COLLECT_MODES}.
230 @raise ValueError: If the value is not valid.
231 """
232 if value is not None:
233 if value not in VALID_COLLECT_MODES:
234 raise ValueError("Collect mode must be one of %s." % VALID_COLLECT_MODES)
235 self._collectMode = value
236
238 """
239 Property target used to get the collect mode.
240 """
241 return self._collectMode
242
244 """
245 Property target used to set the compress mode.
246 If not C{None}, the mode must be one of the values in L{VALID_COMPRESS_MODES}.
247 @raise ValueError: If the value is not valid.
248 """
249 if value is not None:
250 if value not in VALID_COMPRESS_MODES:
251 raise ValueError("Compress mode must be one of %s." % VALID_COMPRESS_MODES)
252 self._compressMode = value
253
255 """
256 Property target used to get the compress mode.
257 """
258 return self._compressMode
259
260 absolutePath = property(_getAbsolutePath, _setAbsolutePath, None, doc="Absolute path to the mbox file.")
261 collectMode = property(_getCollectMode, _setCollectMode, None, doc="Overridden collect mode for this mbox file.")
262 compressMode = property(_getCompressMode, _setCompressMode, None, doc="Overridden compress mode for this mbox file.")
263
264
265
266
267
268
269 @total_ordering
270 -class MboxDir(object):
271
272 """
273 Class representing mbox directory configuration..
274
275 The following restrictions exist on data in this class:
276
277 - The absolute path must be absolute.
278 - The collect mode must be one of the values in L{VALID_COLLECT_MODES}.
279 - The compress mode must be one of the values in L{VALID_COMPRESS_MODES}.
280
281 Unlike collect directory configuration, this is the only place exclusions
282 are allowed (no global exclusions at the <mbox> configuration level). Also,
283 we only allow relative exclusions and there is no configured ignore file.
284 This is because mbox directory backups are not recursive.
285
286 @sort: __init__, __repr__, __str__, __cmp__, __eq__, __lt__, __gt__,
287 absolutePath, collectMode, compressMode, relativeExcludePaths,
288 excludePatterns
289 """
290
291 - def __init__(self, absolutePath=None, collectMode=None, compressMode=None,
292 relativeExcludePaths=None, excludePatterns=None):
293 """
294 Constructor for the C{MboxDir} class.
295
296 You should never directly instantiate this class.
297
298 @param absolutePath: Absolute path to a mbox file on disk.
299 @param collectMode: Overridden collect mode for this directory.
300 @param compressMode: Overridden compression mode for this directory.
301 @param relativeExcludePaths: List of relative paths to exclude.
302 @param excludePatterns: List of regular expression patterns to exclude
303 """
304 self._absolutePath = None
305 self._collectMode = None
306 self._compressMode = None
307 self._relativeExcludePaths = None
308 self._excludePatterns = None
309 self.absolutePath = absolutePath
310 self.collectMode = collectMode
311 self.compressMode = compressMode
312 self.relativeExcludePaths = relativeExcludePaths
313 self.excludePatterns = excludePatterns
314
321
323 """
324 Informal string representation for class instance.
325 """
326 return self.__repr__()
327
329 """Equals operator, iplemented in terms of original Python 2 compare operator."""
330 return self.__cmp__(other) == 0
331
333 """Less-than operator, iplemented in terms of original Python 2 compare operator."""
334 return self.__cmp__(other) < 0
335
337 """Greater-than operator, iplemented in terms of original Python 2 compare operator."""
338 return self.__cmp__(other) > 0
339
374
376 """
377 Property target used to set the absolute path.
378 The value must be an absolute path if it is not C{None}.
379 It does not have to exist on disk at the time of assignment.
380 @raise ValueError: If the value is not an absolute path.
381 @raise ValueError: If the value cannot be encoded properly.
382 """
383 if value is not None:
384 if not os.path.isabs(value):
385 raise ValueError("Absolute path must be, er, an absolute path.")
386 self._absolutePath = encodePath(value)
387
389 """
390 Property target used to get the absolute path.
391 """
392 return self._absolutePath
393
395 """
396 Property target used to set the collect mode.
397 If not C{None}, the mode must be one of the values in L{VALID_COLLECT_MODES}.
398 @raise ValueError: If the value is not valid.
399 """
400 if value is not None:
401 if value not in VALID_COLLECT_MODES:
402 raise ValueError("Collect mode must be one of %s." % VALID_COLLECT_MODES)
403 self._collectMode = value
404
406 """
407 Property target used to get the collect mode.
408 """
409 return self._collectMode
410
412 """
413 Property target used to set the compress mode.
414 If not C{None}, the mode must be one of the values in L{VALID_COMPRESS_MODES}.
415 @raise ValueError: If the value is not valid.
416 """
417 if value is not None:
418 if value not in VALID_COMPRESS_MODES:
419 raise ValueError("Compress mode must be one of %s." % VALID_COMPRESS_MODES)
420 self._compressMode = value
421
423 """
424 Property target used to get the compress mode.
425 """
426 return self._compressMode
427
429 """
430 Property target used to set the relative exclude paths list.
431 Elements do not have to exist on disk at the time of assignment.
432 """
433 if value is None:
434 self._relativeExcludePaths = None
435 else:
436 try:
437 saved = self._relativeExcludePaths
438 self._relativeExcludePaths = UnorderedList()
439 self._relativeExcludePaths.extend(value)
440 except Exception as e:
441 self._relativeExcludePaths = saved
442 raise e
443
445 """
446 Property target used to get the relative exclude paths list.
447 """
448 return self._relativeExcludePaths
449
451 """
452 Property target used to set the exclude patterns list.
453 """
454 if value is None:
455 self._excludePatterns = None
456 else:
457 try:
458 saved = self._excludePatterns
459 self._excludePatterns = RegexList()
460 self._excludePatterns.extend(value)
461 except Exception as e:
462 self._excludePatterns = saved
463 raise e
464
466 """
467 Property target used to get the exclude patterns list.
468 """
469 return self._excludePatterns
470
471 absolutePath = property(_getAbsolutePath, _setAbsolutePath, None, doc="Absolute path to the mbox directory.")
472 collectMode = property(_getCollectMode, _setCollectMode, None, doc="Overridden collect mode for this mbox directory.")
473 compressMode = property(_getCompressMode, _setCompressMode, None, doc="Overridden compress mode for this mbox directory.")
474 relativeExcludePaths = property(_getRelativeExcludePaths, _setRelativeExcludePaths, None, "List of relative paths to exclude.")
475 excludePatterns = property(_getExcludePatterns, _setExcludePatterns, None, "List of regular expression patterns to exclude.")
476
477
478
479
480
481
482 @total_ordering
483 -class MboxConfig(object):
484
485 """
486 Class representing mbox configuration.
487
488 Mbox configuration is used for backing up mbox email files.
489
490 The following restrictions exist on data in this class:
491
492 - The collect mode must be one of the values in L{VALID_COLLECT_MODES}.
493 - The compress mode must be one of the values in L{VALID_COMPRESS_MODES}.
494 - The C{mboxFiles} list must be a list of C{MboxFile} objects
495 - The C{mboxDirs} list must be a list of C{MboxDir} objects
496
497 For the C{mboxFiles} and C{mboxDirs} lists, validation is accomplished
498 through the L{util.ObjectTypeList} list implementation that overrides common
499 list methods and transparently ensures that each element is of the proper
500 type.
501
502 Unlike collect configuration, no global exclusions are allowed on this
503 level. We only allow relative exclusions at the mbox directory level.
504 Also, there is no configured ignore file. This is because mbox directory
505 backups are not recursive.
506
507 @note: Lists within this class are "unordered" for equality comparisons.
508
509 @sort: __init__, __repr__, __str__, __cmp__, __eq__, __lt__, __gt__,
510 collectMode, compressMode, mboxFiles, mboxDirs
511 """
512
513 - def __init__(self, collectMode=None, compressMode=None, mboxFiles=None, mboxDirs=None):
514 """
515 Constructor for the C{MboxConfig} class.
516
517 @param collectMode: Default collect mode.
518 @param compressMode: Default compress mode.
519 @param mboxFiles: List of mbox files to back up
520 @param mboxDirs: List of mbox directories to back up
521
522 @raise ValueError: If one of the values is invalid.
523 """
524 self._collectMode = None
525 self._compressMode = None
526 self._mboxFiles = None
527 self._mboxDirs = None
528 self.collectMode = collectMode
529 self.compressMode = compressMode
530 self.mboxFiles = mboxFiles
531 self.mboxDirs = mboxDirs
532
538
540 """
541 Informal string representation for class instance.
542 """
543 return self.__repr__()
544
546 """Equals operator, iplemented in terms of original Python 2 compare operator."""
547 return self.__cmp__(other) == 0
548
550 """Less-than operator, iplemented in terms of original Python 2 compare operator."""
551 return self.__cmp__(other) < 0
552
554 """Greater-than operator, iplemented in terms of original Python 2 compare operator."""
555 return self.__cmp__(other) > 0
556
558 """
559 Original Python 2 comparison operator.
560 Lists within this class are "unordered" for equality comparisons.
561 @param other: Other object to compare to.
562 @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other.
563 """
564 if other is None:
565 return 1
566 if self.collectMode != other.collectMode:
567 if str(self.collectMode or "") < str(other.collectMode or ""):
568 return -1
569 else:
570 return 1
571 if self.compressMode != other.compressMode:
572 if str(self.compressMode or "") < str(other.compressMode or ""):
573 return -1
574 else:
575 return 1
576 if self.mboxFiles != other.mboxFiles:
577 if self.mboxFiles < other.mboxFiles:
578 return -1
579 else:
580 return 1
581 if self.mboxDirs != other.mboxDirs:
582 if self.mboxDirs < other.mboxDirs:
583 return -1
584 else:
585 return 1
586 return 0
587
589 """
590 Property target used to set the collect mode.
591 If not C{None}, the mode must be one of the values in L{VALID_COLLECT_MODES}.
592 @raise ValueError: If the value is not valid.
593 """
594 if value is not None:
595 if value not in VALID_COLLECT_MODES:
596 raise ValueError("Collect mode must be one of %s." % VALID_COLLECT_MODES)
597 self._collectMode = value
598
600 """
601 Property target used to get the collect mode.
602 """
603 return self._collectMode
604
606 """
607 Property target used to set the compress mode.
608 If not C{None}, the mode must be one of the values in L{VALID_COMPRESS_MODES}.
609 @raise ValueError: If the value is not valid.
610 """
611 if value is not None:
612 if value not in VALID_COMPRESS_MODES:
613 raise ValueError("Compress mode must be one of %s." % VALID_COMPRESS_MODES)
614 self._compressMode = value
615
617 """
618 Property target used to get the compress mode.
619 """
620 return self._compressMode
621
623 """
624 Property target used to set the mboxFiles list.
625 Either the value must be C{None} or each element must be an C{MboxFile}.
626 @raise ValueError: If the value is not an C{MboxFile}
627 """
628 if value is None:
629 self._mboxFiles = None
630 else:
631 try:
632 saved = self._mboxFiles
633 self._mboxFiles = ObjectTypeList(MboxFile, "MboxFile")
634 self._mboxFiles.extend(value)
635 except Exception as e:
636 self._mboxFiles = saved
637 raise e
638
640 """
641 Property target used to get the mboxFiles list.
642 """
643 return self._mboxFiles
644
646 """
647 Property target used to set the mboxDirs list.
648 Either the value must be C{None} or each element must be an C{MboxDir}.
649 @raise ValueError: If the value is not an C{MboxDir}
650 """
651 if value is None:
652 self._mboxDirs = None
653 else:
654 try:
655 saved = self._mboxDirs
656 self._mboxDirs = ObjectTypeList(MboxDir, "MboxDir")
657 self._mboxDirs.extend(value)
658 except Exception as e:
659 self._mboxDirs = saved
660 raise e
661
663 """
664 Property target used to get the mboxDirs list.
665 """
666 return self._mboxDirs
667
668 collectMode = property(_getCollectMode, _setCollectMode, None, doc="Default collect mode.")
669 compressMode = property(_getCompressMode, _setCompressMode, None, doc="Default compress mode.")
670 mboxFiles = property(_getMboxFiles, _setMboxFiles, None, doc="List of mbox files to back up.")
671 mboxDirs = property(_getMboxDirs, _setMboxDirs, None, doc="List of mbox directories to back up.")
672
673
674
675
676
677
678 @total_ordering
679 -class LocalConfig(object):
680
681 """
682 Class representing this extension's configuration document.
683
684 This is not a general-purpose configuration object like the main Cedar
685 Backup configuration object. Instead, it just knows how to parse and emit
686 Mbox-specific configuration values. Third parties who need to read and
687 write configuration related to this extension should access it through the
688 constructor, C{validate} and C{addConfig} methods.
689
690 @note: Lists within this class are "unordered" for equality comparisons.
691
692 @sort: __init__, __repr__, __str__, __cmp__, __eq__, __lt__, __gt__, mbox,
693 validate, addConfig
694 """
695
696 - def __init__(self, xmlData=None, xmlPath=None, validate=True):
697 """
698 Initializes a configuration object.
699
700 If you initialize the object without passing either C{xmlData} or
701 C{xmlPath} then configuration will be empty and will be invalid until it
702 is filled in properly.
703
704 No reference to the original XML data or original path is saved off by
705 this class. Once the data has been parsed (successfully or not) this
706 original information is discarded.
707
708 Unless the C{validate} argument is C{False}, the L{LocalConfig.validate}
709 method will be called (with its default arguments) against configuration
710 after successfully parsing any passed-in XML. Keep in mind that even if
711 C{validate} is C{False}, it might not be possible to parse the passed-in
712 XML document if lower-level validations fail.
713
714 @note: It is strongly suggested that the C{validate} option always be set
715 to C{True} (the default) unless there is a specific need to read in
716 invalid configuration from disk.
717
718 @param xmlData: XML data representing configuration.
719 @type xmlData: String data.
720
721 @param xmlPath: Path to an XML file on disk.
722 @type xmlPath: Absolute path to a file on disk.
723
724 @param validate: Validate the document after parsing it.
725 @type validate: Boolean true/false.
726
727 @raise ValueError: If both C{xmlData} and C{xmlPath} are passed-in.
728 @raise ValueError: If the XML data in C{xmlData} or C{xmlPath} cannot be parsed.
729 @raise ValueError: If the parsed configuration document is not valid.
730 """
731 self._mbox = None
732 self.mbox = None
733 if xmlData is not None and xmlPath is not None:
734 raise ValueError("Use either xmlData or xmlPath, but not both.")
735 if xmlData is not None:
736 self._parseXmlData(xmlData)
737 if validate:
738 self.validate()
739 elif xmlPath is not None:
740 with open(xmlPath) as f:
741 xmlData = f.read()
742 self._parseXmlData(xmlData)
743 if validate:
744 self.validate()
745
747 """
748 Official string representation for class instance.
749 """
750 return "LocalConfig(%s)" % (self.mbox)
751
753 """
754 Informal string representation for class instance.
755 """
756 return self.__repr__()
757
759 """Equals operator, iplemented in terms of original Python 2 compare operator."""
760 return self.__cmp__(other) == 0
761
763 """Less-than operator, iplemented in terms of original Python 2 compare operator."""
764 return self.__cmp__(other) < 0
765
767 """Greater-than operator, iplemented in terms of original Python 2 compare operator."""
768 return self.__cmp__(other) > 0
769
771 """
772 Original Python 2 comparison operator.
773 Lists within this class are "unordered" for equality comparisons.
774 @param other: Other object to compare to.
775 @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other.
776 """
777 if other is None:
778 return 1
779 if self.mbox != other.mbox:
780 if self.mbox < other.mbox:
781 return -1
782 else:
783 return 1
784 return 0
785
787 """
788 Property target used to set the mbox configuration value.
789 If not C{None}, the value must be a C{MboxConfig} object.
790 @raise ValueError: If the value is not a C{MboxConfig}
791 """
792 if value is None:
793 self._mbox = None
794 else:
795 if not isinstance(value, MboxConfig):
796 raise ValueError("Value must be a C{MboxConfig} object.")
797 self._mbox = value
798
800 """
801 Property target used to get the mbox configuration value.
802 """
803 return self._mbox
804
805 mbox = property(_getMbox, _setMbox, None, "Mbox configuration in terms of a C{MboxConfig} object.")
806
808 """
809 Validates configuration represented by the object.
810
811 Mbox configuration must be filled in. Within that, the collect mode and
812 compress mode are both optional, but the list of repositories must
813 contain at least one entry.
814
815 Each configured file or directory must contain an absolute path, and then
816 must be either able to take collect mode and compress mode configuration
817 from the parent C{MboxConfig} object, or must set each value on its own.
818
819 @raise ValueError: If one of the validations fails.
820 """
821 if self.mbox is None:
822 raise ValueError("Mbox section is required.")
823 if (self.mbox.mboxFiles is None or len(self.mbox.mboxFiles) < 1) and \
824 (self.mbox.mboxDirs is None or len(self.mbox.mboxDirs) < 1):
825 raise ValueError("At least one mbox file or directory must be configured.")
826 if self.mbox.mboxFiles is not None:
827 for mboxFile in self.mbox.mboxFiles:
828 if mboxFile.absolutePath is None:
829 raise ValueError("Each mbox file must set an absolute path.")
830 if self.mbox.collectMode is None and mboxFile.collectMode is None:
831 raise ValueError("Collect mode must either be set in parent mbox section or individual mbox file.")
832 if self.mbox.compressMode is None and mboxFile.compressMode is None:
833 raise ValueError("Compress mode must either be set in parent mbox section or individual mbox file.")
834 if self.mbox.mboxDirs is not None:
835 for mboxDir in self.mbox.mboxDirs:
836 if mboxDir.absolutePath is None:
837 raise ValueError("Each mbox directory must set an absolute path.")
838 if self.mbox.collectMode is None and mboxDir.collectMode is None:
839 raise ValueError("Collect mode must either be set in parent mbox section or individual mbox directory.")
840 if self.mbox.compressMode is None and mboxDir.compressMode is None:
841 raise ValueError("Compress mode must either be set in parent mbox section or individual mbox directory.")
842
844 """
845 Adds an <mbox> configuration section as the next child of a parent.
846
847 Third parties should use this function to write configuration related to
848 this extension.
849
850 We add the following fields to the document::
851
852 collectMode //cb_config/mbox/collectMode
853 compressMode //cb_config/mbox/compressMode
854
855 We also add groups of the following items, one list element per
856 item::
857
858 mboxFiles //cb_config/mbox/file
859 mboxDirs //cb_config/mbox/dir
860
861 The mbox files and mbox directories are added by L{_addMboxFile} and
862 L{_addMboxDir}.
863
864 @param xmlDom: DOM tree as from C{impl.createDocument()}.
865 @param parentNode: Parent that the section should be appended to.
866 """
867 if self.mbox is not None:
868 sectionNode = addContainerNode(xmlDom, parentNode, "mbox")
869 addStringNode(xmlDom, sectionNode, "collect_mode", self.mbox.collectMode)
870 addStringNode(xmlDom, sectionNode, "compress_mode", self.mbox.compressMode)
871 if self.mbox.mboxFiles is not None:
872 for mboxFile in self.mbox.mboxFiles:
873 LocalConfig._addMboxFile(xmlDom, sectionNode, mboxFile)
874 if self.mbox.mboxDirs is not None:
875 for mboxDir in self.mbox.mboxDirs:
876 LocalConfig._addMboxDir(xmlDom, sectionNode, mboxDir)
877
879 """
880 Internal method to parse an XML string into the object.
881
882 This method parses the XML document into a DOM tree (C{xmlDom}) and then
883 calls a static method to parse the mbox configuration section.
884
885 @param xmlData: XML data to be parsed
886 @type xmlData: String data
887
888 @raise ValueError: If the XML cannot be successfully parsed.
889 """
890 (xmlDom, parentNode) = createInputDom(xmlData)
891 self._mbox = LocalConfig._parseMbox(parentNode)
892
893 @staticmethod
895 """
896 Parses an mbox configuration section.
897
898 We read the following individual fields::
899
900 collectMode //cb_config/mbox/collect_mode
901 compressMode //cb_config/mbox/compress_mode
902
903 We also read groups of the following item, one list element per
904 item::
905
906 mboxFiles //cb_config/mbox/file
907 mboxDirs //cb_config/mbox/dir
908
909 The mbox files are parsed by L{_parseMboxFiles} and the mbox
910 directories are parsed by L{_parseMboxDirs}.
911
912 @param parent: Parent node to search beneath.
913
914 @return: C{MboxConfig} object or C{None} if the section does not exist.
915 @raise ValueError: If some filled-in value is invalid.
916 """
917 mbox = None
918 section = readFirstChild(parent, "mbox")
919 if section is not None:
920 mbox = MboxConfig()
921 mbox.collectMode = readString(section, "collect_mode")
922 mbox.compressMode = readString(section, "compress_mode")
923 mbox.mboxFiles = LocalConfig._parseMboxFiles(section)
924 mbox.mboxDirs = LocalConfig._parseMboxDirs(section)
925 return mbox
926
927 @staticmethod
929 """
930 Reads a list of C{MboxFile} objects from immediately beneath the parent.
931
932 We read the following individual fields::
933
934 absolutePath abs_path
935 collectMode collect_mode
936 compressMode compess_mode
937
938 @param parent: Parent node to search beneath.
939
940 @return: List of C{MboxFile} objects or C{None} if none are found.
941 @raise ValueError: If some filled-in value is invalid.
942 """
943 lst = []
944 for entry in readChildren(parent, "file"):
945 if isElement(entry):
946 mboxFile = MboxFile()
947 mboxFile.absolutePath = readString(entry, "abs_path")
948 mboxFile.collectMode = readString(entry, "collect_mode")
949 mboxFile.compressMode = readString(entry, "compress_mode")
950 lst.append(mboxFile)
951 if lst == []:
952 lst = None
953 return lst
954
955 @staticmethod
957 """
958 Reads a list of C{MboxDir} objects from immediately beneath the parent.
959
960 We read the following individual fields::
961
962 absolutePath abs_path
963 collectMode collect_mode
964 compressMode compess_mode
965
966 We also read groups of the following items, one list element per
967 item::
968
969 relativeExcludePaths exclude/rel_path
970 excludePatterns exclude/pattern
971
972 The exclusions are parsed by L{_parseExclusions}.
973
974 @param parent: Parent node to search beneath.
975
976 @return: List of C{MboxDir} objects or C{None} if none are found.
977 @raise ValueError: If some filled-in value is invalid.
978 """
979 lst = []
980 for entry in readChildren(parent, "dir"):
981 if isElement(entry):
982 mboxDir = MboxDir()
983 mboxDir.absolutePath = readString(entry, "abs_path")
984 mboxDir.collectMode = readString(entry, "collect_mode")
985 mboxDir.compressMode = readString(entry, "compress_mode")
986 (mboxDir.relativeExcludePaths, mboxDir.excludePatterns) = LocalConfig._parseExclusions(entry)
987 lst.append(mboxDir)
988 if lst == []:
989 lst = None
990 return lst
991
992 @staticmethod
994 """
995 Reads exclusions data from immediately beneath the parent.
996
997 We read groups of the following items, one list element per item::
998
999 relative exclude/rel_path
1000 patterns exclude/pattern
1001
1002 If there are none of some pattern (i.e. no relative path items) then
1003 C{None} will be returned for that item in the tuple.
1004
1005 @param parentNode: Parent node to search beneath.
1006
1007 @return: Tuple of (relative, patterns) exclusions.
1008 """
1009 section = readFirstChild(parentNode, "exclude")
1010 if section is None:
1011 return (None, None)
1012 else:
1013 relative = readStringList(section, "rel_path")
1014 patterns = readStringList(section, "pattern")
1015 return (relative, patterns)
1016
1017 @staticmethod
1019 """
1020 Adds an mbox file container as the next child of a parent.
1021
1022 We add the following fields to the document::
1023
1024 absolutePath file/abs_path
1025 collectMode file/collect_mode
1026 compressMode file/compress_mode
1027
1028 The <file> node itself is created as the next child of the parent node.
1029 This method only adds one mbox file node. The parent must loop for each
1030 mbox file in the C{MboxConfig} object.
1031
1032 If C{mboxFile} is C{None}, this method call will be a no-op.
1033
1034 @param xmlDom: DOM tree as from C{impl.createDocument()}.
1035 @param parentNode: Parent that the section should be appended to.
1036 @param mboxFile: MboxFile to be added to the document.
1037 """
1038 if mboxFile is not None:
1039 sectionNode = addContainerNode(xmlDom, parentNode, "file")
1040 addStringNode(xmlDom, sectionNode, "abs_path", mboxFile.absolutePath)
1041 addStringNode(xmlDom, sectionNode, "collect_mode", mboxFile.collectMode)
1042 addStringNode(xmlDom, sectionNode, "compress_mode", mboxFile.compressMode)
1043
1044 @staticmethod
1046 """
1047 Adds an mbox directory container as the next child of a parent.
1048
1049 We add the following fields to the document::
1050
1051 absolutePath dir/abs_path
1052 collectMode dir/collect_mode
1053 compressMode dir/compress_mode
1054
1055 We also add groups of the following items, one list element per item::
1056
1057 relativeExcludePaths dir/exclude/rel_path
1058 excludePatterns dir/exclude/pattern
1059
1060 The <dir> node itself is created as the next child of the parent node.
1061 This method only adds one mbox directory node. The parent must loop for
1062 each mbox directory in the C{MboxConfig} object.
1063
1064 If C{mboxDir} is C{None}, this method call will be a no-op.
1065
1066 @param xmlDom: DOM tree as from C{impl.createDocument()}.
1067 @param parentNode: Parent that the section should be appended to.
1068 @param mboxDir: MboxDir to be added to the document.
1069 """
1070 if mboxDir is not None:
1071 sectionNode = addContainerNode(xmlDom, parentNode, "dir")
1072 addStringNode(xmlDom, sectionNode, "abs_path", mboxDir.absolutePath)
1073 addStringNode(xmlDom, sectionNode, "collect_mode", mboxDir.collectMode)
1074 addStringNode(xmlDom, sectionNode, "compress_mode", mboxDir.compressMode)
1075 if ((mboxDir.relativeExcludePaths is not None and mboxDir.relativeExcludePaths != []) or
1076 (mboxDir.excludePatterns is not None and mboxDir.excludePatterns != [])):
1077 excludeNode = addContainerNode(xmlDom, sectionNode, "exclude")
1078 if mboxDir.relativeExcludePaths is not None:
1079 for relativePath in mboxDir.relativeExcludePaths:
1080 addStringNode(xmlDom, excludeNode, "rel_path", relativePath)
1081 if mboxDir.excludePatterns is not None:
1082 for pattern in mboxDir.excludePatterns:
1083 addStringNode(xmlDom, excludeNode, "pattern", pattern)
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094 -def executeAction(configPath, options, config):
1095 """
1096 Executes the mbox backup action.
1097
1098 @param configPath: Path to configuration file on disk.
1099 @type configPath: String representing a path on disk.
1100
1101 @param options: Program command-line options.
1102 @type options: Options object.
1103
1104 @param config: Program configuration.
1105 @type config: Config object.
1106
1107 @raise ValueError: Under many generic error conditions
1108 @raise IOError: If a backup could not be written for some reason.
1109 """
1110 logger.debug("Executing mbox extended action.")
1111 newRevision = datetime.datetime.today()
1112 if config.options is None or config.collect is None:
1113 raise ValueError("Cedar Backup configuration is not properly filled in.")
1114 local = LocalConfig(xmlPath=configPath)
1115 todayIsStart = isStartOfWeek(config.options.startingDay)
1116 fullBackup = options.full or todayIsStart
1117 logger.debug("Full backup flag is [%s]", fullBackup)
1118 if local.mbox.mboxFiles is not None:
1119 for mboxFile in local.mbox.mboxFiles:
1120 logger.debug("Working with mbox file [%s]", mboxFile.absolutePath)
1121 collectMode = _getCollectMode(local, mboxFile)
1122 compressMode = _getCompressMode(local, mboxFile)
1123 lastRevision = _loadLastRevision(config, mboxFile, fullBackup, collectMode)
1124 if fullBackup or (collectMode in ['daily', 'incr', ]) or (collectMode == 'weekly' and todayIsStart):
1125 logger.debug("Mbox file meets criteria to be backed up today.")
1126 _backupMboxFile(config, mboxFile.absolutePath, fullBackup,
1127 collectMode, compressMode, lastRevision, newRevision)
1128 else:
1129 logger.debug("Mbox file will not be backed up, per collect mode.")
1130 if collectMode == 'incr':
1131 _writeNewRevision(config, mboxFile, newRevision)
1132 if local.mbox.mboxDirs is not None:
1133 for mboxDir in local.mbox.mboxDirs:
1134 logger.debug("Working with mbox directory [%s]", mboxDir.absolutePath)
1135 collectMode = _getCollectMode(local, mboxDir)
1136 compressMode = _getCompressMode(local, mboxDir)
1137 lastRevision = _loadLastRevision(config, mboxDir, fullBackup, collectMode)
1138 (excludePaths, excludePatterns) = _getExclusions(mboxDir)
1139 if fullBackup or (collectMode in ['daily', 'incr', ]) or (collectMode == 'weekly' and todayIsStart):
1140 logger.debug("Mbox directory meets criteria to be backed up today.")
1141 _backupMboxDir(config, mboxDir.absolutePath,
1142 fullBackup, collectMode, compressMode,
1143 lastRevision, newRevision,
1144 excludePaths, excludePatterns)
1145 else:
1146 logger.debug("Mbox directory will not be backed up, per collect mode.")
1147 if collectMode == 'incr':
1148 _writeNewRevision(config, mboxDir, newRevision)
1149 logger.info("Executed the mbox extended action successfully.")
1150
1152 """
1153 Gets the collect mode that should be used for an mbox file or directory.
1154 Use file- or directory-specific value if possible, otherwise take from mbox section.
1155 @param local: LocalConfig object.
1156 @param item: Mbox file or directory
1157 @return: Collect mode to use.
1158 """
1159 if item.collectMode is None:
1160 collectMode = local.mbox.collectMode
1161 else:
1162 collectMode = item.collectMode
1163 logger.debug("Collect mode is [%s]", collectMode)
1164 return collectMode
1165
1167 """
1168 Gets the compress mode that should be used for an mbox file or directory.
1169 Use file- or directory-specific value if possible, otherwise take from mbox section.
1170 @param local: LocalConfig object.
1171 @param item: Mbox file or directory
1172 @return: Compress mode to use.
1173 """
1174 if item.compressMode is None:
1175 compressMode = local.mbox.compressMode
1176 else:
1177 compressMode = item.compressMode
1178 logger.debug("Compress mode is [%s]", compressMode)
1179 return compressMode
1180
1182 """
1183 Gets the path to the revision file associated with a repository.
1184 @param config: Cedar Backup configuration.
1185 @param item: Mbox file or directory
1186 @return: Absolute path to the revision file associated with the repository.
1187 """
1188 normalized = buildNormalizedPath(item.absolutePath)
1189 filename = "%s.%s" % (normalized, REVISION_PATH_EXTENSION)
1190 revisionPath = os.path.join(config.options.workingDir, filename)
1191 logger.debug("Revision file path is [%s]", revisionPath)
1192 return revisionPath
1193
1195 """
1196 Loads the last revision date for this item from disk and returns it.
1197
1198 If this is a full backup, or if the revision file cannot be loaded for some
1199 reason, then C{None} is returned. This indicates that there is no previous
1200 revision, so the entire mail file or directory should be backed up.
1201
1202 @note: We write the actual revision object to disk via pickle, so we don't
1203 deal with the datetime precision or format at all. Whatever's in the object
1204 is what we write.
1205
1206 @param config: Cedar Backup configuration.
1207 @param item: Mbox file or directory
1208 @param fullBackup: Indicates whether this is a full backup
1209 @param collectMode: Indicates the collect mode for this item
1210
1211 @return: Revision date as a datetime.datetime object or C{None}.
1212 """
1213 revisionPath = _getRevisionPath(config, item)
1214 if fullBackup:
1215 revisionDate = None
1216 logger.debug("Revision file ignored because this is a full backup.")
1217 elif collectMode in ['weekly', 'daily']:
1218 revisionDate = None
1219 logger.debug("No revision file based on collect mode [%s].", collectMode)
1220 else:
1221 logger.debug("Revision file will be used for non-full incremental backup.")
1222 if not os.path.isfile(revisionPath):
1223 revisionDate = None
1224 logger.debug("Revision file [%s] does not exist on disk.", revisionPath)
1225 else:
1226 try:
1227 with open(revisionPath, "rb") as f:
1228 revisionDate = pickle.load(f, fix_imports=True)
1229 logger.debug("Loaded revision file [%s] from disk: [%s]", revisionPath, revisionDate)
1230 except Exception as e:
1231 revisionDate = None
1232 logger.error("Failed loading revision file [%s] from disk: %s", revisionPath, e)
1233 return revisionDate
1234
1236 """
1237 Writes new revision information to disk.
1238
1239 If we can't write the revision file successfully for any reason, we'll log
1240 the condition but won't throw an exception.
1241
1242 @note: We write the actual revision object to disk via pickle, so we don't
1243 deal with the datetime precision or format at all. Whatever's in the object
1244 is what we write.
1245
1246 @param config: Cedar Backup configuration.
1247 @param item: Mbox file or directory
1248 @param newRevision: Revision date as a datetime.datetime object.
1249 """
1250 revisionPath = _getRevisionPath(config, item)
1251 try:
1252 with open(revisionPath, "wb") as f:
1253 pickle.dump(newRevision, f, 0, fix_imports=True)
1254 changeOwnership(revisionPath, config.options.backupUser, config.options.backupGroup)
1255 logger.debug("Wrote new revision file [%s] to disk: [%s]", revisionPath, newRevision)
1256 except Exception as e:
1257 logger.error("Failed to write revision file [%s] to disk: %s", revisionPath, e)
1258
1260 """
1261 Gets exclusions (file and patterns) associated with an mbox directory.
1262
1263 The returned files value is a list of absolute paths to be excluded from the
1264 backup for a given directory. It is derived from the mbox directory's
1265 relative exclude paths.
1266
1267 The returned patterns value is a list of patterns to be excluded from the
1268 backup for a given directory. It is derived from the mbox directory's list
1269 of patterns.
1270
1271 @param mboxDir: Mbox directory object.
1272
1273 @return: Tuple (files, patterns) indicating what to exclude.
1274 """
1275 paths = []
1276 if mboxDir.relativeExcludePaths is not None:
1277 for relativePath in mboxDir.relativeExcludePaths:
1278 paths.append(os.path.join(mboxDir.absolutePath, relativePath))
1279 patterns = []
1280 if mboxDir.excludePatterns is not None:
1281 patterns.extend(mboxDir.excludePatterns)
1282 logger.debug("Exclude paths: %s", paths)
1283 logger.debug("Exclude patterns: %s", patterns)
1284 return(paths, patterns)
1285
1286 -def _getBackupPath(config, mboxPath, compressMode, newRevision, targetDir=None):
1287 """
1288 Gets the backup file path (including correct extension) associated with an mbox path.
1289
1290 We assume that if the target directory is passed in, that we're backing up a
1291 directory. Under these circumstances, we'll just use the basename of the
1292 individual path as the output file.
1293
1294 @note: The backup path only contains the current date in YYYYMMDD format,
1295 but that's OK because the index information (stored elsewhere) is the actual
1296 date object.
1297
1298 @param config: Cedar Backup configuration.
1299 @param mboxPath: Path to the indicated mbox file or directory
1300 @param compressMode: Compress mode to use for this mbox path
1301 @param newRevision: Revision this backup path represents
1302 @param targetDir: Target directory in which the path should exist
1303
1304 @return: Absolute path to the backup file associated with the repository.
1305 """
1306 if targetDir is None:
1307 normalizedPath = buildNormalizedPath(mboxPath)
1308 revisionDate = newRevision.strftime("%Y%m%d")
1309 filename = "mbox-%s-%s" % (revisionDate, normalizedPath)
1310 else:
1311 filename = os.path.basename(mboxPath)
1312 if compressMode == 'gzip':
1313 filename = "%s.gz" % filename
1314 elif compressMode == 'bzip2':
1315 filename = "%s.bz2" % filename
1316 if targetDir is None:
1317 backupPath = os.path.join(config.collect.targetDir, filename)
1318 else:
1319 backupPath = os.path.join(targetDir, filename)
1320 logger.debug("Backup file path is [%s]", backupPath)
1321 return backupPath
1322
1324 """
1325 Gets the tarfile backup file path (including correct extension) associated
1326 with an mbox path.
1327
1328 Along with the path, the tar archive mode is returned in a form that can
1329 be used with L{BackupFileList.generateTarfile}.
1330
1331 @note: The tarfile path only contains the current date in YYYYMMDD format,
1332 but that's OK because the index information (stored elsewhere) is the actual
1333 date object.
1334
1335 @param config: Cedar Backup configuration.
1336 @param mboxPath: Path to the indicated mbox file or directory
1337 @param compressMode: Compress mode to use for this mbox path
1338 @param newRevision: Revision this backup path represents
1339
1340 @return: Tuple of (absolute path to tarfile, tar archive mode)
1341 """
1342 normalizedPath = buildNormalizedPath(mboxPath)
1343 revisionDate = newRevision.strftime("%Y%m%d")
1344 filename = "mbox-%s-%s.tar" % (revisionDate, normalizedPath)
1345 if compressMode == 'gzip':
1346 filename = "%s.gz" % filename
1347 archiveMode = "targz"
1348 elif compressMode == 'bzip2':
1349 filename = "%s.bz2" % filename
1350 archiveMode = "tarbz2"
1351 else:
1352 archiveMode = "tar"
1353 tarfilePath = os.path.join(config.collect.targetDir, filename)
1354 logger.debug("Tarfile path is [%s]", tarfilePath)
1355 return (tarfilePath, archiveMode)
1356
1358 """
1359 Opens the output file used for saving backup information.
1360
1361 If the compress mode is "gzip", we'll open a C{GzipFile}, and if the
1362 compress mode is "bzip2", we'll open a C{BZ2File}. Otherwise, we'll just
1363 return an object from the normal C{open()} method.
1364
1365 @param backupPath: Path to file to open.
1366 @param compressMode: Compress mode of file ("none", "gzip", "bzip").
1367
1368 @return: Output file object, opened in binary mode for use with executeCommand()
1369 """
1370 if compressMode == "gzip":
1371 return GzipFile(backupPath, "wb")
1372 elif compressMode == "bzip2":
1373 return BZ2File(backupPath, "wb")
1374 else:
1375 return open(backupPath, "wb")
1376
1377 -def _backupMboxFile(config, absolutePath,
1378 fullBackup, collectMode, compressMode,
1379 lastRevision, newRevision, targetDir=None):
1380 """
1381 Backs up an individual mbox file.
1382
1383 @param config: Cedar Backup configuration.
1384 @param absolutePath: Path to mbox file to back up.
1385 @param fullBackup: Indicates whether this should be a full backup.
1386 @param collectMode: Indicates the collect mode for this item
1387 @param compressMode: Compress mode of file ("none", "gzip", "bzip")
1388 @param lastRevision: Date of last backup as datetime.datetime
1389 @param newRevision: Date of new (current) backup as datetime.datetime
1390 @param targetDir: Target directory to write the backed-up file into
1391
1392 @raise ValueError: If some value is missing or invalid.
1393 @raise IOError: If there is a problem backing up the mbox file.
1394 """
1395 if fullBackup or collectMode != "incr" or lastRevision is None:
1396 args = [ "-a", "-u", absolutePath, ]
1397 else:
1398 revisionDate = lastRevision.strftime("%Y-%m-%dT%H:%M:%S")
1399 args = [ "-a", "-u", "-d", "since %s" % revisionDate, absolutePath, ]
1400 command = resolveCommand(GREPMAIL_COMMAND)
1401 backupPath = _getBackupPath(config, absolutePath, compressMode, newRevision, targetDir=targetDir)
1402 with _getOutputFile(backupPath, compressMode) as outputFile:
1403 result = executeCommand(command, args, returnOutput=False, ignoreStderr=True, doNotLog=True, outputFile=outputFile)[0]
1404 if result != 0:
1405 raise IOError("Error [%d] executing grepmail on [%s]." % (result, absolutePath))
1406 logger.debug("Completed backing up mailbox [%s].", absolutePath)
1407 return backupPath
1408
1409 -def _backupMboxDir(config, absolutePath,
1410 fullBackup, collectMode, compressMode,
1411 lastRevision, newRevision,
1412 excludePaths, excludePatterns):
1413 """
1414 Backs up a directory containing mbox files.
1415
1416 @param config: Cedar Backup configuration.
1417 @param absolutePath: Path to mbox directory to back up.
1418 @param fullBackup: Indicates whether this should be a full backup.
1419 @param collectMode: Indicates the collect mode for this item
1420 @param compressMode: Compress mode of file ("none", "gzip", "bzip")
1421 @param lastRevision: Date of last backup as datetime.datetime
1422 @param newRevision: Date of new (current) backup as datetime.datetime
1423 @param excludePaths: List of absolute paths to exclude.
1424 @param excludePatterns: List of patterns to exclude.
1425
1426 @raise ValueError: If some value is missing or invalid.
1427 @raise IOError: If there is a problem backing up the mbox file.
1428 """
1429 try:
1430 tmpdir = tempfile.mkdtemp(dir=config.options.workingDir)
1431 mboxList = FilesystemList()
1432 mboxList.excludeDirs = True
1433 mboxList.excludePaths = excludePaths
1434 mboxList.excludePatterns = excludePatterns
1435 mboxList.addDirContents(absolutePath, recursive=False)
1436 tarList = BackupFileList()
1437 for item in mboxList:
1438 backupPath = _backupMboxFile(config, item, fullBackup,
1439 collectMode, "none",
1440 lastRevision, newRevision,
1441 targetDir=tmpdir)
1442 tarList.addFile(backupPath)
1443 (tarfilePath, archiveMode) = _getTarfilePath(config, absolutePath, compressMode, newRevision)
1444 tarList.generateTarfile(tarfilePath, archiveMode, ignore=True, flat=True)
1445 changeOwnership(tarfilePath, config.options.backupUser, config.options.backupGroup)
1446 logger.debug("Completed backing up directory [%s].", absolutePath)
1447 finally:
1448 try:
1449 for cleanitem in tarList:
1450 if os.path.exists(cleanitem):
1451 try:
1452 os.remove(cleanitem)
1453 except: pass
1454 except: pass
1455 try:
1456 os.rmdir(tmpdir)
1457 except: pass
1458