1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38 """
39 Provides an extension to back up Subversion repositories.
40
41 This is a Cedar Backup extension used to back up Subversion repositories via
42 the Cedar Backup command line. Each Subversion repository can be backed using
43 the same collect modes allowed for filesystems in the standard Cedar Backup
44 collect action: weekly, daily, incremental.
45
46 This extension requires a new configuration section <subversion> and is
47 intended to be run either immediately before or immediately after the standard
48 collect action. Aside from its own configuration, it requires the options and
49 collect configuration sections in the standard Cedar Backup configuration file.
50
51 There are two different kinds of Subversion repositories at this writing: BDB
52 (Berkeley Database) and FSFS (a "filesystem within a filesystem"). Although
53 the repository type can be specified in configuration, that information is just
54 kept around for reference. It doesn't affect the backup. Both kinds of
55 repositories are backed up in the same way, using C{svnadmin dump} in an
56 incremental mode.
57
58 It turns out that FSFS repositories can also be backed up just like any
59 other filesystem directory. If you would rather do that, then use the normal
60 collect action. This is probably simpler, although it carries its own
61 advantages and disadvantages (plus you will have to be careful to exclude
62 the working directories Subversion uses when building an update to commit).
63 Check the Subversion documentation for more information.
64
65 @author: Kenneth J. Pronovici <pronovic@ieee.org>
66 """
67
68
69
70
71
72
73 import os
74 import logging
75 import pickle
76 from bz2 import BZ2File
77 from gzip import GzipFile
78 from functools import total_ordering
79
80
81 from CedarBackup3.xmlutil import createInputDom, addContainerNode, addStringNode
82 from CedarBackup3.xmlutil import isElement, readChildren, readFirstChild, readString, readStringList
83 from CedarBackup3.config import VALID_COLLECT_MODES, VALID_COMPRESS_MODES
84 from CedarBackup3.filesystem import FilesystemList
85 from CedarBackup3.util import UnorderedList, RegexList
86 from CedarBackup3.util import isStartOfWeek, buildNormalizedPath
87 from CedarBackup3.util import resolveCommand, executeCommand
88 from CedarBackup3.util import ObjectTypeList, encodePath, changeOwnership
89
90
91
92
93
94
95 logger = logging.getLogger("CedarBackup3.log.extend.subversion")
96
97 SVNLOOK_COMMAND = [ "svnlook", ]
98 SVNADMIN_COMMAND = [ "svnadmin", ]
99
100 REVISION_PATH_EXTENSION = "svnlast"
101
102
103
104
105
106
107 @total_ordering
108 -class RepositoryDir(object):
109
110 """
111 Class representing Subversion repository directory.
112
113 A repository directory is a directory that contains one or more Subversion
114 repositories.
115
116 The following restrictions exist on data in this class:
117
118 - The directory path must be absolute.
119 - The collect mode must be one of the values in L{VALID_COLLECT_MODES}.
120 - The compress mode must be one of the values in L{VALID_COMPRESS_MODES}.
121
122 The repository type value is kept around just for reference. It doesn't
123 affect the behavior of the backup.
124
125 Relative exclusions are allowed here. However, there is no configured
126 ignore file, because repository dir backups are not recursive.
127
128 @sort: __init__, __repr__, __str__, __cmp__, __eq__, __lt__, __gt__,
129 directoryPath, collectMode, compressMode
130 """
131
132 - def __init__(self, repositoryType=None, directoryPath=None, collectMode=None, compressMode=None,
133 relativeExcludePaths=None, excludePatterns=None):
134 """
135 Constructor for the C{RepositoryDir} class.
136
137 @param repositoryType: Type of repository, for reference
138 @param directoryPath: Absolute path of the Subversion parent directory
139 @param collectMode: Overridden collect mode for this directory.
140 @param compressMode: Overridden compression mode for this directory.
141 @param relativeExcludePaths: List of relative paths to exclude.
142 @param excludePatterns: List of regular expression patterns to exclude
143 """
144 self._repositoryType = None
145 self._directoryPath = None
146 self._collectMode = None
147 self._compressMode = None
148 self._relativeExcludePaths = None
149 self._excludePatterns = None
150 self.repositoryType = repositoryType
151 self.directoryPath = directoryPath
152 self.collectMode = collectMode
153 self.compressMode = compressMode
154 self.relativeExcludePaths = relativeExcludePaths
155 self.excludePatterns = excludePatterns
156
158 """
159 Official string representation for class instance.
160 """
161 return "RepositoryDir(%s, %s, %s, %s, %s, %s)" % (self.repositoryType, self.directoryPath, self.collectMode,
162 self.compressMode, self.relativeExcludePaths, self.excludePatterns)
163
165 """
166 Informal string representation for class instance.
167 """
168 return self.__repr__()
169
171 """Equals operator, iplemented in terms of original Python 2 compare operator."""
172 return self.__cmp__(other) == 0
173
175 """Less-than operator, iplemented in terms of original Python 2 compare operator."""
176 return self.__cmp__(other) < 0
177
179 """Greater-than operator, iplemented in terms of original Python 2 compare operator."""
180 return self.__cmp__(other) > 0
181
221
223 """
224 Property target used to set the repository type.
225 There is no validation; this value is kept around just for reference.
226 """
227 self._repositoryType = value
228
230 """
231 Property target used to get the repository type.
232 """
233 return self._repositoryType
234
236 """
237 Property target used to set the directory path.
238 The value must be an absolute path if it is not C{None}.
239 It does not have to exist on disk at the time of assignment.
240 @raise ValueError: If the value is not an absolute path.
241 @raise ValueError: If the value cannot be encoded properly.
242 """
243 if value is not None:
244 if not os.path.isabs(value):
245 raise ValueError("Repository path must be an absolute path.")
246 self._directoryPath = encodePath(value)
247
249 """
250 Property target used to get the repository path.
251 """
252 return self._directoryPath
253
255 """
256 Property target used to set the collect mode.
257 If not C{None}, the mode must be one of the values in L{VALID_COLLECT_MODES}.
258 @raise ValueError: If the value is not valid.
259 """
260 if value is not None:
261 if value not in VALID_COLLECT_MODES:
262 raise ValueError("Collect mode must be one of %s." % VALID_COLLECT_MODES)
263 self._collectMode = value
264
266 """
267 Property target used to get the collect mode.
268 """
269 return self._collectMode
270
272 """
273 Property target used to set the compress mode.
274 If not C{None}, the mode must be one of the values in L{VALID_COMPRESS_MODES}.
275 @raise ValueError: If the value is not valid.
276 """
277 if value is not None:
278 if value not in VALID_COMPRESS_MODES:
279 raise ValueError("Compress mode must be one of %s." % VALID_COMPRESS_MODES)
280 self._compressMode = value
281
283 """
284 Property target used to get the compress mode.
285 """
286 return self._compressMode
287
289 """
290 Property target used to set the relative exclude paths list.
291 Elements do not have to exist on disk at the time of assignment.
292 """
293 if value is None:
294 self._relativeExcludePaths = None
295 else:
296 try:
297 saved = self._relativeExcludePaths
298 self._relativeExcludePaths = UnorderedList()
299 self._relativeExcludePaths.extend(value)
300 except Exception as e:
301 self._relativeExcludePaths = saved
302 raise e
303
305 """
306 Property target used to get the relative exclude paths list.
307 """
308 return self._relativeExcludePaths
309
311 """
312 Property target used to set the exclude patterns list.
313 """
314 if value is None:
315 self._excludePatterns = None
316 else:
317 try:
318 saved = self._excludePatterns
319 self._excludePatterns = RegexList()
320 self._excludePatterns.extend(value)
321 except Exception as e:
322 self._excludePatterns = saved
323 raise e
324
326 """
327 Property target used to get the exclude patterns list.
328 """
329 return self._excludePatterns
330
331 repositoryType = property(_getRepositoryType, _setRepositoryType, None, doc="Type of this repository, for reference.")
332 directoryPath = property(_getDirectoryPath, _setDirectoryPath, None, doc="Absolute path of the Subversion parent directory.")
333 collectMode = property(_getCollectMode, _setCollectMode, None, doc="Overridden collect mode for this repository.")
334 compressMode = property(_getCompressMode, _setCompressMode, None, doc="Overridden compress mode for this repository.")
335 relativeExcludePaths = property(_getRelativeExcludePaths, _setRelativeExcludePaths, None, "List of relative paths to exclude.")
336 excludePatterns = property(_getExcludePatterns, _setExcludePatterns, None, "List of regular expression patterns to exclude.")
337
338
339
340
341
342
343 @total_ordering
344 -class Repository(object):
345
346 """
347 Class representing generic Subversion repository configuration..
348
349 The following restrictions exist on data in this class:
350
351 - The respository path must be absolute.
352 - The collect mode must be one of the values in L{VALID_COLLECT_MODES}.
353 - The compress mode must be one of the values in L{VALID_COMPRESS_MODES}.
354
355 The repository type value is kept around just for reference. It doesn't
356 affect the behavior of the backup.
357
358 @sort: __init__, __repr__, __str__, __cmp__, __eq__, __lt__, __gt__,
359 repositoryPath, collectMode, compressMode
360 """
361
362 - def __init__(self, repositoryType=None, repositoryPath=None, collectMode=None, compressMode=None):
363 """
364 Constructor for the C{Repository} class.
365
366 @param repositoryType: Type of repository, for reference
367 @param repositoryPath: Absolute path to a Subversion repository on disk.
368 @param collectMode: Overridden collect mode for this directory.
369 @param compressMode: Overridden compression mode for this directory.
370 """
371 self._repositoryType = None
372 self._repositoryPath = None
373 self._collectMode = None
374 self._compressMode = None
375 self.repositoryType = repositoryType
376 self.repositoryPath = repositoryPath
377 self.collectMode = collectMode
378 self.compressMode = compressMode
379
385
387 """
388 Informal string representation for class instance.
389 """
390 return self.__repr__()
391
393 """Equals operator, iplemented in terms of original Python 2 compare operator."""
394 return self.__cmp__(other) == 0
395
397 """Less-than operator, iplemented in terms of original Python 2 compare operator."""
398 return self.__cmp__(other) < 0
399
401 """Greater-than operator, iplemented in terms of original Python 2 compare operator."""
402 return self.__cmp__(other) > 0
403
433
435 """
436 Property target used to set the repository type.
437 There is no validation; this value is kept around just for reference.
438 """
439 self._repositoryType = value
440
442 """
443 Property target used to get the repository type.
444 """
445 return self._repositoryType
446
448 """
449 Property target used to set the repository path.
450 The value must be an absolute path if it is not C{None}.
451 It does not have to exist on disk at the time of assignment.
452 @raise ValueError: If the value is not an absolute path.
453 @raise ValueError: If the value cannot be encoded properly.
454 """
455 if value is not None:
456 if not os.path.isabs(value):
457 raise ValueError("Repository path must be an absolute path.")
458 self._repositoryPath = encodePath(value)
459
461 """
462 Property target used to get the repository path.
463 """
464 return self._repositoryPath
465
467 """
468 Property target used to set the collect mode.
469 If not C{None}, the mode must be one of the values in L{VALID_COLLECT_MODES}.
470 @raise ValueError: If the value is not valid.
471 """
472 if value is not None:
473 if value not in VALID_COLLECT_MODES:
474 raise ValueError("Collect mode must be one of %s." % VALID_COLLECT_MODES)
475 self._collectMode = value
476
478 """
479 Property target used to get the collect mode.
480 """
481 return self._collectMode
482
484 """
485 Property target used to set the compress mode.
486 If not C{None}, the mode must be one of the values in L{VALID_COMPRESS_MODES}.
487 @raise ValueError: If the value is not valid.
488 """
489 if value is not None:
490 if value not in VALID_COMPRESS_MODES:
491 raise ValueError("Compress mode must be one of %s." % VALID_COMPRESS_MODES)
492 self._compressMode = value
493
495 """
496 Property target used to get the compress mode.
497 """
498 return self._compressMode
499
500 repositoryType = property(_getRepositoryType, _setRepositoryType, None, doc="Type of this repository, for reference.")
501 repositoryPath = property(_getRepositoryPath, _setRepositoryPath, None, doc="Path to the repository to collect.")
502 collectMode = property(_getCollectMode, _setCollectMode, None, doc="Overridden collect mode for this repository.")
503 compressMode = property(_getCompressMode, _setCompressMode, None, doc="Overridden compress mode for this repository.")
504
512
513 """
514 Class representing Subversion configuration.
515
516 Subversion configuration is used for backing up Subversion repositories.
517
518 The following restrictions exist on data in this class:
519
520 - The collect mode must be one of the values in L{VALID_COLLECT_MODES}.
521 - The compress mode must be one of the values in L{VALID_COMPRESS_MODES}.
522 - The repositories list must be a list of C{Repository} objects.
523 - The repositoryDirs list must be a list of C{RepositoryDir} objects.
524
525 For the two lists, validation is accomplished through the
526 L{util.ObjectTypeList} list implementation that overrides common list
527 methods and transparently ensures that each element has the correct type.
528
529 @note: Lists within this class are "unordered" for equality comparisons.
530
531 @sort: __init__, __repr__, __str__, __cmp__, __eq__, __lt__, __gt__,
532 collectMode, compressMode, repositories
533 """
534
535 - def __init__(self, collectMode=None, compressMode=None, repositories=None, repositoryDirs=None):
536 """
537 Constructor for the C{SubversionConfig} class.
538
539 @param collectMode: Default collect mode.
540 @param compressMode: Default compress mode.
541 @param repositories: List of Subversion repositories to back up.
542 @param repositoryDirs: List of Subversion parent directories to back up.
543
544 @raise ValueError: If one of the values is invalid.
545 """
546 self._collectMode = None
547 self._compressMode = None
548 self._repositories = None
549 self._repositoryDirs = None
550 self.collectMode = collectMode
551 self.compressMode = compressMode
552 self.repositories = repositories
553 self.repositoryDirs = repositoryDirs
554
560
562 """
563 Informal string representation for class instance.
564 """
565 return self.__repr__()
566
568 """Equals operator, iplemented in terms of original Python 2 compare operator."""
569 return self.__cmp__(other) == 0
570
572 """Less-than operator, iplemented in terms of original Python 2 compare operator."""
573 return self.__cmp__(other) < 0
574
576 """Greater-than operator, iplemented in terms of original Python 2 compare operator."""
577 return self.__cmp__(other) > 0
578
609
611 """
612 Property target used to set the collect mode.
613 If not C{None}, the mode must be one of the values in L{VALID_COLLECT_MODES}.
614 @raise ValueError: If the value is not valid.
615 """
616 if value is not None:
617 if value not in VALID_COLLECT_MODES:
618 raise ValueError("Collect mode must be one of %s." % VALID_COLLECT_MODES)
619 self._collectMode = value
620
622 """
623 Property target used to get the collect mode.
624 """
625 return self._collectMode
626
628 """
629 Property target used to set the compress mode.
630 If not C{None}, the mode must be one of the values in L{VALID_COMPRESS_MODES}.
631 @raise ValueError: If the value is not valid.
632 """
633 if value is not None:
634 if value not in VALID_COMPRESS_MODES:
635 raise ValueError("Compress mode must be one of %s." % VALID_COMPRESS_MODES)
636 self._compressMode = value
637
639 """
640 Property target used to get the compress mode.
641 """
642 return self._compressMode
643
645 """
646 Property target used to set the repositories list.
647 Either the value must be C{None} or each element must be a C{Repository}.
648 @raise ValueError: If the value is not a C{Repository}
649 """
650 if value is None:
651 self._repositories = None
652 else:
653 try:
654 saved = self._repositories
655 self._repositories = ObjectTypeList(Repository, "Repository")
656 self._repositories.extend(value)
657 except Exception as e:
658 self._repositories = saved
659 raise e
660
662 """
663 Property target used to get the repositories list.
664 """
665 return self._repositories
666
668 """
669 Property target used to set the repositoryDirs list.
670 Either the value must be C{None} or each element must be a C{Repository}.
671 @raise ValueError: If the value is not a C{Repository}
672 """
673 if value is None:
674 self._repositoryDirs = None
675 else:
676 try:
677 saved = self._repositoryDirs
678 self._repositoryDirs = ObjectTypeList(RepositoryDir, "RepositoryDir")
679 self._repositoryDirs.extend(value)
680 except Exception as e:
681 self._repositoryDirs = saved
682 raise e
683
685 """
686 Property target used to get the repositoryDirs list.
687 """
688 return self._repositoryDirs
689
690 collectMode = property(_getCollectMode, _setCollectMode, None, doc="Default collect mode.")
691 compressMode = property(_getCompressMode, _setCompressMode, None, doc="Default compress mode.")
692 repositories = property(_getRepositories, _setRepositories, None, doc="List of Subversion repositories to back up.")
693 repositoryDirs = property(_getRepositoryDirs, _setRepositoryDirs, None, doc="List of Subversion parent directories to back up.")
694
695
696
697
698
699
700 @total_ordering
701 -class LocalConfig(object):
702
703 """
704 Class representing this extension's configuration document.
705
706 This is not a general-purpose configuration object like the main Cedar
707 Backup configuration object. Instead, it just knows how to parse and emit
708 Subversion-specific configuration values. Third parties who need to read
709 and write configuration related to this extension should access it through
710 the constructor, C{validate} and C{addConfig} methods.
711
712 @note: Lists within this class are "unordered" for equality comparisons.
713
714 @sort: __init__, __repr__, __str__, __cmp__, __eq__, __lt__, __gt__,
715 subversion, validate, addConfig
716 """
717
718 - def __init__(self, xmlData=None, xmlPath=None, validate=True):
719 """
720 Initializes a configuration object.
721
722 If you initialize the object without passing either C{xmlData} or
723 C{xmlPath} then configuration will be empty and will be invalid until it
724 is filled in properly.
725
726 No reference to the original XML data or original path is saved off by
727 this class. Once the data has been parsed (successfully or not) this
728 original information is discarded.
729
730 Unless the C{validate} argument is C{False}, the L{LocalConfig.validate}
731 method will be called (with its default arguments) against configuration
732 after successfully parsing any passed-in XML. Keep in mind that even if
733 C{validate} is C{False}, it might not be possible to parse the passed-in
734 XML document if lower-level validations fail.
735
736 @note: It is strongly suggested that the C{validate} option always be set
737 to C{True} (the default) unless there is a specific need to read in
738 invalid configuration from disk.
739
740 @param xmlData: XML data representing configuration.
741 @type xmlData: String data.
742
743 @param xmlPath: Path to an XML file on disk.
744 @type xmlPath: Absolute path to a file on disk.
745
746 @param validate: Validate the document after parsing it.
747 @type validate: Boolean true/false.
748
749 @raise ValueError: If both C{xmlData} and C{xmlPath} are passed-in.
750 @raise ValueError: If the XML data in C{xmlData} or C{xmlPath} cannot be parsed.
751 @raise ValueError: If the parsed configuration document is not valid.
752 """
753 self._subversion = None
754 self.subversion = None
755 if xmlData is not None and xmlPath is not None:
756 raise ValueError("Use either xmlData or xmlPath, but not both.")
757 if xmlData is not None:
758 self._parseXmlData(xmlData)
759 if validate:
760 self.validate()
761 elif xmlPath is not None:
762 xmlData = open(xmlPath).read()
763 self._parseXmlData(xmlData)
764 if validate:
765 self.validate()
766
768 """
769 Official string representation for class instance.
770 """
771 return "LocalConfig(%s)" % (self.subversion)
772
774 """
775 Informal string representation for class instance.
776 """
777 return self.__repr__()
778
780 """Equals operator, iplemented in terms of original Python 2 compare operator."""
781 return self.__cmp__(other) == 0
782
784 """Less-than operator, iplemented in terms of original Python 2 compare operator."""
785 return self.__cmp__(other) < 0
786
788 """Greater-than operator, iplemented in terms of original Python 2 compare operator."""
789 return self.__cmp__(other) > 0
790
792 """
793 Original Python 2 comparison operator.
794 Lists within this class are "unordered" for equality comparisons.
795 @param other: Other object to compare to.
796 @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other.
797 """
798 if other is None:
799 return 1
800 if self.subversion != other.subversion:
801 if self.subversion < other.subversion:
802 return -1
803 else:
804 return 1
805 return 0
806
808 """
809 Property target used to set the subversion configuration value.
810 If not C{None}, the value must be a C{SubversionConfig} object.
811 @raise ValueError: If the value is not a C{SubversionConfig}
812 """
813 if value is None:
814 self._subversion = None
815 else:
816 if not isinstance(value, SubversionConfig):
817 raise ValueError("Value must be a C{SubversionConfig} object.")
818 self._subversion = value
819
821 """
822 Property target used to get the subversion configuration value.
823 """
824 return self._subversion
825
826 subversion = property(_getSubversion, _setSubversion, None, "Subversion configuration in terms of a C{SubversionConfig} object.")
827
829 """
830 Validates configuration represented by the object.
831
832 Subversion configuration must be filled in. Within that, the collect
833 mode and compress mode are both optional, but the list of repositories
834 must contain at least one entry.
835
836 Each repository must contain a repository path, and then must be either
837 able to take collect mode and compress mode configuration from the parent
838 C{SubversionConfig} object, or must set each value on its own.
839
840 @raise ValueError: If one of the validations fails.
841 """
842 if self.subversion is None:
843 raise ValueError("Subversion section is required.")
844 if ((self.subversion.repositories is None or len(self.subversion.repositories) < 1) and
845 (self.subversion.repositoryDirs is None or len(self.subversion.repositoryDirs) <1)):
846 raise ValueError("At least one Subversion repository must be configured.")
847 if self.subversion.repositories is not None:
848 for repository in self.subversion.repositories:
849 if repository.repositoryPath is None:
850 raise ValueError("Each repository must set a repository path.")
851 if self.subversion.collectMode is None and repository.collectMode is None:
852 raise ValueError("Collect mode must either be set in parent section or individual repository.")
853 if self.subversion.compressMode is None and repository.compressMode is None:
854 raise ValueError("Compress mode must either be set in parent section or individual repository.")
855 if self.subversion.repositoryDirs is not None:
856 for repositoryDir in self.subversion.repositoryDirs:
857 if repositoryDir.directoryPath is None:
858 raise ValueError("Each repository directory must set a directory path.")
859 if self.subversion.collectMode is None and repositoryDir.collectMode is None:
860 raise ValueError("Collect mode must either be set in parent section or repository directory.")
861 if self.subversion.compressMode is None and repositoryDir.compressMode is None:
862 raise ValueError("Compress mode must either be set in parent section or repository directory.")
863
865 """
866 Adds a <subversion> configuration section as the next child of a parent.
867
868 Third parties should use this function to write configuration related to
869 this extension.
870
871 We add the following fields to the document::
872
873 collectMode //cb_config/subversion/collectMode
874 compressMode //cb_config/subversion/compressMode
875
876 We also add groups of the following items, one list element per
877 item::
878
879 repository //cb_config/subversion/repository
880 repository_dir //cb_config/subversion/repository_dir
881
882 @param xmlDom: DOM tree as from C{impl.createDocument()}.
883 @param parentNode: Parent that the section should be appended to.
884 """
885 if self.subversion is not None:
886 sectionNode = addContainerNode(xmlDom, parentNode, "subversion")
887 addStringNode(xmlDom, sectionNode, "collect_mode", self.subversion.collectMode)
888 addStringNode(xmlDom, sectionNode, "compress_mode", self.subversion.compressMode)
889 if self.subversion.repositories is not None:
890 for repository in self.subversion.repositories:
891 LocalConfig._addRepository(xmlDom, sectionNode, repository)
892 if self.subversion.repositoryDirs is not None:
893 for repositoryDir in self.subversion.repositoryDirs:
894 LocalConfig._addRepositoryDir(xmlDom, sectionNode, repositoryDir)
895
897 """
898 Internal method to parse an XML string into the object.
899
900 This method parses the XML document into a DOM tree (C{xmlDom}) and then
901 calls a static method to parse the subversion configuration section.
902
903 @param xmlData: XML data to be parsed
904 @type xmlData: String data
905
906 @raise ValueError: If the XML cannot be successfully parsed.
907 """
908 (xmlDom, parentNode) = createInputDom(xmlData)
909 self._subversion = LocalConfig._parseSubversion(parentNode)
910
911 @staticmethod
913 """
914 Parses a subversion configuration section.
915
916 We read the following individual fields::
917
918 collectMode //cb_config/subversion/collect_mode
919 compressMode //cb_config/subversion/compress_mode
920
921 We also read groups of the following item, one list element per
922 item::
923
924 repositories //cb_config/subversion/repository
925 repository_dirs //cb_config/subversion/repository_dir
926
927 The repositories are parsed by L{_parseRepositories}, and the repository
928 dirs are parsed by L{_parseRepositoryDirs}.
929
930 @param parent: Parent node to search beneath.
931
932 @return: C{SubversionConfig} object or C{None} if the section does not exist.
933 @raise ValueError: If some filled-in value is invalid.
934 """
935 subversion = None
936 section = readFirstChild(parent, "subversion")
937 if section is not None:
938 subversion = SubversionConfig()
939 subversion.collectMode = readString(section, "collect_mode")
940 subversion.compressMode = readString(section, "compress_mode")
941 subversion.repositories = LocalConfig._parseRepositories(section)
942 subversion.repositoryDirs = LocalConfig._parseRepositoryDirs(section)
943 return subversion
944
945 @staticmethod
947 """
948 Reads a list of C{Repository} objects from immediately beneath the parent.
949
950 We read the following individual fields::
951
952 repositoryType type
953 repositoryPath abs_path
954 collectMode collect_mode
955 compressMode compess_mode
956
957 The type field is optional, and its value is kept around only for
958 reference.
959
960 @param parent: Parent node to search beneath.
961
962 @return: List of C{Repository} objects or C{None} if none are found.
963 @raise ValueError: If some filled-in value is invalid.
964 """
965 lst = []
966 for entry in readChildren(parent, "repository"):
967 if isElement(entry):
968 repository = Repository()
969 repository.repositoryType = readString(entry, "type")
970 repository.repositoryPath = readString(entry, "abs_path")
971 repository.collectMode = readString(entry, "collect_mode")
972 repository.compressMode = readString(entry, "compress_mode")
973 lst.append(repository)
974 if lst == []:
975 lst = None
976 return lst
977
978 @staticmethod
980 """
981 Adds a repository container as the next child of a parent.
982
983 We add the following fields to the document::
984
985 repositoryType repository/type
986 repositoryPath repository/abs_path
987 collectMode repository/collect_mode
988 compressMode repository/compress_mode
989
990 The <repository> node itself is created as the next child of the parent
991 node. This method only adds one repository node. The parent must loop
992 for each repository in the C{SubversionConfig} object.
993
994 If C{repository} is C{None}, this method call will be a no-op.
995
996 @param xmlDom: DOM tree as from C{impl.createDocument()}.
997 @param parentNode: Parent that the section should be appended to.
998 @param repository: Repository to be added to the document.
999 """
1000 if repository is not None:
1001 sectionNode = addContainerNode(xmlDom, parentNode, "repository")
1002 addStringNode(xmlDom, sectionNode, "type", repository.repositoryType)
1003 addStringNode(xmlDom, sectionNode, "abs_path", repository.repositoryPath)
1004 addStringNode(xmlDom, sectionNode, "collect_mode", repository.collectMode)
1005 addStringNode(xmlDom, sectionNode, "compress_mode", repository.compressMode)
1006
1007 @staticmethod
1009 """
1010 Reads a list of C{RepositoryDir} objects from immediately beneath the parent.
1011
1012 We read the following individual fields::
1013
1014 repositoryType type
1015 directoryPath abs_path
1016 collectMode collect_mode
1017 compressMode compess_mode
1018
1019 We also read groups of the following items, one list element per
1020 item::
1021
1022 relativeExcludePaths exclude/rel_path
1023 excludePatterns exclude/pattern
1024
1025 The exclusions are parsed by L{_parseExclusions}.
1026
1027 The type field is optional, and its value is kept around only for
1028 reference.
1029
1030 @param parent: Parent node to search beneath.
1031
1032 @return: List of C{RepositoryDir} objects or C{None} if none are found.
1033 @raise ValueError: If some filled-in value is invalid.
1034 """
1035 lst = []
1036 for entry in readChildren(parent, "repository_dir"):
1037 if isElement(entry):
1038 repositoryDir = RepositoryDir()
1039 repositoryDir.repositoryType = readString(entry, "type")
1040 repositoryDir.directoryPath = readString(entry, "abs_path")
1041 repositoryDir.collectMode = readString(entry, "collect_mode")
1042 repositoryDir.compressMode = readString(entry, "compress_mode")
1043 (repositoryDir.relativeExcludePaths, repositoryDir.excludePatterns) = LocalConfig._parseExclusions(entry)
1044 lst.append(repositoryDir)
1045 if lst == []:
1046 lst = None
1047 return lst
1048
1049 @staticmethod
1051 """
1052 Reads exclusions data from immediately beneath the parent.
1053
1054 We read groups of the following items, one list element per item::
1055
1056 relative exclude/rel_path
1057 patterns exclude/pattern
1058
1059 If there are none of some pattern (i.e. no relative path items) then
1060 C{None} will be returned for that item in the tuple.
1061
1062 @param parentNode: Parent node to search beneath.
1063
1064 @return: Tuple of (relative, patterns) exclusions.
1065 """
1066 section = readFirstChild(parentNode, "exclude")
1067 if section is None:
1068 return (None, None)
1069 else:
1070 relative = readStringList(section, "rel_path")
1071 patterns = readStringList(section, "pattern")
1072 return (relative, patterns)
1073
1074 @staticmethod
1076 """
1077 Adds a repository dir container as the next child of a parent.
1078
1079 We add the following fields to the document::
1080
1081 repositoryType repository_dir/type
1082 directoryPath repository_dir/abs_path
1083 collectMode repository_dir/collect_mode
1084 compressMode repository_dir/compress_mode
1085
1086 We also add groups of the following items, one list element per item::
1087
1088 relativeExcludePaths dir/exclude/rel_path
1089 excludePatterns dir/exclude/pattern
1090
1091 The <repository_dir> node itself is created as the next child of the
1092 parent node. This method only adds one repository node. The parent must
1093 loop for each repository dir in the C{SubversionConfig} object.
1094
1095 If C{repositoryDir} is C{None}, this method call will be a no-op.
1096
1097 @param xmlDom: DOM tree as from C{impl.createDocument()}.
1098 @param parentNode: Parent that the section should be appended to.
1099 @param repositoryDir: Repository dir to be added to the document.
1100 """
1101 if repositoryDir is not None:
1102 sectionNode = addContainerNode(xmlDom, parentNode, "repository_dir")
1103 addStringNode(xmlDom, sectionNode, "type", repositoryDir.repositoryType)
1104 addStringNode(xmlDom, sectionNode, "abs_path", repositoryDir.directoryPath)
1105 addStringNode(xmlDom, sectionNode, "collect_mode", repositoryDir.collectMode)
1106 addStringNode(xmlDom, sectionNode, "compress_mode", repositoryDir.compressMode)
1107 if ((repositoryDir.relativeExcludePaths is not None and repositoryDir.relativeExcludePaths != []) or
1108 (repositoryDir.excludePatterns is not None and repositoryDir.excludePatterns != [])):
1109 excludeNode = addContainerNode(xmlDom, sectionNode, "exclude")
1110 if repositoryDir.relativeExcludePaths is not None:
1111 for relativePath in repositoryDir.relativeExcludePaths:
1112 addStringNode(xmlDom, excludeNode, "rel_path", relativePath)
1113 if repositoryDir.excludePatterns is not None:
1114 for pattern in repositoryDir.excludePatterns:
1115 addStringNode(xmlDom, excludeNode, "pattern", pattern)
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126 -def executeAction(configPath, options, config):
1127 """
1128 Executes the Subversion backup action.
1129
1130 @param configPath: Path to configuration file on disk.
1131 @type configPath: String representing a path on disk.
1132
1133 @param options: Program command-line options.
1134 @type options: Options object.
1135
1136 @param config: Program configuration.
1137 @type config: Config object.
1138
1139 @raise ValueError: Under many generic error conditions
1140 @raise IOError: If a backup could not be written for some reason.
1141 """
1142 logger.debug("Executing Subversion extended action.")
1143 if config.options is None or config.collect is None:
1144 raise ValueError("Cedar Backup configuration is not properly filled in.")
1145 local = LocalConfig(xmlPath=configPath)
1146 todayIsStart = isStartOfWeek(config.options.startingDay)
1147 fullBackup = options.full or todayIsStart
1148 logger.debug("Full backup flag is [%s]", fullBackup)
1149 if local.subversion.repositories is not None:
1150 for repository in local.subversion.repositories:
1151 _backupRepository(config, local, todayIsStart, fullBackup, repository)
1152 if local.subversion.repositoryDirs is not None:
1153 for repositoryDir in local.subversion.repositoryDirs:
1154 logger.debug("Working with repository directory [%s].", repositoryDir.directoryPath)
1155 for repositoryPath in _getRepositoryPaths(repositoryDir):
1156 repository = Repository(repositoryDir.repositoryType, repositoryPath,
1157 repositoryDir.collectMode, repositoryDir.compressMode)
1158 _backupRepository(config, local, todayIsStart, fullBackup, repository)
1159 logger.info("Completed backing up Subversion repository directory [%s].", repositoryDir.directoryPath)
1160 logger.info("Executed the Subversion extended action successfully.")
1161
1175
1190
1192 """
1193 Gets the path to the revision file associated with a repository.
1194 @param config: Config object.
1195 @param repository: Repository object.
1196 @return: Absolute path to the revision file associated with the repository.
1197 """
1198 normalized = buildNormalizedPath(repository.repositoryPath)
1199 filename = "%s.%s" % (normalized, REVISION_PATH_EXTENSION)
1200 revisionPath = os.path.join(config.options.workingDir, filename)
1201 logger.debug("Revision file path is [%s]", revisionPath)
1202 return revisionPath
1203
1204 -def _getBackupPath(config, repositoryPath, compressMode, startRevision, endRevision):
1205 """
1206 Gets the backup file path (including correct extension) associated with a repository.
1207 @param config: Config object.
1208 @param repositoryPath: Path to the indicated repository
1209 @param compressMode: Compress mode to use for this repository.
1210 @param startRevision: Starting repository revision.
1211 @param endRevision: Ending repository revision.
1212 @return: Absolute path to the backup file associated with the repository.
1213 """
1214 normalizedPath = buildNormalizedPath(repositoryPath)
1215 filename = "svndump-%d:%d-%s.txt" % (startRevision, endRevision, normalizedPath)
1216 if compressMode == 'gzip':
1217 filename = "%s.gz" % filename
1218 elif compressMode == 'bzip2':
1219 filename = "%s.bz2" % filename
1220 backupPath = os.path.join(config.collect.targetDir, filename)
1221 logger.debug("Backup file path is [%s]", backupPath)
1222 return backupPath
1223
1237
1239 """
1240 Gets exclusions (file and patterns) associated with an repository directory.
1241
1242 The returned files value is a list of absolute paths to be excluded from the
1243 backup for a given directory. It is derived from the repository directory's
1244 relative exclude paths.
1245
1246 The returned patterns value is a list of patterns to be excluded from the
1247 backup for a given directory. It is derived from the repository directory's
1248 list of patterns.
1249
1250 @param repositoryDir: Repository directory object.
1251
1252 @return: Tuple (files, patterns) indicating what to exclude.
1253 """
1254 paths = []
1255 if repositoryDir.relativeExcludePaths is not None:
1256 for relativePath in repositoryDir.relativeExcludePaths:
1257 paths.append(os.path.join(repositoryDir.directoryPath, relativePath))
1258 patterns = []
1259 if repositoryDir.excludePatterns is not None:
1260 patterns.extend(repositoryDir.excludePatterns)
1261 logger.debug("Exclude paths: %s", paths)
1262 logger.debug("Exclude patterns: %s", patterns)
1263 return(paths, patterns)
1264
1266 """
1267 Backs up an individual Subversion repository.
1268
1269 This internal method wraps the public methods and adds some functionality
1270 to work better with the extended action itself.
1271
1272 @param config: Cedar Backup configuration.
1273 @param local: Local configuration
1274 @param todayIsStart: Indicates whether today is start of week
1275 @param fullBackup: Full backup flag
1276 @param repository: Repository to operate on
1277
1278 @raise ValueError: If some value is missing or invalid.
1279 @raise IOError: If there is a problem executing the Subversion dump.
1280 """
1281 logger.debug("Working with repository [%s]", repository.repositoryPath)
1282 logger.debug("Repository type is [%s]", repository.repositoryType)
1283 collectMode = _getCollectMode(local, repository)
1284 compressMode = _getCompressMode(local, repository)
1285 revisionPath = _getRevisionPath(config, repository)
1286 if not (fullBackup or (collectMode in ['daily', 'incr', ]) or (collectMode == 'weekly' and todayIsStart)):
1287 logger.debug("Repository will not be backed up, per collect mode.")
1288 return
1289 logger.debug("Repository meets criteria to be backed up today.")
1290 if collectMode != "incr" or fullBackup:
1291 startRevision = 0
1292 endRevision = getYoungestRevision(repository.repositoryPath)
1293 logger.debug("Using full backup, revision: (%d, %d).", startRevision, endRevision)
1294 else:
1295 if fullBackup:
1296 startRevision = 0
1297 endRevision = getYoungestRevision(repository.repositoryPath)
1298 else:
1299 startRevision = _loadLastRevision(revisionPath) + 1
1300 endRevision = getYoungestRevision(repository.repositoryPath)
1301 if startRevision > endRevision:
1302 logger.info("No need to back up repository [%s]; no new revisions.", repository.repositoryPath)
1303 return
1304 logger.debug("Using incremental backup, revision: (%d, %d).", startRevision, endRevision)
1305 backupPath = _getBackupPath(config, repository.repositoryPath, compressMode, startRevision, endRevision)
1306 outputFile = _getOutputFile(backupPath, compressMode)
1307 try:
1308 backupRepository(repository.repositoryPath, outputFile, startRevision, endRevision)
1309 finally:
1310 outputFile.close()
1311 if not os.path.exists(backupPath):
1312 raise IOError("Dump file [%s] does not seem to exist after backup completed." % backupPath)
1313 changeOwnership(backupPath, config.options.backupUser, config.options.backupGroup)
1314 if collectMode == "incr":
1315 _writeLastRevision(config, revisionPath, endRevision)
1316 logger.info("Completed backing up Subversion repository [%s].", repository.repositoryPath)
1317
1319 """
1320 Opens the output file used for saving the Subversion dump.
1321
1322 If the compress mode is "gzip", we'll open a C{GzipFile}, and if the
1323 compress mode is "bzip2", we'll open a C{BZ2File}. Otherwise, we'll just
1324 return an object from the normal C{open()} method.
1325
1326 @param backupPath: Path to file to open.
1327 @param compressMode: Compress mode of file ("none", "gzip", "bzip").
1328
1329 @return: Output file object.
1330 """
1331 if compressMode == "gzip":
1332 return GzipFile(backupPath, "w")
1333 elif compressMode == "bzip2":
1334 return BZ2File(backupPath, "w")
1335 else:
1336 return open(backupPath, "wb")
1337
1339 """
1340 Loads the indicated revision file from disk into an integer.
1341
1342 If we can't load the revision file successfully (either because it doesn't
1343 exist or for some other reason), then a revision of -1 will be returned -
1344 but the condition will be logged. This way, we err on the side of backing
1345 up too much, because anyone using this will presumably be adding 1 to the
1346 revision, so they don't duplicate any backups.
1347
1348 @param revisionPath: Path to the revision file on disk.
1349
1350 @return: Integer representing last backed-up revision, -1 on error or if none can be read.
1351 """
1352 if not os.path.isfile(revisionPath):
1353 startRevision = -1
1354 logger.debug("Revision file [%s] does not exist on disk.", revisionPath)
1355 else:
1356 try:
1357 startRevision = pickle.load(open(revisionPath, "r"))
1358 logger.debug("Loaded revision file [%s] from disk: %d.", revisionPath, startRevision)
1359 except:
1360 startRevision = -1
1361 logger.error("Failed loading revision file [%s] from disk.", revisionPath)
1362 return startRevision
1363
1365 """
1366 Writes the end revision to the indicated revision file on disk.
1367
1368 If we can't write the revision file successfully for any reason, we'll log
1369 the condition but won't throw an exception.
1370
1371 @param config: Config object.
1372 @param revisionPath: Path to the revision file on disk.
1373 @param endRevision: Last revision backed up on this run.
1374 """
1375 try:
1376 pickle.dump(endRevision, open(revisionPath, "w"))
1377 changeOwnership(revisionPath, config.options.backupUser, config.options.backupGroup)
1378 logger.debug("Wrote new revision file [%s] to disk: %d.", revisionPath, endRevision)
1379 except:
1380 logger.error("Failed to write revision file [%s] to disk.", revisionPath)
1381
1382
1383
1384
1385
1386
1387 -def backupRepository(repositoryPath, backupFile, startRevision=None, endRevision=None):
1388 """
1389 Backs up an individual Subversion repository.
1390
1391 The starting and ending revision values control an incremental backup. If
1392 the starting revision is not passed in, then revision zero (the start of the
1393 repository) is assumed. If the ending revision is not passed in, then the
1394 youngest revision in the database will be used as the endpoint.
1395
1396 The backup data will be written into the passed-in back file. Normally,
1397 this would be an object as returned from C{open}, but it is possible to use
1398 something like a C{GzipFile} to write compressed output. The caller is
1399 responsible for closing the passed-in backup file.
1400
1401 @note: This function should either be run as root or as the owner of the
1402 Subversion repository.
1403
1404 @note: It is apparently I{not} a good idea to interrupt this function.
1405 Sometimes, this leaves the repository in a "wedged" state, which requires
1406 recovery using C{svnadmin recover}.
1407
1408 @param repositoryPath: Path to Subversion repository to back up
1409 @type repositoryPath: String path representing Subversion repository on disk.
1410
1411 @param backupFile: Python file object to use for writing backup.
1412 @type backupFile: Python file object as from C{open()} or C{file()}.
1413
1414 @param startRevision: Starting repository revision to back up (for incremental backups)
1415 @type startRevision: Integer value >= 0.
1416
1417 @param endRevision: Ending repository revision to back up (for incremental backups)
1418 @type endRevision: Integer value >= 0.
1419
1420 @raise ValueError: If some value is missing or invalid.
1421 @raise IOError: If there is a problem executing the Subversion dump.
1422 """
1423 if startRevision is None:
1424 startRevision = 0
1425 if endRevision is None:
1426 endRevision = getYoungestRevision(repositoryPath)
1427 if int(startRevision) < 0:
1428 raise ValueError("Start revision must be >= 0.")
1429 if int(endRevision) < 0:
1430 raise ValueError("End revision must be >= 0.")
1431 if startRevision > endRevision:
1432 raise ValueError("Start revision must be <= end revision.")
1433 args = [ "dump", "--quiet", "-r%s:%s" % (startRevision, endRevision), "--incremental", repositoryPath, ]
1434 command = resolveCommand(SVNADMIN_COMMAND)
1435 result = executeCommand(command, args, returnOutput=False, ignoreStderr=True, doNotLog=True, outputFile=backupFile)[0]
1436 if result != 0:
1437 raise IOError("Error [%d] executing Subversion dump for repository [%s]." % (result, repositoryPath))
1438 logger.debug("Completed dumping subversion repository [%s].", repositoryPath)
1439
1446 """
1447 Gets the youngest (newest) revision in a Subversion repository using C{svnlook}.
1448
1449 @note: This function should either be run as root or as the owner of the
1450 Subversion repository.
1451
1452 @param repositoryPath: Path to Subversion repository to look in.
1453 @type repositoryPath: String path representing Subversion repository on disk.
1454
1455 @return: Youngest revision as an integer.
1456
1457 @raise ValueError: If there is a problem parsing the C{svnlook} output.
1458 @raise IOError: If there is a problem executing the C{svnlook} command.
1459 """
1460 args = [ 'youngest', repositoryPath, ]
1461 command = resolveCommand(SVNLOOK_COMMAND)
1462 (result, output) = executeCommand(command, args, returnOutput=True, ignoreStderr=True)
1463 if result != 0:
1464 raise IOError("Error [%d] executing 'svnlook youngest' for repository [%s]." % (result, repositoryPath))
1465 if len(output) != 1:
1466 raise ValueError("Unable to parse 'svnlook youngest' output.")
1467 return int(output[0])
1468
1475
1476 """
1477 Class representing Subversion BDB (Berkeley Database) repository configuration.
1478 This object is deprecated. Use a simple L{Repository} instead.
1479 """
1480
1481 - def __init__(self, repositoryPath=None, collectMode=None, compressMode=None):
1486
1492
1495
1496 """
1497 Class representing Subversion FSFS repository configuration.
1498 This object is deprecated. Use a simple L{Repository} instead.
1499 """
1500
1501 - def __init__(self, repositoryPath=None, collectMode=None, compressMode=None):
1506
1512
1513
1514 -def backupBDBRepository(repositoryPath, backupFile, startRevision=None, endRevision=None):
1515 """
1516 Backs up an individual Subversion BDB repository.
1517 This function is deprecated. Use L{backupRepository} instead.
1518 """
1519 return backupRepository(repositoryPath, backupFile, startRevision, endRevision)
1520
1523 """
1524 Backs up an individual Subversion FSFS repository.
1525 This function is deprecated. Use L{backupRepository} instead.
1526 """
1527 return backupRepository(repositoryPath, backupFile, startRevision, endRevision)
1528