1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38 """
39 Provides an extension to split up large files in staging directories.
40
41 When this extension is executed, it will look through the configured Cedar
42 Backup staging directory for files exceeding a specified size limit, and split
43 them down into smaller files using the 'split' utility. Any directory which
44 has already been split (as indicated by the C{cback.split} file) will be
45 ignored.
46
47 This extension requires a new configuration section <split> and is intended
48 to be run immediately after the standard stage action or immediately before the
49 standard store action. Aside from its own configuration, it requires the
50 options and staging configuration sections in the standard Cedar Backup
51 configuration file.
52
53 @author: Kenneth J. Pronovici <pronovic@ieee.org>
54 """
55
56
57
58
59
60
61 import os
62 import re
63 import logging
64 from functools import total_ordering
65
66
67 from CedarBackup3.util import resolveCommand, executeCommand, changeOwnership
68 from CedarBackup3.xmlutil import createInputDom, addContainerNode
69 from CedarBackup3.xmlutil import readFirstChild
70 from CedarBackup3.actions.util import findDailyDirs, writeIndicatorFile, getBackupFiles
71 from CedarBackup3.config import ByteQuantity, readByteQuantity, addByteQuantityNode
72
73
74
75
76
77
78 logger = logging.getLogger("CedarBackup3.log.extend.split")
79
80 SPLIT_COMMAND = [ "split", ]
81 SPLIT_INDICATOR = "cback.split"
82
83
84
85
86
87
88 @total_ordering
89 -class SplitConfig(object):
90
91 """
92 Class representing split configuration.
93
94 Split configuration is used for splitting staging directories.
95
96 The following restrictions exist on data in this class:
97
98 - The size limit must be a ByteQuantity
99 - The split size must be a ByteQuantity
100
101 @sort: __init__, __repr__, __str__, __cmp__, __eq__, __lt__, __gt__,
102 sizeLimit, splitSize
103 """
104
105 - def __init__(self, sizeLimit=None, splitSize=None):
106 """
107 Constructor for the C{SplitCOnfig} class.
108
109 @param sizeLimit: Size limit of the files, in bytes
110 @param splitSize: Size that files exceeding the limit will be split into, in bytes
111
112 @raise ValueError: If one of the values is invalid.
113 """
114 self._sizeLimit = None
115 self._splitSize = None
116 self.sizeLimit = sizeLimit
117 self.splitSize = splitSize
118
120 """
121 Official string representation for class instance.
122 """
123 return "SplitConfig(%s, %s)" % (self.sizeLimit, self.splitSize)
124
126 """
127 Informal string representation for class instance.
128 """
129 return self.__repr__()
130
132 """Equals operator, iplemented in terms of original Python 2 compare operator."""
133 return self.__cmp__(other) == 0
134
136 """Less-than operator, iplemented in terms of original Python 2 compare operator."""
137 return self.__cmp__(other) < 0
138
140 """Greater-than operator, iplemented in terms of original Python 2 compare operator."""
141 return self.__cmp__(other) > 0
142
144 """
145 Original Python 2 comparison operator.
146 Lists within this class are "unordered" for equality comparisons.
147 @param other: Other object to compare to.
148 @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other.
149 """
150 if other is None:
151 return 1
152 if self.sizeLimit != other.sizeLimit:
153 if (self.sizeLimit or ByteQuantity()) < (other.sizeLimit or ByteQuantity()):
154 return -1
155 else:
156 return 1
157 if self.splitSize != other.splitSize:
158 if (self.splitSize or ByteQuantity()) < (other.splitSize or ByteQuantity()):
159 return -1
160 else:
161 return 1
162 return 0
163
165 """
166 Property target used to set the size limit.
167 If not C{None}, the value must be a C{ByteQuantity} object.
168 @raise ValueError: If the value is not a C{ByteQuantity}
169 """
170 if value is None:
171 self._sizeLimit = None
172 else:
173 if not isinstance(value, ByteQuantity):
174 raise ValueError("Value must be a C{ByteQuantity} object.")
175 self._sizeLimit = value
176
178 """
179 Property target used to get the size limit.
180 """
181 return self._sizeLimit
182
184 """
185 Property target used to set the split size.
186 If not C{None}, the value must be a C{ByteQuantity} object.
187 @raise ValueError: If the value is not a C{ByteQuantity}
188 """
189 if value is None:
190 self._splitSize = None
191 else:
192 if not isinstance(value, ByteQuantity):
193 raise ValueError("Value must be a C{ByteQuantity} object.")
194 self._splitSize = value
195
197 """
198 Property target used to get the split size.
199 """
200 return self._splitSize
201
202 sizeLimit = property(_getSizeLimit, _setSizeLimit, None, doc="Size limit, as a ByteQuantity")
203 splitSize = property(_getSplitSize, _setSplitSize, None, doc="Split size, as a ByteQuantity")
204
205
206
207
208
209
210 @total_ordering
211 -class LocalConfig(object):
212
213 """
214 Class representing this extension's configuration document.
215
216 This is not a general-purpose configuration object like the main Cedar
217 Backup configuration object. Instead, it just knows how to parse and emit
218 split-specific configuration values. Third parties who need to read and
219 write configuration related to this extension should access it through the
220 constructor, C{validate} and C{addConfig} methods.
221
222 @note: Lists within this class are "unordered" for equality comparisons.
223
224 @sort: __init__, __repr__, __str__, __cmp__, __eq__, __lt__, __gt__, split,
225 validate, addConfig
226 """
227
228 - def __init__(self, xmlData=None, xmlPath=None, validate=True):
229 """
230 Initializes a configuration object.
231
232 If you initialize the object without passing either C{xmlData} or
233 C{xmlPath} then configuration will be empty and will be invalid until it
234 is filled in properly.
235
236 No reference to the original XML data or original path is saved off by
237 this class. Once the data has been parsed (successfully or not) this
238 original information is discarded.
239
240 Unless the C{validate} argument is C{False}, the L{LocalConfig.validate}
241 method will be called (with its default arguments) against configuration
242 after successfully parsing any passed-in XML. Keep in mind that even if
243 C{validate} is C{False}, it might not be possible to parse the passed-in
244 XML document if lower-level validations fail.
245
246 @note: It is strongly suggested that the C{validate} option always be set
247 to C{True} (the default) unless there is a specific need to read in
248 invalid configuration from disk.
249
250 @param xmlData: XML data representing configuration.
251 @type xmlData: String data.
252
253 @param xmlPath: Path to an XML file on disk.
254 @type xmlPath: Absolute path to a file on disk.
255
256 @param validate: Validate the document after parsing it.
257 @type validate: Boolean true/false.
258
259 @raise ValueError: If both C{xmlData} and C{xmlPath} are passed-in.
260 @raise ValueError: If the XML data in C{xmlData} or C{xmlPath} cannot be parsed.
261 @raise ValueError: If the parsed configuration document is not valid.
262 """
263 self._split = None
264 self.split = None
265 if xmlData is not None and xmlPath is not None:
266 raise ValueError("Use either xmlData or xmlPath, but not both.")
267 if xmlData is not None:
268 self._parseXmlData(xmlData)
269 if validate:
270 self.validate()
271 elif xmlPath is not None:
272 xmlData = open(xmlPath).read()
273 self._parseXmlData(xmlData)
274 if validate:
275 self.validate()
276
278 """
279 Official string representation for class instance.
280 """
281 return "LocalConfig(%s)" % (self.split)
282
284 """
285 Informal string representation for class instance.
286 """
287 return self.__repr__()
288
290 """Equals operator, iplemented in terms of original Python 2 compare operator."""
291 return self.__cmp__(other) == 0
292
294 """Less-than operator, iplemented in terms of original Python 2 compare operator."""
295 return self.__cmp__(other) < 0
296
298 """Greater-than operator, iplemented in terms of original Python 2 compare operator."""
299 return self.__cmp__(other) > 0
300
302 """
303 Original Python 2 comparison operator.
304 Lists within this class are "unordered" for equality comparisons.
305 @param other: Other object to compare to.
306 @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other.
307 """
308 if other is None:
309 return 1
310 if self.split != other.split:
311 if self.split < other.split:
312 return -1
313 else:
314 return 1
315 return 0
316
318 """
319 Property target used to set the split configuration value.
320 If not C{None}, the value must be a C{SplitConfig} object.
321 @raise ValueError: If the value is not a C{SplitConfig}
322 """
323 if value is None:
324 self._split = None
325 else:
326 if not isinstance(value, SplitConfig):
327 raise ValueError("Value must be a C{SplitConfig} object.")
328 self._split = value
329
331 """
332 Property target used to get the split configuration value.
333 """
334 return self._split
335
336 split = property(_getSplit, _setSplit, None, "Split configuration in terms of a C{SplitConfig} object.")
337
339 """
340 Validates configuration represented by the object.
341
342 Split configuration must be filled in. Within that, both the size limit
343 and split size must be filled in.
344
345 @raise ValueError: If one of the validations fails.
346 """
347 if self.split is None:
348 raise ValueError("Split section is required.")
349 if self.split.sizeLimit is None:
350 raise ValueError("Size limit must be set.")
351 if self.split.splitSize is None:
352 raise ValueError("Split size must be set.")
353
355 """
356 Adds a <split> configuration section as the next child of a parent.
357
358 Third parties should use this function to write configuration related to
359 this extension.
360
361 We add the following fields to the document::
362
363 sizeLimit //cb_config/split/size_limit
364 splitSize //cb_config/split/split_size
365
366 @param xmlDom: DOM tree as from C{impl.createDocument()}.
367 @param parentNode: Parent that the section should be appended to.
368 """
369 if self.split is not None:
370 sectionNode = addContainerNode(xmlDom, parentNode, "split")
371 addByteQuantityNode(xmlDom, sectionNode, "size_limit", self.split.sizeLimit)
372 addByteQuantityNode(xmlDom, sectionNode, "split_size", self.split.splitSize)
373
375 """
376 Internal method to parse an XML string into the object.
377
378 This method parses the XML document into a DOM tree (C{xmlDom}) and then
379 calls a static method to parse the split configuration section.
380
381 @param xmlData: XML data to be parsed
382 @type xmlData: String data
383
384 @raise ValueError: If the XML cannot be successfully parsed.
385 """
386 (xmlDom, parentNode) = createInputDom(xmlData)
387 self._split = LocalConfig._parseSplit(parentNode)
388
389 @staticmethod
391 """
392 Parses an split configuration section.
393
394 We read the following individual fields::
395
396 sizeLimit //cb_config/split/size_limit
397 splitSize //cb_config/split/split_size
398
399 @param parent: Parent node to search beneath.
400
401 @return: C{EncryptConfig} object or C{None} if the section does not exist.
402 @raise ValueError: If some filled-in value is invalid.
403 """
404 split = None
405 section = readFirstChild(parent, "split")
406 if section is not None:
407 split = SplitConfig()
408 split.sizeLimit = readByteQuantity(section, "size_limit")
409 split.splitSize = readByteQuantity(section, "split_size")
410 return split
411
412
413
414
415
416
417
418
419
420
421 -def executeAction(configPath, options, config):
447
448
449
450
451
452
453 -def _splitDailyDir(dailyDir, sizeLimit, splitSize, backupUser, backupGroup):
454 """
455 Splits large files in a daily staging directory.
456
457 Files that match INDICATOR_PATTERNS (i.e. C{"cback.store"},
458 C{"cback.stage"}, etc.) are assumed to be indicator files and are ignored.
459 All other files are split.
460
461 @param dailyDir: Daily directory to encrypt
462 @param sizeLimit: Size limit, in bytes
463 @param splitSize: Split size, in bytes
464 @param backupUser: User that target files should be owned by
465 @param backupGroup: Group that target files should be owned by
466
467 @raise ValueError: If the encrypt mode is not supported.
468 @raise ValueError: If the daily staging directory does not exist.
469 """
470 logger.debug("Begin splitting contents of [%s].", dailyDir)
471 fileList = getBackupFiles(dailyDir)
472 for path in fileList:
473 size = float(os.stat(path).st_size)
474 if size > sizeLimit.bytes:
475 _splitFile(path, splitSize, backupUser, backupGroup, removeSource=True)
476 logger.debug("Completed splitting contents of [%s].", dailyDir)
477
478
479
480
481
482
483 -def _splitFile(sourcePath, splitSize, backupUser, backupGroup, removeSource=False):
484 """
485 Splits the source file into chunks of the indicated size.
486
487 The split files will be owned by the indicated backup user and group. If
488 C{removeSource} is C{True}, then the source file will be removed after it is
489 successfully split.
490
491 @param sourcePath: Absolute path of the source file to split
492 @param splitSize: Encryption mode (only "gpg" is allowed)
493 @param backupUser: User that target files should be owned by
494 @param backupGroup: Group that target files should be owned by
495 @param removeSource: Indicates whether to remove the source file
496
497 @raise IOError: If there is a problem accessing, splitting or removing the source file.
498 """
499 cwd = os.getcwd()
500 try:
501 if not os.path.exists(sourcePath):
502 raise ValueError("Source path [%s] does not exist." % sourcePath)
503 dirname = os.path.dirname(sourcePath)
504 filename = os.path.basename(sourcePath)
505 prefix = "%s_" % filename
506 bytes = int(splitSize.bytes)
507 os.chdir(dirname)
508 command = resolveCommand(SPLIT_COMMAND)
509 args = [ "--verbose", "--numeric-suffixes", "--suffix-length=5", "--bytes=%d" % bytes, filename, prefix, ]
510 (result, output) = executeCommand(command, args, returnOutput=True, ignoreStderr=False)
511 if result != 0:
512 raise IOError("Error [%d] calling split for [%s]." % (result, sourcePath))
513 pattern = re.compile(r"(creating file [`'])(%s)(.*)(')" % prefix)
514 match = pattern.search(output[-1:][0])
515 if match is None:
516 raise IOError("Unable to parse output from split command.")
517 value = int(match.group(3).strip())
518 for index in range(0, value):
519 path = "%s%05d" % (prefix, index)
520 if not os.path.exists(path):
521 raise IOError("After call to split, expected file [%s] does not exist." % path)
522 changeOwnership(path, backupUser, backupGroup)
523 if removeSource:
524 if os.path.exists(sourcePath):
525 try:
526 os.remove(sourcePath)
527 logger.debug("Completed removing old file [%s].", sourcePath)
528 except:
529 raise IOError("Failed to remove file [%s] after splitting it." % (sourcePath))
530 finally:
531 os.chdir(cwd)
532