1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38 """
39 Store-type extension that writes data to Amazon S3.
40
41 This extension requires a new configuration section <amazons3> and is intended
42 to be run immediately after the standard stage action, replacing the standard
43 store action. Aside from its own configuration, it requires the options and
44 staging configuration sections in the standard Cedar Backup configuration file.
45 Since it is intended to replace the store action, it does not rely on any store
46 configuration.
47
48 The underlying functionality relies on the U{AWS CLI interface
49 <http://aws.amazon.com/documentation/cli/>}. Before you use this extension,
50 you need to set up your Amazon S3 account and configure the AWS CLI connection
51 per Amazon's documentation. The extension assumes that the backup is being
52 executed as root, and switches over to the configured backup user to
53 communicate with AWS. So, make sure you configure AWS CLI as the backup user
54 and not root.
55
56 You can optionally configure Cedar Backup to encrypt data before sending it
57 to S3. To do that, provide a complete command line using the C{${input}} and
58 C{${output}} variables to represent the original input file and the encrypted
59 output file. This command will be executed as the backup user.
60
61 For instance, you can use something like this with GPG::
62
63 /usr/bin/gpg -c --no-use-agent --batch --yes --passphrase-file /home/backup/.passphrase -o ${output} ${input}
64
65 The GPG mechanism depends on a strong passphrase for security. One way to
66 generate a strong passphrase is using your system random number generator, i.e.::
67
68 dd if=/dev/urandom count=20 bs=1 | xxd -ps
69
70 (See U{StackExchange <http://security.stackexchange.com/questions/14867/gpg-encryption-security>}
71 for more details about that advice.) If you decide to use encryption, make sure
72 you save off the passphrase in a safe place, so you can get at your backup data
73 later if you need to. And obviously, make sure to set permissions on the
74 passphrase file so it can only be read by the backup user.
75
76 This extension was written for and tested on Linux. It will throw an exception
77 if run on Windows.
78
79 @author: Kenneth J. Pronovici <pronovic@ieee.org>
80 """
81
82
83
84
85
86
87 import sys
88 import os
89 import logging
90 import tempfile
91 import datetime
92 import json
93 import shutil
94 from functools import total_ordering
95
96
97 from CedarBackup3.filesystem import FilesystemList, BackupFileList
98 from CedarBackup3.util import resolveCommand, executeCommand, isRunningAsRoot, changeOwnership, isStartOfWeek
99 from CedarBackup3.xmlutil import createInputDom, addContainerNode, addBooleanNode, addStringNode, addLongNode
100 from CedarBackup3.xmlutil import readFirstChild, readString, readBoolean, readLong
101 from CedarBackup3.actions.util import writeIndicatorFile
102 from CedarBackup3.actions.constants import DIR_TIME_FORMAT, STAGE_INDICATOR
103
104
105
106
107
108
109 logger = logging.getLogger("CedarBackup3.log.extend.amazons3")
110
111 SU_COMMAND = [ "su" ]
112 AWS_COMMAND = [ "aws" ]
113
114 STORE_INDICATOR = "cback.amazons3"
115
116
117
118
119
120
121 @total_ordering
122 -class AmazonS3Config(object):
123
124 """
125 Class representing Amazon S3 configuration.
126
127 Amazon S3 configuration is used for storing backup data in Amazon's S3 cloud
128 storage using the C{s3cmd} tool.
129
130 The following restrictions exist on data in this class:
131
132 - The s3Bucket value must be a non-empty string
133 - The encryptCommand value, if set, must be a non-empty string
134 - The full backup size limit, if set, must be a number of bytes >= 0
135 - The incremental backup size limit, if set, must be a number of bytes >= 0
136
137 @sort: __init__, __repr__, __str__, __cmp__, __eq__, __lt__, __gt__,
138 warnMidnite, s3Bucket
139 """
140
141 - def __init__(self, warnMidnite=None, s3Bucket=None, encryptCommand=None,
142 fullBackupSizeLimit=None, incrementalBackupSizeLimit=None):
143 """
144 Constructor for the C{AmazonS3Config} class.
145
146 @param warnMidnite: Whether to generate warnings for crossing midnite.
147 @param s3Bucket: Name of the Amazon S3 bucket in which to store the data
148 @param encryptCommand: Command used to encrypt backup data before upload to S3
149 @param fullBackupSizeLimit: Maximum size of a full backup, in bytes
150 @param incrementalBackupSizeLimit: Maximum size of an incremental backup, in bytes
151
152 @raise ValueError: If one of the values is invalid.
153 """
154 self._warnMidnite = None
155 self._s3Bucket = None
156 self._encryptCommand = None
157 self._fullBackupSizeLimit = None
158 self._incrementalBackupSizeLimit = None
159 self.warnMidnite = warnMidnite
160 self.s3Bucket = s3Bucket
161 self.encryptCommand = encryptCommand
162 self.fullBackupSizeLimit = fullBackupSizeLimit
163 self.incrementalBackupSizeLimit = incrementalBackupSizeLimit
164
171
173 """
174 Informal string representation for class instance.
175 """
176 return self.__repr__()
177
179 """Equals operator, iplemented in terms of original Python 2 compare operator."""
180 return self.__cmp__(other) == 0
181
183 """Less-than operator, iplemented in terms of original Python 2 compare operator."""
184 return self.__cmp__(other) < 0
185
187 """Greater-than operator, iplemented in terms of original Python 2 compare operator."""
188 return self.__cmp__(other) > 0
189
224
226 """
227 Property target used to set the midnite warning flag.
228 No validations, but we normalize the value to C{True} or C{False}.
229 """
230 if value:
231 self._warnMidnite = True
232 else:
233 self._warnMidnite = False
234
236 """
237 Property target used to get the midnite warning flag.
238 """
239 return self._warnMidnite
240
242 """
243 Property target used to set the S3 bucket.
244 """
245 if value is not None:
246 if len(value) < 1:
247 raise ValueError("S3 bucket must be non-empty string.")
248 self._s3Bucket = value
249
251 """
252 Property target used to get the S3 bucket.
253 """
254 return self._s3Bucket
255
257 """
258 Property target used to set the encrypt command.
259 """
260 if value is not None:
261 if len(value) < 1:
262 raise ValueError("Encrypt command must be non-empty string.")
263 self._encryptCommand = value
264
266 """
267 Property target used to get the encrypt command.
268 """
269 return self._encryptCommand
270
272 """
273 Property target used to set the full backup size limit.
274 The value must be an integer >= 0.
275 @raise ValueError: If the value is not valid.
276 """
277 if value is None:
278 self._fullBackupSizeLimit = None
279 else:
280 try:
281 value = int(value)
282 except TypeError:
283 raise ValueError("Full backup size limit must be an integer >= 0.")
284 if value < 0:
285 raise ValueError("Full backup size limit must be an integer >= 0.")
286 self._fullBackupSizeLimit = value
287
289 """
290 Property target used to get the full backup size limit.
291 """
292 return self._fullBackupSizeLimit
293
295 """
296 Property target used to set the incremental backup size limit.
297 The value must be an integer >= 0.
298 @raise ValueError: If the value is not valid.
299 """
300 if value is None:
301 self._incrementalBackupSizeLimit = None
302 else:
303 try:
304 value = int(value)
305 except TypeError:
306 raise ValueError("Incremental backup size limit must be an integer >= 0.")
307 if value < 0:
308 raise ValueError("Incremental backup size limit must be an integer >= 0.")
309 self._incrementalBackupSizeLimit = value
310
312 """
313 Property target used to get the incremental backup size limit.
314 """
315 return self._incrementalBackupSizeLimit
316
317 warnMidnite = property(_getWarnMidnite, _setWarnMidnite, None, "Whether to generate warnings for crossing midnite.")
318 s3Bucket = property(_getS3Bucket, _setS3Bucket, None, doc="Amazon S3 Bucket in which to store data")
319 encryptCommand = property(_getEncryptCommand, _setEncryptCommand, None, doc="Command used to encrypt data before upload to S3")
320 fullBackupSizeLimit = property(_getFullBackupSizeLimit, _setFullBackupSizeLimit, None,
321 doc="Maximum size of a full backup, in bytes")
322 incrementalBackupSizeLimit = property(_getIncrementalBackupSizeLimit, _setIncrementalBackupSizeLimit, None,
323 doc="Maximum size of an incremental backup, in bytes")
324
325
326
327
328
329
330 @total_ordering
331 -class LocalConfig(object):
332
333 """
334 Class representing this extension's configuration document.
335
336 This is not a general-purpose configuration object like the main Cedar
337 Backup configuration object. Instead, it just knows how to parse and emit
338 amazons3-specific configuration values. Third parties who need to read and
339 write configuration related to this extension should access it through the
340 constructor, C{validate} and C{addConfig} methods.
341
342 @note: Lists within this class are "unordered" for equality comparisons.
343
344 @sort: __init__, __repr__, __str__, __cmp__, __eq__, __lt__, __gt__,
345 amazons3, validate, addConfig
346 """
347
348 - def __init__(self, xmlData=None, xmlPath=None, validate=True):
349 """
350 Initializes a configuration object.
351
352 If you initialize the object without passing either C{xmlData} or
353 C{xmlPath} then configuration will be empty and will be invalid until it
354 is filled in properly.
355
356 No reference to the original XML data or original path is saved off by
357 this class. Once the data has been parsed (successfully or not) this
358 original information is discarded.
359
360 Unless the C{validate} argument is C{False}, the L{LocalConfig.validate}
361 method will be called (with its default arguments) against configuration
362 after successfully parsing any passed-in XML. Keep in mind that even if
363 C{validate} is C{False}, it might not be possible to parse the passed-in
364 XML document if lower-level validations fail.
365
366 @note: It is strongly suggested that the C{validate} option always be set
367 to C{True} (the default) unless there is a specific need to read in
368 invalid configuration from disk.
369
370 @param xmlData: XML data representing configuration.
371 @type xmlData: String data.
372
373 @param xmlPath: Path to an XML file on disk.
374 @type xmlPath: Absolute path to a file on disk.
375
376 @param validate: Validate the document after parsing it.
377 @type validate: Boolean true/false.
378
379 @raise ValueError: If both C{xmlData} and C{xmlPath} are passed-in.
380 @raise ValueError: If the XML data in C{xmlData} or C{xmlPath} cannot be parsed.
381 @raise ValueError: If the parsed configuration document is not valid.
382 """
383 self._amazons3 = None
384 self.amazons3 = None
385 if xmlData is not None and xmlPath is not None:
386 raise ValueError("Use either xmlData or xmlPath, but not both.")
387 if xmlData is not None:
388 self._parseXmlData(xmlData)
389 if validate:
390 self.validate()
391 elif xmlPath is not None:
392 xmlData = open(xmlPath).read()
393 self._parseXmlData(xmlData)
394 if validate:
395 self.validate()
396
398 """
399 Official string representation for class instance.
400 """
401 return "LocalConfig(%s)" % (self.amazons3)
402
404 """
405 Informal string representation for class instance.
406 """
407 return self.__repr__()
408
410 """Equals operator, iplemented in terms of original Python 2 compare operator."""
411 return self.__cmp__(other) == 0
412
414 """Less-than operator, iplemented in terms of original Python 2 compare operator."""
415 return self.__cmp__(other) < 0
416
418 """Greater-than operator, iplemented in terms of original Python 2 compare operator."""
419 return self.__cmp__(other) > 0
420
422 """
423 Original Python 2 comparison operator.
424 Lists within this class are "unordered" for equality comparisons.
425 @param other: Other object to compare to.
426 @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other.
427 """
428 if other is None:
429 return 1
430 if self.amazons3 != other.amazons3:
431 if self.amazons3 < other.amazons3:
432 return -1
433 else:
434 return 1
435 return 0
436
438 """
439 Property target used to set the amazons3 configuration value.
440 If not C{None}, the value must be a C{AmazonS3Config} object.
441 @raise ValueError: If the value is not a C{AmazonS3Config}
442 """
443 if value is None:
444 self._amazons3 = None
445 else:
446 if not isinstance(value, AmazonS3Config):
447 raise ValueError("Value must be a C{AmazonS3Config} object.")
448 self._amazons3 = value
449
451 """
452 Property target used to get the amazons3 configuration value.
453 """
454 return self._amazons3
455
456 amazons3 = property(_getAmazonS3, _setAmazonS3, None, "AmazonS3 configuration in terms of a C{AmazonS3Config} object.")
457
459 """
460 Validates configuration represented by the object.
461
462 AmazonS3 configuration must be filled in. Within that, the s3Bucket target must be filled in
463
464 @raise ValueError: If one of the validations fails.
465 """
466 if self.amazons3 is None:
467 raise ValueError("AmazonS3 section is required.")
468 if self.amazons3.s3Bucket is None:
469 raise ValueError("AmazonS3 s3Bucket must be set.")
470
472 """
473 Adds an <amazons3> configuration section as the next child of a parent.
474
475 Third parties should use this function to write configuration related to
476 this extension.
477
478 We add the following fields to the document::
479
480 warnMidnite //cb_config/amazons3/warn_midnite
481 s3Bucket //cb_config/amazons3/s3_bucket
482 encryptCommand //cb_config/amazons3/encrypt
483 fullBackupSizeLimit //cb_config/amazons3/full_size_limit
484 incrementalBackupSizeLimit //cb_config/amazons3/incr_size_limit
485
486 @param xmlDom: DOM tree as from C{impl.createDocument()}.
487 @param parentNode: Parent that the section should be appended to.
488 """
489 if self.amazons3 is not None:
490 sectionNode = addContainerNode(xmlDom, parentNode, "amazons3")
491 addBooleanNode(xmlDom, sectionNode, "warn_midnite", self.amazons3.warnMidnite)
492 addStringNode(xmlDom, sectionNode, "s3_bucket", self.amazons3.s3Bucket)
493 addStringNode(xmlDom, sectionNode, "encrypt", self.amazons3.encryptCommand)
494 addLongNode(xmlDom, sectionNode, "full_size_limit", self.amazons3.fullBackupSizeLimit)
495 addLongNode(xmlDom, sectionNode, "incr_size_limit", self.amazons3.incrementalBackupSizeLimit)
496
498 """
499 Internal method to parse an XML string into the object.
500
501 This method parses the XML document into a DOM tree (C{xmlDom}) and then
502 calls a static method to parse the amazons3 configuration section.
503
504 @param xmlData: XML data to be parsed
505 @type xmlData: String data
506
507 @raise ValueError: If the XML cannot be successfully parsed.
508 """
509 (xmlDom, parentNode) = createInputDom(xmlData)
510 self._amazons3 = LocalConfig._parseAmazonS3(parentNode)
511
512 @staticmethod
540
541
542
543
544
545
546
547
548
549
550 -def executeAction(configPath, options, config):
551 """
552 Executes the amazons3 backup action.
553
554 @param configPath: Path to configuration file on disk.
555 @type configPath: String representing a path on disk.
556
557 @param options: Program command-line options.
558 @type options: Options object.
559
560 @param config: Program configuration.
561 @type config: Config object.
562
563 @raise ValueError: Under many generic error conditions
564 @raise IOError: If there are I/O problems reading or writing files
565 """
566 logger.debug("Executing amazons3 extended action.")
567 if not isRunningAsRoot():
568 logger.error("Error: the amazons3 extended action must be run as root.")
569 raise ValueError("The amazons3 extended action must be run as root.")
570 if sys.platform == "win32":
571 logger.error("Error: the amazons3 extended action is not supported on Windows.")
572 raise ValueError("The amazons3 extended action is not supported on Windows.")
573 if config.options is None or config.stage is None:
574 raise ValueError("Cedar Backup configuration is not properly filled in.")
575 local = LocalConfig(xmlPath=configPath)
576 stagingDirs = _findCorrectDailyDir(options, config, local)
577 _applySizeLimits(options, config, local, stagingDirs)
578 _writeToAmazonS3(config, local, stagingDirs)
579 _writeStoreIndicator(config, stagingDirs)
580 logger.info("Executed the amazons3 extended action successfully.")
581
592 """
593 Finds the correct daily staging directory to be written to Amazon S3.
594
595 This is substantially similar to the same function in store.py. The
596 main difference is that it doesn't rely on store configuration at all.
597
598 @param options: Options object.
599 @param config: Config object.
600 @param local: Local config object.
601
602 @return: Correct staging dir, as a dict mapping directory to date suffix.
603 @raise IOError: If the staging directory cannot be found.
604 """
605 oneDay = datetime.timedelta(days=1)
606 today = datetime.date.today()
607 yesterday = today - oneDay
608 tomorrow = today + oneDay
609 todayDate = today.strftime(DIR_TIME_FORMAT)
610 yesterdayDate = yesterday.strftime(DIR_TIME_FORMAT)
611 tomorrowDate = tomorrow.strftime(DIR_TIME_FORMAT)
612 todayPath = os.path.join(config.stage.targetDir, todayDate)
613 yesterdayPath = os.path.join(config.stage.targetDir, yesterdayDate)
614 tomorrowPath = os.path.join(config.stage.targetDir, tomorrowDate)
615 todayStageInd = os.path.join(todayPath, STAGE_INDICATOR)
616 yesterdayStageInd = os.path.join(yesterdayPath, STAGE_INDICATOR)
617 tomorrowStageInd = os.path.join(tomorrowPath, STAGE_INDICATOR)
618 todayStoreInd = os.path.join(todayPath, STORE_INDICATOR)
619 yesterdayStoreInd = os.path.join(yesterdayPath, STORE_INDICATOR)
620 tomorrowStoreInd = os.path.join(tomorrowPath, STORE_INDICATOR)
621 if options.full:
622 if os.path.isdir(todayPath) and os.path.exists(todayStageInd):
623 logger.info("Amazon S3 process will use current day's staging directory [%s]", todayPath)
624 return { todayPath:todayDate }
625 raise IOError("Unable to find staging directory to process (only tried today due to full option).")
626 else:
627 if os.path.isdir(todayPath) and os.path.exists(todayStageInd) and not os.path.exists(todayStoreInd):
628 logger.info("Amazon S3 process will use current day's staging directory [%s]", todayPath)
629 return { todayPath:todayDate }
630 elif os.path.isdir(yesterdayPath) and os.path.exists(yesterdayStageInd) and not os.path.exists(yesterdayStoreInd):
631 logger.info("Amazon S3 process will use previous day's staging directory [%s]", yesterdayPath)
632 if local.amazons3.warnMidnite:
633 logger.warn("Warning: Amazon S3 process crossed midnite boundary to find data.")
634 return { yesterdayPath:yesterdayDate }
635 elif os.path.isdir(tomorrowPath) and os.path.exists(tomorrowStageInd) and not os.path.exists(tomorrowStoreInd):
636 logger.info("Amazon S3 process will use next day's staging directory [%s]", tomorrowPath)
637 if local.amazons3.warnMidnite:
638 logger.warn("Warning: Amazon S3 process crossed midnite boundary to find data.")
639 return { tomorrowPath:tomorrowDate }
640 raise IOError("Unable to find unused staging directory to process (tried today, yesterday, tomorrow).")
641
648 """
649 Apply size limits, throwing an exception if any limits are exceeded.
650
651 Size limits are optional. If a limit is set to None, it does not apply.
652 The full size limit applies if the full option is set or if today is the
653 start of the week. The incremental size limit applies otherwise. Limits
654 are applied to the total size of all the relevant staging directories.
655
656 @param options: Options object.
657 @param config: Config object.
658 @param local: Local config object.
659 @param stagingDirs: Dictionary mapping directory path to date suffix.
660
661 @raise ValueError: Under many generic error conditions
662 @raise ValueError: If a size limit has been exceeded
663 """
664 if options.full or isStartOfWeek(config.options.startingDay):
665 logger.debug("Using Amazon S3 size limit for full backups.")
666 limit = local.amazons3.fullBackupSizeLimit
667 else:
668 logger.debug("Using Amazon S3 size limit for incremental backups.")
669 limit = local.amazons3.incrementalBackupSizeLimit
670 if limit is None:
671 logger.debug("No Amazon S3 size limit will be applied.")
672 else:
673 logger.debug("Amazon S3 size limit is: %d bytes", limit)
674 contents = BackupFileList()
675 for stagingDir in stagingDirs:
676 contents.addDirContents(stagingDir)
677 total = contents.totalSize()
678 logger.debug("Amazon S3 backup size is is: %d bytes", total)
679 if total > limit:
680 logger.error("Amazon S3 size limit exceeded: %.0f bytes > %d bytes", total, limit)
681 raise ValueError("Amazon S3 size limit exceeded: %.0f bytes > %d bytes" % (total, limit))
682 else:
683 logger.info("Total size does not exceed Amazon S3 size limit, so backup can continue.")
684
691 """
692 Writes the indicated staging directories to an Amazon S3 bucket.
693
694 Each of the staging directories listed in C{stagingDirs} will be written to
695 the configured Amazon S3 bucket from local configuration. The directories
696 will be placed into the image at the root by date, so staging directory
697 C{/opt/stage/2005/02/10} will be placed into the S3 bucket at C{/2005/02/10}.
698 If an encrypt commmand is provided, the files will be encrypted first.
699
700 @param config: Config object.
701 @param local: Local config object.
702 @param stagingDirs: Dictionary mapping directory path to date suffix.
703
704 @raise ValueError: Under many generic error conditions
705 @raise IOError: If there is a problem writing to Amazon S3
706 """
707 for stagingDir in list(stagingDirs.keys()):
708 logger.debug("Storing stage directory to Amazon S3 [%s].", stagingDir)
709 dateSuffix = stagingDirs[stagingDir]
710 s3BucketUrl = "s3://%s/%s" % (local.amazons3.s3Bucket, dateSuffix)
711 logger.debug("S3 bucket URL is [%s]", s3BucketUrl)
712 _clearExistingBackup(config, s3BucketUrl)
713 if local.amazons3.encryptCommand is None:
714 logger.debug("Encryption is disabled; files will be uploaded in cleartext.")
715 _uploadStagingDir(config, stagingDir, s3BucketUrl)
716 _verifyUpload(config, stagingDir, s3BucketUrl)
717 else:
718 logger.debug("Encryption is enabled; files will be uploaded after being encrypted.")
719 encryptedDir = tempfile.mkdtemp(dir=config.options.workingDir)
720 changeOwnership(encryptedDir, config.options.backupUser, config.options.backupGroup)
721 try:
722 _encryptStagingDir(config, local, stagingDir, encryptedDir)
723 _uploadStagingDir(config, encryptedDir, s3BucketUrl)
724 _verifyUpload(config, encryptedDir, s3BucketUrl)
725 finally:
726 if os.path.exists(encryptedDir):
727 shutil.rmtree(encryptedDir)
728
744
751 """
752 Clear any existing backup files for an S3 bucket URL.
753 @param config: Config object.
754 @param s3BucketUrl: S3 bucket URL associated with the staging directory
755 """
756 suCommand = resolveCommand(SU_COMMAND)
757 awsCommand = resolveCommand(AWS_COMMAND)
758 actualCommand = "%s s3 rm --recursive %s/" % (awsCommand[0], s3BucketUrl)
759 result = executeCommand(suCommand, [config.options.backupUser, "-c", actualCommand])[0]
760 if result != 0:
761 raise IOError("Error [%d] calling AWS CLI to clear existing backup for [%s]." % (result, s3BucketUrl))
762 logger.debug("Completed clearing any existing backup in S3 for [%s]", s3BucketUrl)
763
770 """
771 Upload the contents of a staging directory out to the Amazon S3 cloud.
772 @param config: Config object.
773 @param stagingDir: Staging directory to upload
774 @param s3BucketUrl: S3 bucket URL associated with the staging directory
775 """
776 suCommand = resolveCommand(SU_COMMAND)
777 awsCommand = resolveCommand(AWS_COMMAND)
778 actualCommand = "%s s3 cp --recursive %s/ %s/" % (awsCommand[0], stagingDir, s3BucketUrl)
779 result = executeCommand(suCommand, [config.options.backupUser, "-c", actualCommand])[0]
780 if result != 0:
781 raise IOError("Error [%d] calling AWS CLI to upload staging directory to [%s]." % (result, s3BucketUrl))
782 logger.debug("Completed uploading staging dir [%s] to [%s]", stagingDir, s3BucketUrl)
783
784
785
786
787
788
789 -def _verifyUpload(config, stagingDir, s3BucketUrl):
790 """
791 Verify that a staging directory was properly uploaded to the Amazon S3 cloud.
792 @param config: Config object.
793 @param stagingDir: Staging directory to verify
794 @param s3BucketUrl: S3 bucket URL associated with the staging directory
795 """
796 (bucket, prefix) = s3BucketUrl.replace("s3://", "").split("/", 1)
797 suCommand = resolveCommand(SU_COMMAND)
798 awsCommand = resolveCommand(AWS_COMMAND)
799 query = "Contents[].{Key: Key, Size: Size}"
800 actualCommand = "%s s3api list-objects --bucket %s --prefix %s --query '%s'" % (awsCommand[0], bucket, prefix, query)
801 (result, data) = executeCommand(suCommand, [config.options.backupUser, "-c", actualCommand], returnOutput=True)
802 if result != 0:
803 raise IOError("Error [%d] calling AWS CLI verify upload to [%s]." % (result, s3BucketUrl))
804 contents = { }
805 for entry in json.loads("".join(data)):
806 key = entry["Key"].replace(prefix, "")
807 size = int(entry["Size"])
808 contents[key] = size
809 files = FilesystemList()
810 files.addDirContents(stagingDir)
811 for entry in files:
812 if os.path.isfile(entry):
813 key = entry.replace(stagingDir, "")
814 size = int(os.stat(entry).st_size)
815 if not key in contents:
816 raise IOError("File was apparently not uploaded: [%s]" % entry)
817 else:
818 if size != contents[key]:
819 raise IOError("File size differs [%s], expected %s bytes but got %s bytes" % (entry, size, contents[key]))
820 logger.debug("Completed verifying upload from [%s] to [%s].", stagingDir, s3BucketUrl)
821
828 """
829 Encrypt a staging directory, creating a new directory in the process.
830 @param config: Config object.
831 @param stagingDir: Staging directory to use as source
832 @param encryptedDir: Target directory into which encrypted files should be written
833 """
834 suCommand = resolveCommand(SU_COMMAND)
835 files = FilesystemList()
836 files.addDirContents(stagingDir)
837 for cleartext in files:
838 if os.path.isfile(cleartext):
839 encrypted = "%s%s" % (encryptedDir, cleartext.replace(stagingDir, ""))
840 if int(os.stat(cleartext).st_size) == 0:
841 open(encrypted, 'a').close()
842 else:
843 actualCommand = local.amazons3.encryptCommand.replace("${input}", cleartext).replace("${output}", encrypted)
844 subdir = os.path.dirname(encrypted)
845 if not os.path.isdir(subdir):
846 os.makedirs(subdir)
847 changeOwnership(subdir, config.options.backupUser, config.options.backupGroup)
848 result = executeCommand(suCommand, [config.options.backupUser, "-c", actualCommand])[0]
849 if result != 0:
850 raise IOError("Error [%d] encrypting [%s]." % (result, cleartext))
851 logger.debug("Completed encrypting staging directory [%s] into [%s]", stagingDir, encryptedDir)
852