CedarBackup3.peer

87 88 ###################### 89 # Class documentation 90 ###################### 91 92 """ 93 Backup peer representing a local peer in a backup pool. 94 95 This is a class representing a local (non-network) peer in a backup pool. 96 Local peers are backed up by simple filesystem copy operations. A local 97 peer has associated with it a name (typically, but not necessarily, a 98 hostname) and a collect directory. 99 100 The public methods other than the constructor are part of a "backup peer" 101 interface shared with the C{RemotePeer} class. 102 103 @sort: __init__, stagePeer, checkCollectIndicator, writeStageIndicator, 104 _copyLocalDir, _copyLocalFile, name, collectDir 105 """ 106 107 ############## 108 # Constructor 109 ############## 110

111 - def __init__(self, name, collectDir, ignoreFailureMode=None):

112 """ 113 Initializes a local backup peer. 114 115 Note that the collect directory must be an absolute path, but does not 116 have to exist when the object is instantiated. We do a lazy validation 117 on this value since we could (potentially) be creating peer objects 118 before an ongoing backup completed. 119 120 @param name: Name of the backup peer 121 @type name: String, typically a hostname 122 123 @param collectDir: Path to the peer's collect directory 124 @type collectDir: String representing an absolute local path on disk 125 126 @param ignoreFailureMode: Ignore failure mode for this peer 127 @type ignoreFailureMode: One of VALID_FAILURE_MODES 128 129 @raise ValueError: If the name is empty. 130 @raise ValueError: If collect directory is not an absolute path. 131 """ 132 self._name = None 133 self._collectDir = None 134 self._ignoreFailureMode = None 135 self.name = name 136 self.collectDir = collectDir 137 self.ignoreFailureMode = ignoreFailureMode

138 139 140 ############# 141 # Properties 142 ############# 143

144 - def _setName(self, value):

145 """ 146 Property target used to set the peer name. 147 The value must be a non-empty string and cannot be C{None}. 148 @raise ValueError: If the value is an empty string or C{None}. 149 """ 150 if value is None or len(value) < 1: 151 raise ValueError("Peer name must be a non-empty string.") 152 self._name = value

153

154 - def _getName(self):

155 """ 156 Property target used to get the peer name. 157 """ 158 return self._name

159

160 - def _setCollectDir(self, value):

161 """ 162 Property target used to set the collect directory. 163 The value must be an absolute path and cannot be C{None}. 164 It does not have to exist on disk at the time of assignment. 165 @raise ValueError: If the value is C{None} or is not an absolute path. 166 @raise ValueError: If a path cannot be encoded properly. 167 """ 168 if value is None or not os.path.isabs(value): 169 raise ValueError("Collect directory must be an absolute path.") 170 self._collectDir = encodePath(value)

171

172 - def _getCollectDir(self):

173 """ 174 Property target used to get the collect directory. 175 """ 176 return self._collectDir

177

178 - def _setIgnoreFailureMode(self, value):

179 """ 180 Property target used to set the ignoreFailure mode. 181 If not C{None}, the mode must be one of the values in L{VALID_FAILURE_MODES}. 182 @raise ValueError: If the value is not valid. 183 """ 184 if value is not None: 185 if value not in VALID_FAILURE_MODES: 186 raise ValueError("Ignore failure mode must be one of %s." % VALID_FAILURE_MODES) 187 self._ignoreFailureMode = value

188

189 - def _getIgnoreFailureMode(self):

190 """ 191 Property target used to get the ignoreFailure mode. 192 """ 193 return self._ignoreFailureMode

194 195 name = property(_getName, _setName, None, "Name of the peer.") 196 collectDir = property(_getCollectDir, _setCollectDir, None, "Path to the peer's collect directory (an absolute local path).") 197 ignoreFailureMode = property(_getIgnoreFailureMode, _setIgnoreFailureMode, None, "Ignore failure mode for peer.") 198 199 200 ################# 201 # Public methods 202 ################# 203

204 - def stagePeer(self, targetDir, ownership=None, permissions=None):

205 """ 206 Stages data from the peer into the indicated local target directory. 207 208 The collect and target directories must both already exist before this 209 method is called. If passed in, ownership and permissions will be 210 applied to the files that are copied. 211 212 @note: The caller is responsible for checking that the indicator exists, 213 if they care. This function only stages the files within the directory. 214 215 @note: If you have user/group as strings, call the L{util.getUidGid} function 216 to get the associated uid/gid as an ownership tuple. 217 218 @param targetDir: Target directory to write data into 219 @type targetDir: String representing a directory on disk 220 221 @param ownership: Owner and group that the staged files should have 222 @type ownership: Tuple of numeric ids C{(uid, gid)} 223 224 @param permissions: Permissions that the staged files should have 225 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}). 226 227 @return: Number of files copied from the source directory to the target directory. 228 229 @raise ValueError: If collect directory is not a directory or does not exist 230 @raise ValueError: If target directory is not a directory, does not exist or is not absolute. 231 @raise ValueError: If a path cannot be encoded properly. 232 @raise IOError: If there were no files to stage (i.e. the directory was empty) 233 @raise IOError: If there is an IO error copying a file. 234 @raise OSError: If there is an OS error copying or changing permissions on a file 235 """ 236 targetDir = encodePath(targetDir) 237 if not os.path.isabs(targetDir): 238 logger.debug("Target directory [%s] not an absolute path.", targetDir) 239 raise ValueError("Target directory must be an absolute path.") 240 if not os.path.exists(self.collectDir) or not os.path.isdir(self.collectDir): 241 logger.debug("Collect directory [%s] is not a directory or does not exist on disk.", self.collectDir) 242 raise ValueError("Collect directory is not a directory or does not exist on disk.") 243 if not os.path.exists(targetDir) or not os.path.isdir(targetDir): 244 logger.debug("Target directory [%s] is not a directory or does not exist on disk.", targetDir) 245 raise ValueError("Target directory is not a directory or does not exist on disk.") 246 count = LocalPeer._copyLocalDir(self.collectDir, targetDir, ownership, permissions) 247 if count == 0: 248 raise IOError("Did not copy any files from local peer.") 249 return count

250

251 - def checkCollectIndicator(self, collectIndicator=None):

252 """ 253 Checks the collect indicator in the peer's staging directory. 254 255 When a peer has completed collecting its backup files, it will write an 256 empty indicator file into its collect directory. This method checks to 257 see whether that indicator has been written. We're "stupid" here - if 258 the collect directory doesn't exist, you'll naturally get back C{False}. 259 260 If you need to, you can override the name of the collect indicator file 261 by passing in a different name. 262 263 @param collectIndicator: Name of the collect indicator file to check 264 @type collectIndicator: String representing name of a file in the collect directory 265 266 @return: Boolean true/false depending on whether the indicator exists. 267 @raise ValueError: If a path cannot be encoded properly. 268 """ 269 collectIndicator = encodePath(collectIndicator) 270 if collectIndicator is None: 271 return os.path.exists(os.path.join(self.collectDir, DEF_COLLECT_INDICATOR)) 272 else: 273 return os.path.exists(os.path.join(self.collectDir, collectIndicator))

274

275 - def writeStageIndicator(self, stageIndicator=None, ownership=None, permissions=None):

276 """ 277 Writes the stage indicator in the peer's staging directory. 278 279 When the master has completed collecting its backup files, it will write 280 an empty indicator file into the peer's collect directory. The presence 281 of this file implies that the staging process is complete. 282 283 If you need to, you can override the name of the stage indicator file by 284 passing in a different name. 285 286 @note: If you have user/group as strings, call the L{util.getUidGid} 287 function to get the associated uid/gid as an ownership tuple. 288 289 @param stageIndicator: Name of the indicator file to write 290 @type stageIndicator: String representing name of a file in the collect directory 291 292 @param ownership: Owner and group that the indicator file should have 293 @type ownership: Tuple of numeric ids C{(uid, gid)} 294 295 @param permissions: Permissions that the indicator file should have 296 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}). 297 298 @raise ValueError: If collect directory is not a directory or does not exist 299 @raise ValueError: If a path cannot be encoded properly. 300 @raise IOError: If there is an IO error creating the file. 301 @raise OSError: If there is an OS error creating or changing permissions on the file 302 """ 303 stageIndicator = encodePath(stageIndicator) 304 if not os.path.exists(self.collectDir) or not os.path.isdir(self.collectDir): 305 logger.debug("Collect directory [%s] is not a directory or does not exist on disk.", self.collectDir) 306 raise ValueError("Collect directory is not a directory or does not exist on disk.") 307 if stageIndicator is None: 308 fileName = os.path.join(self.collectDir, DEF_STAGE_INDICATOR) 309 else: 310 fileName = os.path.join(self.collectDir, stageIndicator) 311 LocalPeer._copyLocalFile(None, fileName, ownership, permissions) # None for sourceFile results in an empty target

312 313 314 ################## 315 # Private methods 316 ################## 317 318 @staticmethod

319 - def _copyLocalDir(sourceDir, targetDir, ownership=None, permissions=None):

320 """ 321 Copies files from the source directory to the target directory. 322 323 This function is not recursive. Only the files in the directory will be 324 copied. Ownership and permissions will be left at their default values 325 if new values are not specified. The source and target directories are 326 allowed to be soft links to a directory, but besides that soft links are 327 ignored. 328 329 @note: If you have user/group as strings, call the L{util.getUidGid} 330 function to get the associated uid/gid as an ownership tuple. 331 332 @param sourceDir: Source directory 333 @type sourceDir: String representing a directory on disk 334 335 @param targetDir: Target directory 336 @type targetDir: String representing a directory on disk 337 338 @param ownership: Owner and group that the copied files should have 339 @type ownership: Tuple of numeric ids C{(uid, gid)} 340 341 @param permissions: Permissions that the staged files should have 342 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}). 343 344 @return: Number of files copied from the source directory to the target directory. 345 346 @raise ValueError: If source or target is not a directory or does not exist. 347 @raise ValueError: If a path cannot be encoded properly. 348 @raise IOError: If there is an IO error copying the files. 349 @raise OSError: If there is an OS error copying or changing permissions on a files 350 """ 351 filesCopied = 0 352 sourceDir = encodePath(sourceDir) 353 targetDir = encodePath(targetDir) 354 for fileName in os.listdir(sourceDir): 355 sourceFile = os.path.join(sourceDir, fileName) 356 targetFile = os.path.join(targetDir, fileName) 357 LocalPeer._copyLocalFile(sourceFile, targetFile, ownership, permissions) 358 filesCopied += 1 359 return filesCopied

360 361 @staticmethod

362 - def _copyLocalFile(sourceFile=None, targetFile=None, ownership=None, permissions=None, overwrite=True):

363 """ 364 Copies a source file to a target file. 365 366 If the source file is C{None} then the target file will be created or 367 overwritten as an empty file. If the target file is C{None}, this method 368 is a no-op. Attempting to copy a soft link or a directory will result in 369 an exception. 370 371 @note: If you have user/group as strings, call the L{util.getUidGid} 372 function to get the associated uid/gid as an ownership tuple. 373 374 @note: We will not overwrite a target file that exists when this method 375 is invoked. If the target already exists, we'll raise an exception. 376 377 @param sourceFile: Source file to copy 378 @type sourceFile: String representing a file on disk, as an absolute path 379 380 @param targetFile: Target file to create 381 @type targetFile: String representing a file on disk, as an absolute path 382 383 @param ownership: Owner and group that the copied should have 384 @type ownership: Tuple of numeric ids C{(uid, gid)} 385 386 @param permissions: Permissions that the staged files should have 387 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}). 388 389 @param overwrite: Indicates whether it's OK to overwrite the target file. 390 @type overwrite: Boolean true/false. 391 392 @raise ValueError: If the passed-in source file is not a regular file. 393 @raise ValueError: If a path cannot be encoded properly. 394 @raise IOError: If the target file already exists. 395 @raise IOError: If there is an IO error copying the file 396 @raise OSError: If there is an OS error copying or changing permissions on a file 397 """ 398 targetFile = encodePath(targetFile) 399 sourceFile = encodePath(sourceFile) 400 if targetFile is None: 401 return 402 if not overwrite: 403 if os.path.exists(targetFile): 404 raise IOError("Target file [%s] already exists." % targetFile) 405 if sourceFile is None: 406 open(targetFile, "w").write("") 407 else: 408 if os.path.isfile(sourceFile) and not os.path.islink(sourceFile): 409 shutil.copy(sourceFile, targetFile) 410 else: 411 logger.debug("Source [%s] is not a regular file.", sourceFile) 412 raise ValueError("Source is not a regular file.") 413 if ownership is not None: 414 os.chown(targetFile, ownership[0], ownership[1]) 415 if permissions is not None: 416 os.chmod(targetFile, permissions)

424 425 ###################### 426 # Class documentation 427 ###################### 428 429 """ 430 Backup peer representing a remote peer in a backup pool. 431 432 This is a class representing a remote (networked) peer in a backup pool. 433 Remote peers are backed up using an rcp-compatible copy command. A remote 434 peer has associated with it a name (which must be a valid hostname), a 435 collect directory, a working directory and a copy method (an rcp-compatible 436 command). 437 438 You can also set an optional local user value. This username will be used 439 as the local user for any remote copies that are required. It can only be 440 used if the root user is executing the backup. The root user will C{su} to 441 the local user and execute the remote copies as that user. 442 443 The copy method is associated with the peer and not with the actual request 444 to copy, because we can envision that each remote host might have a 445 different connect method. 446 447 The public methods other than the constructor are part of a "backup peer" 448 interface shared with the C{LocalPeer} class. 449 450 @sort: __init__, stagePeer, checkCollectIndicator, writeStageIndicator, 451 executeRemoteCommand, executeManagedAction, _getDirContents, 452 _copyRemoteDir, _copyRemoteFile, _pushLocalFile, name, collectDir, 453 remoteUser, rcpCommand, rshCommand, cbackCommand 454 """ 455 456 ############## 457 # Constructor 458 ############## 459

460 - def __init__(self, name=None, collectDir=None, workingDir=None, remoteUser=None, 461 rcpCommand=None, localUser=None, rshCommand=None, cbackCommand=None, 462 ignoreFailureMode=None):

463 """ 464 Initializes a remote backup peer. 465 466 @note: If provided, each command will eventually be parsed into a list of 467 strings suitable for passing to C{util.executeCommand} in order to avoid 468 security holes related to shell interpolation. This parsing will be 469 done by the L{util.splitCommandLine} function. See the documentation for 470 that function for some important notes about its limitations. 471 472 @param name: Name of the backup peer 473 @type name: String, must be a valid DNS hostname 474 475 @param collectDir: Path to the peer's collect directory 476 @type collectDir: String representing an absolute path on the remote peer 477 478 @param workingDir: Working directory that can be used to create temporary files, etc. 479 @type workingDir: String representing an absolute path on the current host. 480 481 @param remoteUser: Name of the Cedar Backup user on the remote peer 482 @type remoteUser: String representing a username, valid via remote shell to the peer 483 484 @param localUser: Name of the Cedar Backup user on the current host 485 @type localUser: String representing a username, valid on the current host 486 487 @param rcpCommand: An rcp-compatible copy command to use for copying files from the peer 488 @type rcpCommand: String representing a system command including required arguments 489 490 @param rshCommand: An rsh-compatible copy command to use for remote shells to the peer 491 @type rshCommand: String representing a system command including required arguments 492 493 @param cbackCommand: A chack-compatible command to use for executing managed actions 494 @type cbackCommand: String representing a system command including required arguments 495 496 @param ignoreFailureMode: Ignore failure mode for this peer 497 @type ignoreFailureMode: One of VALID_FAILURE_MODES 498 499 @raise ValueError: If collect directory is not an absolute path 500 """ 501 self._name = None 502 self._collectDir = None 503 self._workingDir = None 504 self._remoteUser = None 505 self._localUser = None 506 self._rcpCommand = None 507 self._rcpCommandList = None 508 self._rshCommand = None 509 self._rshCommandList = None 510 self._cbackCommand = None 511 self._ignoreFailureMode = None 512 self.name = name 513 self.collectDir = collectDir 514 self.workingDir = workingDir 515 self.remoteUser = remoteUser 516 self.localUser = localUser 517 self.rcpCommand = rcpCommand 518 self.rshCommand = rshCommand 519 self.cbackCommand = cbackCommand 520 self.ignoreFailureMode = ignoreFailureMode

521 522 523 ############# 524 # Properties 525 ############# 526

527 - def _setName(self, value):

528 """ 529 Property target used to set the peer name. 530 The value must be a non-empty string and cannot be C{None}. 531 @raise ValueError: If the value is an empty string or C{None}. 532 """ 533 if value is None or len(value) < 1: 534 raise ValueError("Peer name must be a non-empty string.") 535 self._name = value

536

537 - def _getName(self):

538 """ 539 Property target used to get the peer name. 540 """ 541 return self._name

542

543 - def _setCollectDir(self, value):

544 """ 545 Property target used to set the collect directory. 546 The value must be an absolute path and cannot be C{None}. 547 It does not have to exist on disk at the time of assignment. 548 @raise ValueError: If the value is C{None} or is not an absolute path. 549 @raise ValueError: If the value cannot be encoded properly. 550 """ 551 if value is not None: 552 if not os.path.isabs(value): 553 raise ValueError("Collect directory must be an absolute path.") 554 self._collectDir = encodePath(value)

555

556 - def _getCollectDir(self):

557 """ 558 Property target used to get the collect directory. 559 """ 560 return self._collectDir

561

562 - def _setWorkingDir(self, value):

563 """ 564 Property target used to set the working directory. 565 The value must be an absolute path and cannot be C{None}. 566 @raise ValueError: If the value is C{None} or is not an absolute path. 567 @raise ValueError: If the value cannot be encoded properly. 568 """ 569 if value is not None: 570 if not os.path.isabs(value): 571 raise ValueError("Working directory must be an absolute path.") 572 self._workingDir = encodePath(value)

573

574 - def _getWorkingDir(self):

575 """ 576 Property target used to get the working directory. 577 """ 578 return self._workingDir

579

580 - def _setRemoteUser(self, value):

581 """ 582 Property target used to set the remote user. 583 The value must be a non-empty string and cannot be C{None}. 584 @raise ValueError: If the value is an empty string or C{None}. 585 """ 586 if value is None or len(value) < 1: 587 raise ValueError("Peer remote user must be a non-empty string.") 588 self._remoteUser = value

589

590 - def _getRemoteUser(self):

591 """ 592 Property target used to get the remote user. 593 """ 594 return self._remoteUser

595

596 - def _setLocalUser(self, value):

597 """ 598 Property target used to set the local user. 599 The value must be a non-empty string if it is not C{None}. 600 @raise ValueError: If the value is an empty string. 601 """ 602 if value is not None: 603 if len(value) < 1: 604 raise ValueError("Peer local user must be a non-empty string.") 605 self._localUser = value

606

607 - def _getLocalUser(self):

608 """ 609 Property target used to get the local user. 610 """ 611 return self._localUser

612

613 - def _setRcpCommand(self, value):

614 """ 615 Property target to set the rcp command. 616 617 The value must be a non-empty string or C{None}. Its value is stored in 618 the two forms: "raw" as provided by the client, and "parsed" into a list 619 suitable for being passed to L{util.executeCommand} via 620 L{util.splitCommandLine}. 621 622 However, all the caller will ever see via the property is the actual 623 value they set (which includes seeing C{None}, even if we translate that 624 internally to C{DEF_RCP_COMMAND}). Internally, we should always use 625 C{self._rcpCommandList} if we want the actual command list. 626 627 @raise ValueError: If the value is an empty string. 628 """ 629 if value is None: 630 self._rcpCommand = None 631 self._rcpCommandList = DEF_RCP_COMMAND 632 else: 633 if len(value) >= 1: 634 self._rcpCommand = value 635 self._rcpCommandList = splitCommandLine(self._rcpCommand) 636 else: 637 raise ValueError("The rcp command must be a non-empty string.")

638

639 - def _getRcpCommand(self):

640 """ 641 Property target used to get the rcp command. 642 """ 643 return self._rcpCommand

644

645 - def _setRshCommand(self, value):

646 """ 647 Property target to set the rsh command. 648 649 The value must be a non-empty string or C{None}. Its value is stored in 650 the two forms: "raw" as provided by the client, and "parsed" into a list 651 suitable for being passed to L{util.executeCommand} via 652 L{util.splitCommandLine}. 653 654 However, all the caller will ever see via the property is the actual 655 value they set (which includes seeing C{None}, even if we translate that 656 internally to C{DEF_RSH_COMMAND}). Internally, we should always use 657 C{self._rshCommandList} if we want the actual command list. 658 659 @raise ValueError: If the value is an empty string. 660 """ 661 if value is None: 662 self._rshCommand = None 663 self._rshCommandList = DEF_RSH_COMMAND 664 else: 665 if len(value) >= 1: 666 self._rshCommand = value 667 self._rshCommandList = splitCommandLine(self._rshCommand) 668 else: 669 raise ValueError("The rsh command must be a non-empty string.")

670

671 - def _getRshCommand(self):

672 """ 673 Property target used to get the rsh command. 674 """ 675 return self._rshCommand

676

677 - def _setCbackCommand(self, value):

678 """ 679 Property target to set the cback command. 680 681 The value must be a non-empty string or C{None}. Unlike the other 682 command, this value is only stored in the "raw" form provided by the 683 client. 684 685 @raise ValueError: If the value is an empty string. 686 """ 687 if value is None: 688 self._cbackCommand = None 689 else: 690 if len(value) >= 1: 691 self._cbackCommand = value 692 else: 693 raise ValueError("The cback command must be a non-empty string.")

694

695 - def _getCbackCommand(self):

696 """ 697 Property target used to get the cback command. 698 """ 699 return self._cbackCommand

700

701 - def _setIgnoreFailureMode(self, value):

702 """ 703 Property target used to set the ignoreFailure mode. 704 If not C{None}, the mode must be one of the values in L{VALID_FAILURE_MODES}. 705 @raise ValueError: If the value is not valid. 706 """ 707 if value is not None: 708 if value not in VALID_FAILURE_MODES: 709 raise ValueError("Ignore failure mode must be one of %s." % VALID_FAILURE_MODES) 710 self._ignoreFailureMode = value

711

712 - def _getIgnoreFailureMode(self):

713 """ 714 Property target used to get the ignoreFailure mode. 715 """ 716 return self._ignoreFailureMode

717 718 name = property(_getName, _setName, None, "Name of the peer (a valid DNS hostname).") 719 collectDir = property(_getCollectDir, _setCollectDir, None, "Path to the peer's collect directory (an absolute local path).") 720 workingDir = property(_getWorkingDir, _setWorkingDir, None, "Path to the peer's working directory (an absolute local path).") 721 remoteUser = property(_getRemoteUser, _setRemoteUser, None, "Name of the Cedar Backup user on the remote peer.") 722 localUser = property(_getLocalUser, _setLocalUser, None, "Name of the Cedar Backup user on the current host.") 723 rcpCommand = property(_getRcpCommand, _setRcpCommand, None, "An rcp-compatible copy command to use for copying files.") 724 rshCommand = property(_getRshCommand, _setRshCommand, None, "An rsh-compatible command to use for remote shells to the peer.") 725 cbackCommand = property(_getCbackCommand, _setCbackCommand, None, "A chack-compatible command to use for executing managed actions.") 726 ignoreFailureMode = property(_getIgnoreFailureMode, _setIgnoreFailureMode, None, "Ignore failure mode for peer.") 727 728 729 ################# 730 # Public methods 731 ################# 732

733 - def stagePeer(self, targetDir, ownership=None, permissions=None):

734 """ 735 Stages data from the peer into the indicated local target directory. 736 737 The target directory must already exist before this method is called. If 738 passed in, ownership and permissions will be applied to the files that 739 are copied. 740 741 @note: The returned count of copied files might be inaccurate if some of 742 the copied files already existed in the staging directory prior to the 743 copy taking place. We don't clear the staging directory first, because 744 some extension might also be using it. 745 746 @note: If you have user/group as strings, call the L{util.getUidGid} function 747 to get the associated uid/gid as an ownership tuple. 748 749 @note: Unlike the local peer version of this method, an I/O error might 750 or might not be raised if the directory is empty. Since we're using a 751 remote copy method, we just don't have the fine-grained control over our 752 exceptions that's available when we can look directly at the filesystem, 753 and we can't control whether the remote copy method thinks an empty 754 directory is an error. 755 756 @param targetDir: Target directory to write data into 757 @type targetDir: String representing a directory on disk 758 759 @param ownership: Owner and group that the staged files should have 760 @type ownership: Tuple of numeric ids C{(uid, gid)} 761 762 @param permissions: Permissions that the staged files should have 763 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}). 764 765 @return: Number of files copied from the source directory to the target directory. 766 767 @raise ValueError: If target directory is not a directory, does not exist or is not absolute. 768 @raise ValueError: If a path cannot be encoded properly. 769 @raise IOError: If there were no files to stage (i.e. the directory was empty) 770 @raise IOError: If there is an IO error copying a file. 771 @raise OSError: If there is an OS error copying or changing permissions on a file 772 """ 773 targetDir = encodePath(targetDir) 774 if not os.path.isabs(targetDir): 775 logger.debug("Target directory [%s] not an absolute path.", targetDir) 776 raise ValueError("Target directory must be an absolute path.") 777 if not os.path.exists(targetDir) or not os.path.isdir(targetDir): 778 logger.debug("Target directory [%s] is not a directory or does not exist on disk.", targetDir) 779 raise ValueError("Target directory is not a directory or does not exist on disk.") 780 count = RemotePeer._copyRemoteDir(self.remoteUser, self.localUser, self.name, 781 self._rcpCommand, self._rcpCommandList, 782 self.collectDir, targetDir, 783 ownership, permissions) 784 if count == 0: 785 raise IOError("Did not copy any files from local peer.") 786 return count

787

788 - def checkCollectIndicator(self, collectIndicator=None):

789 """ 790 Checks the collect indicator in the peer's staging directory. 791 792 When a peer has completed collecting its backup files, it will write an 793 empty indicator file into its collect directory. This method checks to 794 see whether that indicator has been written. If the remote copy command 795 fails, we return C{False} as if the file weren't there. 796 797 If you need to, you can override the name of the collect indicator file 798 by passing in a different name. 799 800 @note: Apparently, we can't count on all rcp-compatible implementations 801 to return sensible errors for some error conditions. As an example, the 802 C{scp} command in Debian 'woody' returns a zero (normal) status even when 803 it can't find a host or if the login or path is invalid. Because of 804 this, the implementation of this method is rather convoluted. 805 806 @param collectIndicator: Name of the collect indicator file to check 807 @type collectIndicator: String representing name of a file in the collect directory 808 809 @return: Boolean true/false depending on whether the indicator exists. 810 @raise ValueError: If a path cannot be encoded properly. 811 """ 812 try: 813 if collectIndicator is None: 814 sourceFile = os.path.join(self.collectDir, DEF_COLLECT_INDICATOR) 815 targetFile = os.path.join(self.workingDir, DEF_COLLECT_INDICATOR) 816 else: 817 collectIndicator = encodePath(collectIndicator) 818 sourceFile = os.path.join(self.collectDir, collectIndicator) 819 targetFile = os.path.join(self.workingDir, collectIndicator) 820 logger.debug("Fetch remote [%s] into [%s].", sourceFile, targetFile) 821 if os.path.exists(targetFile): 822 try: 823 os.remove(targetFile) 824 except: 825 raise Exception("Error: collect indicator [%s] already exists!" % targetFile) 826 try: 827 RemotePeer._copyRemoteFile(self.remoteUser, self.localUser, self.name, 828 self._rcpCommand, self._rcpCommandList, 829 sourceFile, targetFile, 830 overwrite=False) 831 if os.path.exists(targetFile): 832 return True 833 else: 834 return False 835 except Exception as e: 836 logger.info("Failed looking for collect indicator: %s", e) 837 return False 838 finally: 839 if os.path.exists(targetFile): 840 try: 841 os.remove(targetFile) 842 except: pass

843

844 - def writeStageIndicator(self, stageIndicator=None):

845 """ 846 Writes the stage indicator in the peer's staging directory. 847 848 When the master has completed collecting its backup files, it will write 849 an empty indicator file into the peer's collect directory. The presence 850 of this file implies that the staging process is complete. 851 852 If you need to, you can override the name of the stage indicator file by 853 passing in a different name. 854 855 @note: If you have user/group as strings, call the L{util.getUidGid} function 856 to get the associated uid/gid as an ownership tuple. 857 858 @param stageIndicator: Name of the indicator file to write 859 @type stageIndicator: String representing name of a file in the collect directory 860 861 @raise ValueError: If a path cannot be encoded properly. 862 @raise IOError: If there is an IO error creating the file. 863 @raise OSError: If there is an OS error creating or changing permissions on the file 864 """ 865 stageIndicator = encodePath(stageIndicator) 866 if stageIndicator is None: 867 sourceFile = os.path.join(self.workingDir, DEF_STAGE_INDICATOR) 868 targetFile = os.path.join(self.collectDir, DEF_STAGE_INDICATOR) 869 else: 870 sourceFile = os.path.join(self.workingDir, DEF_STAGE_INDICATOR) 871 targetFile = os.path.join(self.collectDir, stageIndicator) 872 try: 873 if not os.path.exists(sourceFile): 874 open(sourceFile, "w").write("") 875 RemotePeer._pushLocalFile(self.remoteUser, self.localUser, self.name, 876 self._rcpCommand, self._rcpCommandList, 877 sourceFile, targetFile) 878 finally: 879 if os.path.exists(sourceFile): 880 try: 881 os.remove(sourceFile) 882 except: pass

883

884 - def executeRemoteCommand(self, command):

885 """ 886 Executes a command on the peer via remote shell. 887 888 @param command: Command to execute 889 @type command: String command-line suitable for use with rsh. 890 891 @raise IOError: If there is an error executing the command on the remote peer. 892 """ 893 RemotePeer._executeRemoteCommand(self.remoteUser, self.localUser, 894 self.name, self._rshCommand, 895 self._rshCommandList, command)

896

897 - def executeManagedAction(self, action, fullBackup):

898 """ 899 Executes a managed action on this peer. 900 901 @param action: Name of the action to execute. 902 @param fullBackup: Whether a full backup should be executed. 903 904 @raise IOError: If there is an error executing the action on the remote peer. 905 """ 906 try: 907 command = RemotePeer._buildCbackCommand(self.cbackCommand, action, fullBackup) 908 self.executeRemoteCommand(command) 909 except IOError as e: 910 logger.info(e) 911 raise IOError("Failed to execute action [%s] on managed client [%s]." % (action, self.name))

912 913 914 ################## 915 # Private methods 916 ################## 917 918 @staticmethod

919 - def _getDirContents(path):

920 """ 921 Returns the contents of a directory in terms of a Set. 922 923 The directory's contents are read as a L{FilesystemList} containing only 924 files, and then the list is converted into a set object for later use. 925 926 @param path: Directory path to get contents for 927 @type path: String representing a path on disk 928 929 @return: Set of files in the directory 930 @raise ValueError: If path is not a directory or does not exist. 931 """ 932 contents = FilesystemList() 933 contents.excludeDirs = True 934 contents.excludeLinks = True 935 contents.addDirContents(path) 936 return set(contents)

937 938 @staticmethod

939 - def _copyRemoteDir(remoteUser, localUser, remoteHost, rcpCommand, rcpCommandList, 940 sourceDir, targetDir, ownership=None, permissions=None):

941 """ 942 Copies files from the source directory to the target directory. 943 944 This function is not recursive. Only the files in the directory will be 945 copied. Ownership and permissions will be left at their default values 946 if new values are not specified. Behavior when copying soft links from 947 the collect directory is dependent on the behavior of the specified rcp 948 command. 949 950 @note: The returned count of copied files might be inaccurate if some of 951 the copied files already existed in the staging directory prior to the 952 copy taking place. We don't clear the staging directory first, because 953 some extension might also be using it. 954 955 @note: If you have user/group as strings, call the L{util.getUidGid} function 956 to get the associated uid/gid as an ownership tuple. 957 958 @note: We don't have a good way of knowing exactly what files we copied 959 down from the remote peer, unless we want to parse the output of the rcp 960 command (ugh). We could change permissions on everything in the target 961 directory, but that's kind of ugly too. Instead, we use Python's set 962 functionality to figure out what files were added while we executed the 963 rcp command. This isn't perfect - for instance, it's not correct if 964 someone else is messing with the directory at the same time we're doing 965 the remote copy - but it's about as good as we're going to get. 966 967 @note: Apparently, we can't count on all rcp-compatible implementations 968 to return sensible errors for some error conditions. As an example, the 969 C{scp} command in Debian 'woody' returns a zero (normal) status even 970 when it can't find a host or if the login or path is invalid. We try 971 to work around this by issuing C{IOError} if we don't copy any files from 972 the remote host. 973 974 @param remoteUser: Name of the Cedar Backup user on the remote peer 975 @type remoteUser: String representing a username, valid via the copy command 976 977 @param localUser: Name of the Cedar Backup user on the current host 978 @type localUser: String representing a username, valid on the current host 979 980 @param remoteHost: Hostname of the remote peer 981 @type remoteHost: String representing a hostname, accessible via the copy command 982 983 @param rcpCommand: An rcp-compatible copy command to use for copying files from the peer 984 @type rcpCommand: String representing a system command including required arguments 985 986 @param rcpCommandList: An rcp-compatible copy command to use for copying files 987 @type rcpCommandList: Command as a list to be passed to L{util.executeCommand} 988 989 @param sourceDir: Source directory 990 @type sourceDir: String representing a directory on disk 991 992 @param targetDir: Target directory 993 @type targetDir: String representing a directory on disk 994 995 @param ownership: Owner and group that the copied files should have 996 @type ownership: Tuple of numeric ids C{(uid, gid)} 997 998 @param permissions: Permissions that the staged files should have 999 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}). 1000 1001 @return: Number of files copied from the source directory to the target directory. 1002 1003 @raise ValueError: If source or target is not a directory or does not exist. 1004 @raise IOError: If there is an IO error copying the files. 1005 """ 1006 beforeSet = RemotePeer._getDirContents(targetDir) 1007 if localUser is not None: 1008 try: 1009 if not isRunningAsRoot(): 1010 raise IOError("Only root can remote copy as another user.") 1011 except AttributeError: pass 1012 actualCommand = "%s %s@%s:%s/* %s" % (rcpCommand, remoteUser, remoteHost, sourceDir, targetDir) 1013 command = resolveCommand(SU_COMMAND) 1014 result = executeCommand(command, [localUser, "-c", actualCommand])[0] 1015 if result != 0: 1016 raise IOError("Error (%d) copying files from remote host as local user [%s]." % (result, localUser)) 1017 else: 1018 copySource = "%s@%s:%s/*" % (remoteUser, remoteHost, sourceDir) 1019 command = resolveCommand(rcpCommandList) 1020 result = executeCommand(command, [copySource, targetDir])[0] 1021 if result != 0: 1022 raise IOError("Error (%d) copying files from remote host." % result) 1023 afterSet = RemotePeer._getDirContents(targetDir) 1024 if len(afterSet) == 0: 1025 raise IOError("Did not copy any files from remote peer.") 1026 differenceSet = afterSet.difference(beforeSet) # files we added as part of copy 1027 if len(differenceSet) == 0: 1028 raise IOError("Apparently did not copy any new files from remote peer.") 1029 for targetFile in differenceSet: 1030 if ownership is not None: 1031 os.chown(targetFile, ownership[0], ownership[1]) 1032 if permissions is not None: 1033 os.chmod(targetFile, permissions) 1034 return len(differenceSet)

1035 1036 @staticmethod

1037 - def _copyRemoteFile(remoteUser, localUser, remoteHost, 1038 rcpCommand, rcpCommandList, 1039 sourceFile, targetFile, ownership=None, 1040 permissions=None, overwrite=True):

1041 """ 1042 Copies a remote source file to a target file. 1043 1044 @note: Internally, we have to go through and escape any spaces in the 1045 source path with double-backslash, otherwise things get screwed up. It 1046 doesn't seem to be required in the target path. I hope this is portable 1047 to various different rcp methods, but I guess it might not be (all I have 1048 to test with is OpenSSH). 1049 1050 @note: If you have user/group as strings, call the L{util.getUidGid} function 1051 to get the associated uid/gid as an ownership tuple. 1052 1053 @note: We will not overwrite a target file that exists when this method 1054 is invoked. If the target already exists, we'll raise an exception. 1055 1056 @note: Apparently, we can't count on all rcp-compatible implementations 1057 to return sensible errors for some error conditions. As an example, the 1058 C{scp} command in Debian 'woody' returns a zero (normal) status even when 1059 it can't find a host or if the login or path is invalid. We try to work 1060 around this by issuing C{IOError} the target file does not exist when 1061 we're done. 1062 1063 @param remoteUser: Name of the Cedar Backup user on the remote peer 1064 @type remoteUser: String representing a username, valid via the copy command 1065 1066 @param remoteHost: Hostname of the remote peer 1067 @type remoteHost: String representing a hostname, accessible via the copy command 1068 1069 @param localUser: Name of the Cedar Backup user on the current host 1070 @type localUser: String representing a username, valid on the current host 1071 1072 @param rcpCommand: An rcp-compatible copy command to use for copying files from the peer 1073 @type rcpCommand: String representing a system command including required arguments 1074 1075 @param rcpCommandList: An rcp-compatible copy command to use for copying files 1076 @type rcpCommandList: Command as a list to be passed to L{util.executeCommand} 1077 1078 @param sourceFile: Source file to copy 1079 @type sourceFile: String representing a file on disk, as an absolute path 1080 1081 @param targetFile: Target file to create 1082 @type targetFile: String representing a file on disk, as an absolute path 1083 1084 @param ownership: Owner and group that the copied should have 1085 @type ownership: Tuple of numeric ids C{(uid, gid)} 1086 1087 @param permissions: Permissions that the staged files should have 1088 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}). 1089 1090 @param overwrite: Indicates whether it's OK to overwrite the target file. 1091 @type overwrite: Boolean true/false. 1092 1093 @raise IOError: If the target file already exists. 1094 @raise IOError: If there is an IO error copying the file 1095 @raise OSError: If there is an OS error changing permissions on the file 1096 """ 1097 if not overwrite: 1098 if os.path.exists(targetFile): 1099 raise IOError("Target file [%s] already exists." % targetFile) 1100 if localUser is not None: 1101 try: 1102 if not isRunningAsRoot(): 1103 raise IOError("Only root can remote copy as another user.") 1104 except AttributeError: pass 1105 actualCommand = "%s %s@%s:%s %s" % (rcpCommand, remoteUser, remoteHost, sourceFile.replace(" ", "\\ "), targetFile) 1106 command = resolveCommand(SU_COMMAND) 1107 result = executeCommand(command, [localUser, "-c", actualCommand])[0] 1108 if result != 0: 1109 raise IOError("Error (%d) copying [%s] from remote host as local user [%s]." % (result, sourceFile, localUser)) 1110 else: 1111 copySource = "%s@%s:%s" % (remoteUser, remoteHost, sourceFile.replace(" ", "\\ ")) 1112 command = resolveCommand(rcpCommandList) 1113 result = executeCommand(command, [copySource, targetFile])[0] 1114 if result != 0: 1115 raise IOError("Error (%d) copying [%s] from remote host." % (result, sourceFile)) 1116 if not os.path.exists(targetFile): 1117 raise IOError("Apparently unable to copy file from remote host.") 1118 if ownership is not None: 1119 os.chown(targetFile, ownership[0], ownership[1]) 1120 if permissions is not None: 1121 os.chmod(targetFile, permissions)

1122 1123 @staticmethod

1124 - def _pushLocalFile(remoteUser, localUser, remoteHost, 1125 rcpCommand, rcpCommandList, 1126 sourceFile, targetFile, overwrite=True):

1127 """ 1128 Copies a local source file to a remote host. 1129 1130 @note: We will not overwrite a target file that exists when this method 1131 is invoked. If the target already exists, we'll raise an exception. 1132 1133 @note: Internally, we have to go through and escape any spaces in the 1134 source and target paths with double-backslash, otherwise things get 1135 screwed up. I hope this is portable to various different rcp methods, 1136 but I guess it might not be (all I have to test with is OpenSSH). 1137 1138 @note: If you have user/group as strings, call the L{util.getUidGid} function 1139 to get the associated uid/gid as an ownership tuple. 1140 1141 @param remoteUser: Name of the Cedar Backup user on the remote peer 1142 @type remoteUser: String representing a username, valid via the copy command 1143 1144 @param localUser: Name of the Cedar Backup user on the current host 1145 @type localUser: String representing a username, valid on the current host 1146 1147 @param remoteHost: Hostname of the remote peer 1148 @type remoteHost: String representing a hostname, accessible via the copy command 1149 1150 @param rcpCommand: An rcp-compatible copy command to use for copying files from the peer 1151 @type rcpCommand: String representing a system command including required arguments 1152 1153 @param rcpCommandList: An rcp-compatible copy command to use for copying files 1154 @type rcpCommandList: Command as a list to be passed to L{util.executeCommand} 1155 1156 @param sourceFile: Source file to copy 1157 @type sourceFile: String representing a file on disk, as an absolute path 1158 1159 @param targetFile: Target file to create 1160 @type targetFile: String representing a file on disk, as an absolute path 1161 1162 @param overwrite: Indicates whether it's OK to overwrite the target file. 1163 @type overwrite: Boolean true/false. 1164 1165 @raise IOError: If there is an IO error copying the file 1166 @raise OSError: If there is an OS error changing permissions on the file 1167 """ 1168 if not overwrite: 1169 if os.path.exists(targetFile): 1170 raise IOError("Target file [%s] already exists." % targetFile) 1171 if localUser is not None: 1172 try: 1173 if not isRunningAsRoot(): 1174 raise IOError("Only root can remote copy as another user.") 1175 except AttributeError: pass 1176 actualCommand = '%s "%s" "%s@%s:%s"' % (rcpCommand, sourceFile, remoteUser, remoteHost, targetFile) 1177 command = resolveCommand(SU_COMMAND) 1178 result = executeCommand(command, [localUser, "-c", actualCommand])[0] 1179 if result != 0: 1180 raise IOError("Error (%d) copying [%s] to remote host as local user [%s]." % (result, sourceFile, localUser)) 1181 else: 1182 copyTarget = "%s@%s:%s" % (remoteUser, remoteHost, targetFile.replace(" ", "\\ ")) 1183 command = resolveCommand(rcpCommandList) 1184 result = executeCommand(command, [sourceFile.replace(" ", "\\ "), copyTarget])[0] 1185 if result != 0: 1186 raise IOError("Error (%d) copying [%s] to remote host." % (result, sourceFile))

1187 1188 @staticmethod

1189 - def _executeRemoteCommand(remoteUser, localUser, remoteHost, rshCommand, rshCommandList, remoteCommand):

1190 """ 1191 Executes a command on the peer via remote shell. 1192 1193 @param remoteUser: Name of the Cedar Backup user on the remote peer 1194 @type remoteUser: String representing a username, valid on the remote host 1195 1196 @param localUser: Name of the Cedar Backup user on the current host 1197 @type localUser: String representing a username, valid on the current host 1198 1199 @param remoteHost: Hostname of the remote peer 1200 @type remoteHost: String representing a hostname, accessible via the copy command 1201 1202 @param rshCommand: An rsh-compatible copy command to use for remote shells to the peer 1203 @type rshCommand: String representing a system command including required arguments 1204 1205 @param rshCommandList: An rsh-compatible copy command to use for remote shells to the peer 1206 @type rshCommandList: Command as a list to be passed to L{util.executeCommand} 1207 1208 @param remoteCommand: The command to be executed on the remote host 1209 @type remoteCommand: String command-line, with no special shell characters ($, <, etc.) 1210 1211 @raise IOError: If there is an error executing the remote command 1212 """ 1213 actualCommand = "%s %s@%s '%s'" % (rshCommand, remoteUser, remoteHost, remoteCommand) 1214 if localUser is not None: 1215 try: 1216 if not isRunningAsRoot(): 1217 raise IOError("Only root can remote shell as another user.") 1218 except AttributeError: pass 1219 command = resolveCommand(SU_COMMAND) 1220 result = executeCommand(command, [localUser, "-c", actualCommand])[0] 1221 if result != 0: 1222 raise IOError("Command failed [su -c %s \"%s\"]" % (localUser, actualCommand)) 1223 else: 1224 command = resolveCommand(rshCommandList) 1225 result = executeCommand(command, ["%s@%s" % (remoteUser, remoteHost), "%s" % remoteCommand])[0] 1226 if result != 0: 1227 raise IOError("Command failed [%s]" % (actualCommand))

1228 1229 @staticmethod

1230 - def _buildCbackCommand(cbackCommand, action, fullBackup):

1231 """ 1232 Builds a Cedar Backup command line for the named action. 1233 1234 @note: If the cback command is None, then DEF_CBACK_COMMAND is used. 1235 1236 @param cbackCommand: cback command to execute, including required options 1237 @param action: Name of the action to execute. 1238 @param fullBackup: Whether a full backup should be executed. 1239 1240 @return: String suitable for passing to L{_executeRemoteCommand} as remoteCommand. 1241 @raise ValueError: If action is None. 1242 """ 1243 if action is None: 1244 raise ValueError("Action cannot be None.") 1245 if cbackCommand is None: 1246 cbackCommand = DEF_CBACK_COMMAND 1247 if fullBackup: 1248 return "%s --full %s" % (cbackCommand, action) 1249 else: 1250 return "%s %s" % (cbackCommand, action)

Source Code for Module CedarBackup3.peer