1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41 """
42 Provides an extension to back up PostgreSQL databases.
43
44 This is a Cedar Backup extension used to back up PostgreSQL databases via the
45 Cedar Backup command line. It requires a new configurations section
46 <postgresql> and is intended to be run either immediately before or immediately
47 after the standard collect action. Aside from its own configuration, it
48 requires the options and collect configuration sections in the standard Cedar
49 Backup configuration file.
50
51 The backup is done via the C{pg_dump} or C{pg_dumpall} commands included with
52 the PostgreSQL product. Output can be compressed using C{gzip} or C{bzip2}.
53 Administrators can configure the extension either to back up all databases or
54 to back up only specific databases. The extension assumes that the current
55 user has passwordless access to the database since there is no easy way to pass
56 a password to the C{pg_dump} client. This can be accomplished using appropriate
57 voodoo in the C{pg_hda.conf} file.
58
59 Note that this code always produces a full backup. There is currently no
60 facility for making incremental backups.
61
62 You should always make C{/etc/cback3.conf} unreadble to non-root users once you
63 place postgresql configuration into it, since postgresql configuration will
64 contain information about available PostgreSQL databases and usernames.
65
66 Use of this extension I{may} expose usernames in the process listing (via
67 C{ps}) when the backup is running if the username is specified in the
68 configuration.
69
70 @author: Kenneth J. Pronovici <pronovic@ieee.org>
71 @author: Antoine Beaupre <anarcat@koumbit.org>
72 """
73
74
75
76
77
78
79 import os
80 import logging
81 from gzip import GzipFile
82 from bz2 import BZ2File
83 from functools import total_ordering
84
85
86 from CedarBackup3.xmlutil import createInputDom, addContainerNode, addStringNode, addBooleanNode
87 from CedarBackup3.xmlutil import readFirstChild, readString, readStringList, readBoolean
88 from CedarBackup3.config import VALID_COMPRESS_MODES
89 from CedarBackup3.util import resolveCommand, executeCommand
90 from CedarBackup3.util import ObjectTypeList, changeOwnership
91
92
93
94
95
96
97 logger = logging.getLogger("CedarBackup3.log.extend.postgresql")
98 POSTGRESQLDUMP_COMMAND = [ "pg_dump", ]
99 POSTGRESQLDUMPALL_COMMAND = [ "pg_dumpall", ]
100
101
102
103
104
105
106 @total_ordering
107 -class PostgresqlConfig(object):
108
109 """
110 Class representing PostgreSQL configuration.
111
112 The PostgreSQL configuration information is used for backing up PostgreSQL databases.
113
114 The following restrictions exist on data in this class:
115
116 - The compress mode must be one of the values in L{VALID_COMPRESS_MODES}.
117 - The 'all' flag must be 'Y' if no databases are defined.
118 - The 'all' flag must be 'N' if any databases are defined.
119 - Any values in the databases list must be strings.
120
121 @sort: __init__, __repr__, __str__, __cmp__, __eq__, __lt__, __gt__, user,
122 all, databases
123 """
124
125 - def __init__(self, user=None, compressMode=None, all=None, databases=None):
126 """
127 Constructor for the C{PostgresqlConfig} class.
128
129 @param user: User to execute backup as.
130 @param compressMode: Compress mode for backed-up files.
131 @param all: Indicates whether to back up all databases.
132 @param databases: List of databases to back up.
133 """
134 self._user = None
135 self._compressMode = None
136 self._all = None
137 self._databases = None
138 self.user = user
139 self.compressMode = compressMode
140 self.all = all
141 self.databases = databases
142
143 - def __repr__(self):
144 """
145 Official string representation for class instance.
146 """
147 return "PostgresqlConfig(%s, %s, %s)" % (self.user, self.all, self.databases)
148
150 """
151 Informal string representation for class instance.
152 """
153 return self.__repr__()
154
155 - def __eq__(self, other):
156 """Equals operator, iplemented in terms of original Python 2 compare operator."""
157 return self.__cmp__(other) == 0
158
159 - def __lt__(self, other):
160 """Less-than operator, iplemented in terms of original Python 2 compare operator."""
161 return self.__cmp__(other) < 0
162
163 - def __gt__(self, other):
164 """Greater-than operator, iplemented in terms of original Python 2 compare operator."""
165 return self.__cmp__(other) > 0
166
167 - def __cmp__(self, other):
168 """
169 Original Python 2 comparison operator.
170 @param other: Other object to compare to.
171 @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other.
172 """
173 if other is None:
174 return 1
175 if self.user != other.user:
176 if str(self.user or "") < str(other.user or ""):
177 return -1
178 else:
179 return 1
180 if self.compressMode != other.compressMode:
181 if str(self.compressMode or "") < str(other.compressMode or ""):
182 return -1
183 else:
184 return 1
185 if self.all != other.all:
186 if self.all < other.all:
187 return -1
188 else:
189 return 1
190 if self.databases != other.databases:
191 if self.databases < other.databases:
192 return -1
193 else:
194 return 1
195 return 0
196
197 - def _setUser(self, value):
198 """
199 Property target used to set the user value.
200 """
201 if value is not None:
202 if len(value) < 1:
203 raise ValueError("User must be non-empty string.")
204 self._user = value
205
206 - def _getUser(self):
207 """
208 Property target used to get the user value.
209 """
210 return self._user
211
212 - def _setCompressMode(self, value):
213 """
214 Property target used to set the compress mode.
215 If not C{None}, the mode must be one of the values in L{VALID_COMPRESS_MODES}.
216 @raise ValueError: If the value is not valid.
217 """
218 if value is not None:
219 if value not in VALID_COMPRESS_MODES:
220 raise ValueError("Compress mode must be one of %s." % VALID_COMPRESS_MODES)
221 self._compressMode = value
222
224 """
225 Property target used to get the compress mode.
226 """
227 return self._compressMode
228
229 - def _setAll(self, value):
230 """
231 Property target used to set the 'all' flag.
232 No validations, but we normalize the value to C{True} or C{False}.
233 """
234 if value:
235 self._all = True
236 else:
237 self._all = False
238
240 """
241 Property target used to get the 'all' flag.
242 """
243 return self._all
244
245 - def _setDatabases(self, value):
246 """
247 Property target used to set the databases list.
248 Either the value must be C{None} or each element must be a string.
249 @raise ValueError: If the value is not a string.
250 """
251 if value is None:
252 self._databases = None
253 else:
254 for database in value:
255 if len(database) < 1:
256 raise ValueError("Each database must be a non-empty string.")
257 try:
258 saved = self._databases
259 self._databases = ObjectTypeList(str, "string")
260 self._databases.extend(value)
261 except Exception as e:
262 self._databases = saved
263 raise e
264
265 - def _getDatabases(self):
266 """
267 Property target used to get the databases list.
268 """
269 return self._databases
270
271 user = property(_getUser, _setUser, None, "User to execute backup as.")
272 compressMode = property(_getCompressMode, _setCompressMode, None, "Compress mode to be used for backed-up files.")
273 all = property(_getAll, _setAll, None, "Indicates whether to back up all databases.")
274 databases = property(_getDatabases, _setDatabases, None, "List of databases to back up.")
275
276
277
278
279
280
281 @total_ordering
282 -class LocalConfig(object):
283
284 """
285 Class representing this extension's configuration document.
286
287 This is not a general-purpose configuration object like the main Cedar
288 Backup configuration object. Instead, it just knows how to parse and emit
289 PostgreSQL-specific configuration values. Third parties who need to read and
290 write configuration related to this extension should access it through the
291 constructor, C{validate} and C{addConfig} methods.
292
293 @note: Lists within this class are "unordered" for equality comparisons.
294
295 @sort: __init__, __repr__, __str__, __cmp__, __eq__, __lt__, __gt__,
296 postgresql, validate, addConfig
297 """
298
299 - def __init__(self, xmlData=None, xmlPath=None, validate=True):
300 """
301 Initializes a configuration object.
302
303 If you initialize the object without passing either C{xmlData} or
304 C{xmlPath} then configuration will be empty and will be invalid until it
305 is filled in properly.
306
307 No reference to the original XML data or original path is saved off by
308 this class. Once the data has been parsed (successfully or not) this
309 original information is discarded.
310
311 Unless the C{validate} argument is C{False}, the L{LocalConfig.validate}
312 method will be called (with its default arguments) against configuration
313 after successfully parsing any passed-in XML. Keep in mind that even if
314 C{validate} is C{False}, it might not be possible to parse the passed-in
315 XML document if lower-level validations fail.
316
317 @note: It is strongly suggested that the C{validate} option always be set
318 to C{True} (the default) unless there is a specific need to read in
319 invalid configuration from disk.
320
321 @param xmlData: XML data representing configuration.
322 @type xmlData: String data.
323
324 @param xmlPath: Path to an XML file on disk.
325 @type xmlPath: Absolute path to a file on disk.
326
327 @param validate: Validate the document after parsing it.
328 @type validate: Boolean true/false.
329
330 @raise ValueError: If both C{xmlData} and C{xmlPath} are passed-in.
331 @raise ValueError: If the XML data in C{xmlData} or C{xmlPath} cannot be parsed.
332 @raise ValueError: If the parsed configuration document is not valid.
333 """
334 self._postgresql = None
335 self.postgresql = None
336 if xmlData is not None and xmlPath is not None:
337 raise ValueError("Use either xmlData or xmlPath, but not both.")
338 if xmlData is not None:
339 self._parseXmlData(xmlData)
340 if validate:
341 self.validate()
342 elif xmlPath is not None:
343 xmlData = open(xmlPath).read()
344 self._parseXmlData(xmlData)
345 if validate:
346 self.validate()
347
349 """
350 Official string representation for class instance.
351 """
352 return "LocalConfig(%s)" % (self.postgresql)
353
355 """
356 Informal string representation for class instance.
357 """
358 return self.__repr__()
359
361 """Equals operator, iplemented in terms of original Python 2 compare operator."""
362 return self.__cmp__(other) == 0
363
365 """Less-than operator, iplemented in terms of original Python 2 compare operator."""
366 return self.__cmp__(other) < 0
367
369 """Greater-than operator, iplemented in terms of original Python 2 compare operator."""
370 return self.__cmp__(other) > 0
371
373 """
374 Original Python 2 comparison operator.
375 Lists within this class are "unordered" for equality comparisons.
376 @param other: Other object to compare to.
377 @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other.
378 """
379 if other is None:
380 return 1
381 if self.postgresql != other.postgresql:
382 if self.postgresql < other.postgresql:
383 return -1
384 else:
385 return 1
386 return 0
387
388 - def _setPostgresql(self, value):
389 """
390 Property target used to set the postgresql configuration value.
391 If not C{None}, the value must be a C{PostgresqlConfig} object.
392 @raise ValueError: If the value is not a C{PostgresqlConfig}
393 """
394 if value is None:
395 self._postgresql = None
396 else:
397 if not isinstance(value, PostgresqlConfig):
398 raise ValueError("Value must be a C{PostgresqlConfig} object.")
399 self._postgresql = value
400
401 - def _getPostgresql(self):
402 """
403 Property target used to get the postgresql configuration value.
404 """
405 return self._postgresql
406
407 postgresql = property(_getPostgresql, _setPostgresql, None, "Postgresql configuration in terms of a C{PostgresqlConfig} object.")
408
410 """
411 Validates configuration represented by the object.
412
413 The compress mode must be filled in. Then, if the 'all' flag
414 I{is} set, no databases are allowed, and if the 'all' flag is
415 I{not} set, at least one database is required.
416
417 @raise ValueError: If one of the validations fails.
418 """
419 if self.postgresql is None:
420 raise ValueError("PostgreSQL section is required.")
421 if self.postgresql.compressMode is None:
422 raise ValueError("Compress mode value is required.")
423 if self.postgresql.all:
424 if self.postgresql.databases is not None and self.postgresql.databases != []:
425 raise ValueError("Databases cannot be specified if 'all' flag is set.")
426 else:
427 if self.postgresql.databases is None or len(self.postgresql.databases) < 1:
428 raise ValueError("At least one PostgreSQL database must be indicated if 'all' flag is not set.")
429
431 """
432 Adds a <postgresql> configuration section as the next child of a parent.
433
434 Third parties should use this function to write configuration related to
435 this extension.
436
437 We add the following fields to the document::
438
439 user //cb_config/postgresql/user
440 compressMode //cb_config/postgresql/compress_mode
441 all //cb_config/postgresql/all
442
443 We also add groups of the following items, one list element per
444 item::
445
446 database //cb_config/postgresql/database
447
448 @param xmlDom: DOM tree as from C{impl.createDocument()}.
449 @param parentNode: Parent that the section should be appended to.
450 """
451 if self.postgresql is not None:
452 sectionNode = addContainerNode(xmlDom, parentNode, "postgresql")
453 addStringNode(xmlDom, sectionNode, "user", self.postgresql.user)
454 addStringNode(xmlDom, sectionNode, "compress_mode", self.postgresql.compressMode)
455 addBooleanNode(xmlDom, sectionNode, "all", self.postgresql.all)
456 if self.postgresql.databases is not None:
457 for database in self.postgresql.databases:
458 addStringNode(xmlDom, sectionNode, "database", database)
459
461 """
462 Internal method to parse an XML string into the object.
463
464 This method parses the XML document into a DOM tree (C{xmlDom}) and then
465 calls a static method to parse the postgresql configuration section.
466
467 @param xmlData: XML data to be parsed
468 @type xmlData: String data
469
470 @raise ValueError: If the XML cannot be successfully parsed.
471 """
472 (xmlDom, parentNode) = createInputDom(xmlData)
473 self._postgresql = LocalConfig._parsePostgresql(parentNode)
474
475 @staticmethod
476 - def _parsePostgresql(parent):
477 """
478 Parses a postgresql configuration section.
479
480 We read the following fields::
481
482 user //cb_config/postgresql/user
483 compressMode //cb_config/postgresql/compress_mode
484 all //cb_config/postgresql/all
485
486 We also read groups of the following item, one list element per
487 item::
488
489 databases //cb_config/postgresql/database
490
491 @param parent: Parent node to search beneath.
492
493 @return: C{PostgresqlConfig} object or C{None} if the section does not exist.
494 @raise ValueError: If some filled-in value is invalid.
495 """
496 postgresql = None
497 section = readFirstChild(parent, "postgresql")
498 if section is not None:
499 postgresql = PostgresqlConfig()
500 postgresql.user = readString(section, "user")
501 postgresql.compressMode = readString(section, "compress_mode")
502 postgresql.all = readBoolean(section, "all")
503 postgresql.databases = readStringList(section, "database")
504 return postgresql
505
506
507
508
509
510
511
512
513
514
515 -def executeAction(configPath, options, config):
516 """
517 Executes the PostgreSQL backup action.
518
519 @param configPath: Path to configuration file on disk.
520 @type configPath: String representing a path on disk.
521
522 @param options: Program command-line options.
523 @type options: Options object.
524
525 @param config: Program configuration.
526 @type config: Config object.
527
528 @raise ValueError: Under many generic error conditions
529 @raise IOError: If a backup could not be written for some reason.
530 """
531 logger.debug("Executing PostgreSQL extended action.")
532 if config.options is None or config.collect is None:
533 raise ValueError("Cedar Backup configuration is not properly filled in.")
534 local = LocalConfig(xmlPath=configPath)
535 if local.postgresql.all:
536 logger.info("Backing up all databases.")
537 _backupDatabase(config.collect.targetDir, local.postgresql.compressMode, local.postgresql.user,
538 config.options.backupUser, config.options.backupGroup, None)
539 if local.postgresql.databases is not None and local.postgresql.databases != []:
540 logger.debug("Backing up %d individual databases.", len(local.postgresql.databases))
541 for database in local.postgresql.databases:
542 logger.info("Backing up database [%s].", database)
543 _backupDatabase(config.collect.targetDir, local.postgresql.compressMode, local.postgresql.user,
544 config.options.backupUser, config.options.backupGroup, database)
545 logger.info("Executed the PostgreSQL extended action successfully.")
546
547 -def _backupDatabase(targetDir, compressMode, user, backupUser, backupGroup, database=None):
548 """
549 Backs up an individual PostgreSQL database, or all databases.
550
551 This internal method wraps the public method and adds some functionality,
552 like figuring out a filename, etc.
553
554 @param targetDir: Directory into which backups should be written.
555 @param compressMode: Compress mode to be used for backed-up files.
556 @param user: User to use for connecting to the database.
557 @param backupUser: User to own resulting file.
558 @param backupGroup: Group to own resulting file.
559 @param database: Name of database, or C{None} for all databases.
560
561 @return: Name of the generated backup file.
562
563 @raise ValueError: If some value is missing or invalid.
564 @raise IOError: If there is a problem executing the PostgreSQL dump.
565 """
566 (outputFile, filename) = _getOutputFile(targetDir, database, compressMode)
567 try:
568 backupDatabase(user, outputFile, database)
569 finally:
570 outputFile.close()
571 if not os.path.exists(filename):
572 raise IOError("Dump file [%s] does not seem to exist after backup completed." % filename)
573 changeOwnership(filename, backupUser, backupGroup)
574
576 """
577 Opens the output file used for saving the PostgreSQL dump.
578
579 The filename is either C{"postgresqldump.txt"} or
580 C{"postgresqldump-<database>.txt"}. The C{".gz"} or C{".bz2"} extension is
581 added if C{compress} is C{True}.
582
583 @param targetDir: Target directory to write file in.
584 @param database: Name of the database (if any)
585 @param compressMode: Compress mode to be used for backed-up files.
586
587 @return: Tuple of (Output file object, filename)
588 """
589 if database is None:
590 filename = os.path.join(targetDir, "postgresqldump.txt")
591 else:
592 filename = os.path.join(targetDir, "postgresqldump-%s.txt" % database)
593 if compressMode == "gzip":
594 filename = "%s.gz" % filename
595 outputFile = GzipFile(filename, "w")
596 elif compressMode == "bzip2":
597 filename = "%s.bz2" % filename
598 outputFile = BZ2File(filename, "w")
599 else:
600 outputFile = open(filename, "wb")
601 logger.debug("PostgreSQL dump file will be [%s].", filename)
602 return (outputFile, filename)
603
604
605
606
607
608
609 -def backupDatabase(user, backupFile, database=None):
610 """
611 Backs up an individual PostgreSQL database, or all databases.
612
613 This function backs up either a named local PostgreSQL database or all local
614 PostgreSQL databases, using the passed in user for connectivity.
615 This is I{always} a full backup. There is no facility for incremental
616 backups.
617
618 The backup data will be written into the passed-in back file. Normally,
619 this would be an object as returned from C{open()}, but it is possible to
620 use something like a C{GzipFile} to write compressed output. The caller is
621 responsible for closing the passed-in backup file.
622
623 @note: Typically, you would use the C{root} user to back up all databases.
624
625 @param user: User to use for connecting to the database.
626 @type user: String representing PostgreSQL username.
627
628 @param backupFile: File use for writing backup.
629 @type backupFile: Python file object as from C{open()} or C{file()}.
630
631 @param database: Name of the database to be backed up.
632 @type database: String representing database name, or C{None} for all databases.
633
634 @raise ValueError: If some value is missing or invalid.
635 @raise IOError: If there is a problem executing the PostgreSQL dump.
636 """
637 args = []
638 if user is not None:
639 args.append('-U')
640 args.append(user)
641
642 if database is None:
643 command = resolveCommand(POSTGRESQLDUMPALL_COMMAND)
644 else:
645 command = resolveCommand(POSTGRESQLDUMP_COMMAND)
646 args.append(database)
647
648 result = executeCommand(command, args, returnOutput=False, ignoreStderr=True, doNotLog=True, outputFile=backupFile)[0]
649 if result != 0:
650 if database is None:
651 raise IOError("Error [%d] executing PostgreSQL database dump for all databases." % result)
652 else:
653 raise IOError("Error [%d] executing PostgreSQL database dump for database [%s]." % (result, database))
654