001: /**
002: * Sequoia: Database clustering technology.
003: * Copyright (C) 2005 Emic Networks
004: * Contact: sequoia@continuent.org
005: *
006: * Licensed under the Apache License, Version 2.0 (the "License");
007: * you may not use this file except in compliance with the License.
008: * You may obtain a copy of the License at
009: *
010: * http://www.apache.org/licenses/LICENSE-2.0
011: *
012: * Unless required by applicable law or agreed to in writing, software
013: * distributed under the License is distributed on an "AS IS" BASIS,
014: * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015: * See the License for the specific language governing permissions and
016: * limitations under the License.
017: *
018: * Initial developer(s): Emmanuel Cecchet.
019: * Contributor(s): Dylan Hansen, Mathieu Peltier.
020: */package org.continuent.sequoia.controller.backup.backupers;
021:
022: import java.io.File;
023: import java.io.IOException;
024: import java.util.ArrayList;
025: import java.util.Iterator;
026: import java.util.regex.Matcher;
027: import java.util.regex.Pattern;
028:
029: import org.continuent.sequoia.common.exceptions.BackupException;
030: import org.continuent.sequoia.common.log.Trace;
031: import org.continuent.sequoia.controller.backup.Backuper;
032:
033: /**
034: * This abstract class provides base methods for PostgreSQL backupers.
035: * <p>
036: * Currently the Backupers takes 5 parameters (all are optional):
037: * <ul>
038: * <li>bindir: path to PostgreSQL binaries (if not set, the commands are
039: * searched in the path)</li>
040: * <li>encoding: the encoding of the database that is created upon restore</li>
041: * <li>authentication: flag to use PostgreSQL authentication (default false)</li>
042: * <li>dumpServer: address of interface to offer dumps</li>
043: * <li>preRestoreScript: location of script to run before restoring</li>
044: * <li>postRestoreScript: location of script to run after restoring</li>
045: * <li>dumpTimeout: Timeout period (seconds) while performing DB dump.</li>
046: * <li>restoreTimeout: Timeout period (seconds) while performin DB restore.</li>
047: * <li>pgDumpFlags: Extra pg_dump command-line options to use while performing
048: * DB dump.</li>
049: * </ul>
050: * More options can be easily added. This class makes calls to the pg_dump,
051: * createdb, dropdb, psql and pg_restore commands.
052: *
053: * @author <a href="mailto:dhansen@h2st.com">Dylan Hansen</a>
054: * @author <a href="mailto:mathieu.peltier@emicnetworks.com">Mathieu Peltier</a>
055: * @version 1.1
056: */
057: public abstract class AbstractPostgreSQLBackuper extends
058: AbstractBackuper {
059: private static final String DEFAULT_POSTGRESQL_HOST = "localhost";
060:
061: // Logger
062: private static Trace logger = Trace
063: .getLogger(AbstractPostgreSQLBackuper.class.getName());
064:
065: /** end user logger */
066: static Trace endUserLogger = Trace
067: .getLogger("org.continuent.sequoia.enduser");
068:
069: // CommandExec instance for running native commands.
070: protected NativeCommandExec nativeCmdExec = new NativeCommandExec();
071:
072: // Static variables for option values
073: protected static String binDir = null;
074: protected static String encoding = null;
075: protected static boolean useAuthentication = false;
076: protected static String dumpServer = null;
077: protected static String preRestoreScript = null;
078: protected static String postRestoreScript = null;
079: protected static String pgDumpFlags = null;
080: protected static String dumpTimeout = null;
081: protected static String restoreTimeout = null;
082: protected static String splitSize = "1000m";
083:
084: protected String getPsqlCommand() {
085: return "psql";
086: }
087:
088: protected String getJdbcUrlPrefix() {
089: return "jdbc:postgresql:";
090: }
091:
092: protected int getDefaultPort() {
093: return 5432;
094: }
095:
096: /**
097: * Creates a new <code>AbstractPostgreSQLBackuper</code> object
098: */
099: public AbstractPostgreSQLBackuper() {
100: }
101:
102: /**
103: * @see Backuper#getOptions()
104: */
105: public String getOptions() {
106: return optionsString;
107: }
108:
109: /**
110: * @see Backuper#setOptions(java.lang.String)
111: */
112: public void setOptions(String options) {
113: super .setOptions(options);
114:
115: // Check the HashMap for options. Only set once, easier than
116: // checking HashMap every time a value is needed.
117: if (optionsMap.containsKey("bindir"))
118: binDir = (String) optionsMap.get("bindir");
119: if (optionsMap.containsKey("encoding"))
120: encoding = (String) optionsMap.get("encoding");
121: if (optionsMap.containsKey("authentication")
122: && ((String) optionsMap.get("authentication"))
123: .equalsIgnoreCase("true"))
124: useAuthentication = true;
125: if (optionsMap.containsKey("dumpServer"))
126: dumpServer = (String) optionsMap.get("dumpServer");
127: if (optionsMap.containsKey("preRestoreScript"))
128: preRestoreScript = (String) optionsMap
129: .get("preRestoreScript");
130: if (optionsMap.containsKey("postRestoreScript"))
131: postRestoreScript = (String) optionsMap
132: .get("postRestoreScript");
133: if (optionsMap.containsKey("dumpTimeout"))
134: dumpTimeout = (String) optionsMap.get("dumpTimeout");
135: if (optionsMap.containsKey("restoreTimeout"))
136: restoreTimeout = (String) optionsMap.get("restoreTimeout");
137: if (optionsMap.containsKey("pgDumpFlags"))
138: pgDumpFlags = (String) optionsMap.get("pgDumpFlags");
139: if (optionsMap.containsKey("splitSize"))
140: splitSize = (String) optionsMap.get("splitSize");
141:
142: }
143:
144: /**
145: * @see org.continuent.sequoia.controller.backup.Backuper#deleteDump(java.lang.String,
146: * java.lang.String)
147: */
148: public void deleteDump(String path, String dumpName)
149: throws BackupException {
150: File toRemove = new File(getDumpPhysicalPath(path, dumpName));
151: if (logger.isDebugEnabled())
152: logger.debug("Deleting dump " + toRemove);
153: toRemove.delete();
154: }
155:
156: /**
157: * Get the dump physical path from its logical name
158: *
159: * @param path the path where the dump is stored
160: * @param dumpName dump logical name
161: * @return path to dump file
162: */
163: protected String getDumpPhysicalPath(String path, String dumpName) {
164: String fullPath = null;
165:
166: if (path.endsWith(File.separator))
167: fullPath = path + dumpName;
168: else
169: fullPath = path + File.separator + dumpName;
170:
171: return fullPath;
172: }
173:
174: /**
175: * Creates a command string based on given info. Will append binDir if
176: * supplied.
177: *
178: * @param command Command to execute
179: * @param info URL info to parse and create parameters
180: * @param options Additional parameters
181: * @param login User login
182: * @return String containing full command to run
183: */
184: protected String makeCommand(String command,
185: PostgreSQLUrlInfo info, String options, String login) {
186: String prefix = binDir != null ? binDir + File.separator : "";
187: return prefix + command + " " + info.getHostParametersString()
188: + " -U " + login + " " + options + " " + info.dbName;
189: }
190:
191: /**
192: * Creates a command String array used my exec(). This method uses the
193: * "expect" command to pass the password parameter to the command being run,
194: * thus allowing PostgreSQL to allow authentication. Also uses binDir if
195: * supplied. Note the "psql" command has different output needed by "expect"
196: * than other commands.
197: *
198: * @param command Command to execute
199: * @param info URL info to parse adn add parameters
200: * @param options Additional parameters
201: * @param login User login
202: * @param password User password
203: * @param isPsql Is the command being run a psql command?
204: * @return String array containing full command to run
205: */
206: protected String[] makeCommandWithAuthentication(String command,
207: PostgreSQLUrlInfo info, String options, String login,
208: String password, boolean isPsql) {
209: // Build params for "spawn" command used by expect
210: StringBuffer cmdBuff = new StringBuffer("spawn ");
211:
212: if (binDir != null)
213: cmdBuff.append(binDir + File.separator);
214:
215: cmdBuff.append(command);
216: cmdBuff.append(" ");
217: cmdBuff.append(info.getHostParametersString());
218: cmdBuff.append(" -U ");
219: cmdBuff.append(login);
220: cmdBuff.append(" ");
221: cmdBuff.append(options);
222: cmdBuff.append(" ");
223: cmdBuff.append(info.dbName);
224: cmdBuff.append("; ");
225: // "psql" command has different message for password input
226: if (isPsql)
227: cmdBuff.append("expect \"Password:\"; ");
228: else
229: cmdBuff.append("expect \"Password for user " + login
230: + ":\"; ");
231: cmdBuff.append("send \"");
232: cmdBuff.append(password);
233: cmdBuff.append("\"; ");
234: cmdBuff.append("send \"\\r\"; ");
235: cmdBuff.append("expect eof;");
236:
237: String[] commands = { "expect", "-c", cmdBuff.toString() };
238: return commands;
239: }
240:
241: /**
242: * Creates a Splitted command string based on given info. Will append BINDIR
243: * if supplied.
244: *
245: * @param command Command to execute
246: * @param info URL info to parse and create parameters
247: * @param options Additional parameters
248: * @param login User login
249: * @return String containing full command to run
250: */
251: protected String[] makeSplitCommand(String command,
252: PostgreSQLUrlInfo info, String options, String login) {
253: String prefix = binDir != null ? binDir + File.separator : "";
254: String cmd = prefix + command + " " + info.dbName + " "
255: + info.getHostParametersString() + " -U " + login + " "
256: + options;
257: String[] cmdArray = { "bash", "-c", cmd };
258: return cmdArray;
259: // return prefix + command + " " + info.dbName + " " +
260: // info.getHostParametersString() + " -U " + login + " " + options;
261: }
262:
263: /**
264: * Creates an expect dialogue String array to be used as stadard input to a
265: * "expect" command, used to pass the password parameter to the command being
266: * run, thus allowing PostgreSQL to allow authentication. Also uses binDir if
267: * supplied. Note that the "psql" command has different password promting than
268: * other Postgres utilities. This expect-dialogue do a much better job at
269: * detecting errors than the detection you get when using
270: * makeCommandWithAuthentication() and executeNativeCommand(). Above all, it
271: * will report failure if the PG- command never promts for a password.
272: * Secondly, it will report failure if the invoked command exits with a
273: * non-zero exit-status.
274: *
275: * @param command Command to execute
276: * @param info URL info to parse adn add parameters
277: * @param options Additional parameters
278: * @param login User login
279: * @param password User password
280: * @param timeout Is the command being run a psql command?
281: * @return String array containing full Expect dialogue
282: */
283: protected String[] makeExpectDialogueWithAuthentication(
284: String command, PostgreSQLUrlInfo info, String options,
285: String login, String password, int timeout) {
286: String[] cmdBuff = new String[27];
287:
288: if (binDir != null)
289: cmdBuff[0] = new String("spawn " + binDir + File.separator
290: + command + " " + info.getHostParametersString()
291: + " -U " + login + " -W " + options + " "
292: + info.getDbName());
293: else
294: cmdBuff[0] = new String("spawn " + command + " "
295: + info.getHostParametersString() + " -U " + login
296: + " -W " + options + " " + info.getDbName());
297: cmdBuff[1] = new String("proc abort {} {");
298: cmdBuff[2] = new String(" system kill [exp_pid]");
299: cmdBuff[3] = new String(" exit 1");
300: cmdBuff[4] = new String("}");
301: cmdBuff[5] = new String("set exitcode 1");
302: cmdBuff[6] = new String("expect {");
303: cmdBuff[7] = new String(" \"Password for user " + login
304: + ":\" { set exitcode 0 }");
305: cmdBuff[8] = new String(" \"assword:\" { set exitcode 0 }");
306: cmdBuff[9] = new String(" timeout { abort }");
307: cmdBuff[10] = new String("}");
308: cmdBuff[11] = new String("send \"" + password + "\"");
309: cmdBuff[12] = new String("send \"\r\"");
310: cmdBuff[13] = new String("set timeout " + timeout);
311: cmdBuff[14] = new String("expect {");
312: cmdBuff[15] = new String(
313: " \"ERROR:\" { set exitcode 1; exp_continue -continue_timer }");
314: cmdBuff[16] = new String(
315: " \"FATAL:\" { set exitcode 1; exp_continue -continue_timer }");
316: cmdBuff[17] = new String(" timeout { abort }");
317: cmdBuff[18] = new String(
318: " eof { if {$exitcode != 0} { abort }}");
319: cmdBuff[19] = new String("}");
320: cmdBuff[20] = new String("set rc [wait]");
321: cmdBuff[21] = new String("set os_error [lindex $rc 2]");
322: cmdBuff[22] = new String("set status [lindex $rc 3]");
323: cmdBuff[23] = new String(
324: "if {$os_error != 0 || $status != 0} {");
325: cmdBuff[24] = new String(" exit 1");
326: cmdBuff[25] = new String("}");
327: cmdBuff[26] = new String("exit 0");
328: return cmdBuff;
329: }
330:
331: /**
332: * Creates a Splitted command String array used my exec(). This method uses
333: * the "expect" command to pass the password parameter to the command being
334: * run, thus allowing PostgreSQL to allow authentication. Also uses BINDIR if
335: * supplied. Note the "psql" command has different output needed by "expect"
336: * than other commands.
337: *
338: * @param command Command to execute
339: * @param info URL info to parse adn add parameters
340: * @param options Additional parameters
341: * @param login User login
342: * @param password User password
343: * @param isPsql Is the command being run a psql command?
344: * @return String array containing full command to run
345: */
346: protected String[] makeSplitCommandWithAuthentication(
347: String command, PostgreSQLUrlInfo info, String options,
348: String login, String password, boolean isPsql) {
349: // Build params for "spawn" command used by expect
350: StringBuffer cmdBuff = new StringBuffer("spawn ");
351:
352: if (binDir != null)
353: cmdBuff.append(binDir + File.separator);
354:
355: cmdBuff.append(command);
356: cmdBuff.append(" ");
357: cmdBuff.append(info.dbName); // gurkan
358: cmdBuff.append(" "); // gurkan
359: cmdBuff.append(info.getHostParametersString());
360: cmdBuff.append(" -U ");
361: cmdBuff.append(login);
362: cmdBuff.append(" ");
363: cmdBuff.append(options);
364: // cmdBuff.append(" "); //gurkan
365: // cmdBuff.append(info.dbName); //gurkan
366: cmdBuff.append("; ");
367: // "psql" command has different message for password input
368: if (isPsql)
369: cmdBuff.append("expect \"Password:\"; ");
370: else
371: cmdBuff.append("expect \"Password for user " + login
372: + ":\"; ");
373: cmdBuff.append("send \"");
374: cmdBuff.append(password);
375: cmdBuff.append("\"; ");
376: cmdBuff.append("send \"\\r\"; ");
377: cmdBuff.append("expect eof;");
378:
379: // may need to be bash not expect //not sure ; gurkan
380: String[] commands = { "expect", "-c", "bash", "-c",
381: cmdBuff.toString() }; // gurkan
382: // String[] commands = {"expect", "-c", cmdBuff.toString()};
383: return commands;
384: }
385:
386: /**
387: * Executes a native operating system command with a standard-input feed. The
388: * commands are supposed to be one of the Postgres-utilities psql - dropdb -
389: * createdb - pg_dump - pg_restore. Output of the command is captured and
390: * logged.
391: *
392: * @param stdinFeed Array of input strings
393: * @param commands Array of strings (command + args) to execute
394: * @return 0 if successful, any number otherwise
395: * @throws IOException
396: * @throws InterruptedException
397: */
398: protected int executeNativeCommand(String[] stdinFeed,
399: String[] commands) throws IOException, InterruptedException {
400: NativeCommandInputSource input = NativeCommandInputSource
401: .createArrayInputSource(stdinFeed);
402: return nativeCmdExec.executeNativeCommand(commands, input,
403: null, 0, getIgnoreStdErrOutput());
404: }
405:
406: /**
407: * Creates a expect command reading the expect-script from standard-input.
408: * Hours of experiments has shown that the only reliable way of
409: * programatically invoking expect with an expect-script, is via stdin or
410: * explicit script-file. The expect "-c command" option simply does not work
411: * the same way, and do cause a lot of problems.
412: *
413: * @return String array containing full command to run
414: */
415: protected String[] makeExpectCommandReadingStdin() {
416: String[] commands = { "expect", "-" };
417: // String[] commands = {"expect", "-d", "-"}; // Diagnostic output
418: // String[] commands = {"cat"}; // Echoes the expect-script instead of
419: // running it
420: return commands;
421: }
422:
423: /**
424: * Executes a native operating system command, which currently are one of: -
425: * psql - dropdb - createdb - pg_dump - pg_restore Output of these commands is
426: * captured and logged.
427: *
428: * @param command String of command to execute
429: * @return 0 if successful, any number otherwise
430: * @throws IOException
431: * @throws InterruptedException
432: */
433: protected int executeNativeCommand(String command)
434: throws IOException, InterruptedException {
435: return nativeCmdExec.executeNativeCommand(command, null, null,
436: 0, getIgnoreStdErrOutput());
437: }
438:
439: /**
440: * Executes a native operating system command, which currently are one of:
441: *
442: * <pre>
443: * - psql
444: * - dropdb
445: * - createdb
446: * - pg_dump
447: * - pg_restore
448: * </pre>
449: *
450: * Output of these commands is captured and logged.
451: *
452: * @param commands String array of command to execute
453: * @return 0 if successful, any number otherwise
454: * @throws IOException
455: * @throws InterruptedException
456: */
457: protected int executeNativeCommand(String[] commands)
458: throws IOException, InterruptedException {
459: return nativeCmdExec.executeNativeCommand(commands, null, null,
460: 0, getIgnoreStdErrOutput());
461: }
462:
463: /**
464: * Execute a native command with careful logging of output and errors.
465: *
466: * @param cmd Command to execute
467: * @param inputArray Array of input lines
468: * @param timeout Timeout or 0 for no timeout
469: * @return True if command is successful
470: */
471: protected boolean safelyExecNativeCommand(String cmd,
472: String[] inputArray, int timeout) {
473: NativeCommandInputSource input = NativeCommandInputSource
474: .createArrayInputSource(inputArray);
475: return nativeCmdExec.safelyExecNativeCommand(cmd, input, null,
476: timeout, getIgnoreStdErrOutput());
477: }
478:
479: /**
480: * Execute a native command with careful logging of output and errors.
481: *
482: * @param cmds Command array to execute
483: * @param inputArray Array of input lines
484: * @param timeout Timeout or 0 for no timeout
485: * @return True if command is successful
486: */
487: protected boolean safelyExecNativeCommand(String[] cmds,
488: String[] inputArray, int timeout) {
489: NativeCommandInputSource input = NativeCommandInputSource
490: .createArrayInputSource(inputArray);
491: return nativeCmdExec.safelyExecNativeCommand(cmds, input, null,
492: timeout, getIgnoreStdErrOutput());
493: }
494:
495: /**
496: * Prints contents of error output (stderr).
497: */
498: protected void printErrors() {
499: ArrayList errors = nativeCmdExec.getStderr();
500: Iterator it = errors.iterator();
501: while (it.hasNext()) {
502: String msg = (String) it.next();
503: logger.info(msg);
504: endUserLogger.error(msg);
505: }
506: }
507:
508: /**
509: * Prints contents of regular output (stdout).
510: */
511: protected void printOutput() {
512: ArrayList errors = nativeCmdExec.getStderr();
513: Iterator it = errors.iterator();
514: while (it.hasNext()) {
515: String msg = (String) it.next();
516: logger.info(msg);
517: endUserLogger.error(msg);
518: }
519: }
520:
521: /**
522: * Allow to parse PostgreSQL URL.
523: */
524: protected class PostgreSQLUrlInfo {
525: private boolean isLocal;
526: private String host;
527: private String port;
528: private String dbName;
529:
530: // Used to parse url
531: // private Pattern pattern =
532: // Pattern.compile("jdbc:postgresql:((//([a-zA-Z0-9_\\-.]+|\\[[a-fA-F0-9:]+])((:(\\d+))|))/|)([a-zA-Z][a-zA-Z0-9_]*)");
533: private Pattern pattern = Pattern
534: .compile(getJdbcUrlPrefix()
535: + "((//([a-zA-Z0-9_\\-.]+|\\[[a-fA-F0-9:]+])((:(\\d+))|))/|)([^\\s?]*).*$");
536: Matcher matcher;
537:
538: /**
539: * Creates a new <code>PostgreSQLUrlInfo</code> object, used to parse the
540: * postgresql jdbc options. If host and/or port aren't specified, will
541: * default to localhost:5432. Note that database name must be specified.
542: *
543: * @param url the Postgresql jdbc url to parse
544: */
545: public PostgreSQLUrlInfo(String url) {
546: matcher = pattern.matcher(url);
547:
548: if (matcher.matches()) {
549: if (matcher.group(3) != null)
550: host = matcher.group(3);
551: else
552: host = DEFAULT_POSTGRESQL_HOST;
553:
554: if (matcher.group(6) != null)
555: port = matcher.group(6);
556: else
557: port = String.valueOf(getDefaultPort());
558:
559: dbName = matcher.group(7);
560: }
561: }
562:
563: /**
564: * Gets the HostParameters of this postgresql jdbc url as a String that can
565: * be used to pass into cmd line/shell calls.
566: *
567: * @return a string that can be used to pass into a cmd line/shell call.
568: */
569: public String getHostParametersString() {
570: if (isLocal) {
571: return "";
572: } else {
573: return "-h " + host + " -p " + port;
574: }
575: }
576:
577: /**
578: * Gets the database name part of this postgresql jdbc url.
579: *
580: * @return the database name part of this postgresql jdbc url.
581: */
582: public String getDbName() {
583: return dbName;
584: }
585:
586: /**
587: * Gets the host part of this postgresql jdbc url.
588: *
589: * @return the host part of this postgresql jdbc url.
590: */
591: public String getHost() {
592: return host;
593: }
594:
595: /**
596: * Gets the port part of this postgresql jdbc url.
597: *
598: * @return the port part of this postgresql jdbc url.
599: */
600: public String getPort() {
601: return port;
602: }
603:
604: /**
605: * Checks whether this postgresql jdbc url refers to a local db or not, i.e.
606: * has no host specified, e.g. jdbc:postgresql:myDb.
607: *
608: * @return true if this postgresql jdbc url has no host specified, i.e.
609: * refers to a local db.
610: */
611: public boolean isLocal() {
612: return isLocal;
613: }
614:
615: }
616: }
|