001: /*
002: * Copyright (c) 1998 - 2005 Versant Corporation
003: * All rights reserved. This program and the accompanying materials
004: * are made available under the terms of the Eclipse Public License v1.0
005: * which accompanies this distribution, and is available at
006: * http://www.eclipse.org/legal/epl-v10.html
007: *
008: * Contributors:
009: * Versant Corporation - initial API and implementation
010: */
011: package com.versant.core.jdo.tools.ant;
012:
013: import com.versant.core.jdo.QueryDetails;
014: import com.versant.core.common.*;
015: import com.versant.core.common.config.ConfigInfo;
016: import com.versant.core.common.config.ConfigParser;
017: import com.versant.core.jdbc.JdbcConfig;
018: import com.versant.core.metadata.*;
019: import com.versant.core.common.NewObjectOID;
020: import com.versant.core.logging.LogEventStore;
021: import com.versant.core.server.*;
022: import com.versant.core.jdbc.*;
023: import com.versant.core.jdbc.query.JdbcCompiledQuery;
024: import com.versant.core.jdbc.metadata.JdbcLinkCollectionField;
025: import com.versant.core.jdbc.metadata.JdbcTable;
026: import com.versant.core.jdbc.metadata.JdbcClass;
027: import com.versant.core.jdbc.sql.SqlDriver;
028: import com.versant.core.util.CharBuf;
029:
030: import java.io.PrintStream;
031: import java.io.PrintWriter;
032: import java.sql.*;
033: import java.text.SimpleDateFormat;
034: import java.util.*;
035: import java.util.Date;
036:
037: import com.versant.core.common.BindingSupportImpl;
038: import com.versant.core.storagemanager.DummyApplicationContext;
039: import com.versant.core.storagemanager.StorageManagerFactoryBuilder;
040: import com.versant.core.logging.LogEventStore;
041:
042: /**
043: * Configurable utility bean to copy one Open Access JDBC database to a
044: * different URL.
045: */
046: public class CopyDatabaseBean {
047:
048: private Properties srcProps;
049: private String datastore;
050: private Properties destProps;
051: private String db;
052: private String url;
053: private String driver;
054: private String user;
055: private String password;
056: private String properties;
057: private boolean dropTables = true;
058: private boolean createTables = true;
059: private PrintStream out = System.out;
060: private PrintWriter outw;
061: private boolean stopFlag;
062: private ClassLoader loader = getClass().getClassLoader();
063: private int rowsPerTransaction = 1000;
064: private String logEvents = LogEventStore.LOG_EVENTS_ERRORS;
065: private boolean logEventsToSysOut = true;
066:
067: private MiniServer dest;
068: private MiniServer src;
069:
070: private static final SimpleDateFormat DATE_FORMAT = new SimpleDateFormat(
071: "dd MMM yyyy HH:mm:ss");
072:
073: private static final int DOTS_PER_LINE = 70;
074:
075: private static final DummyKeyGen DUMMY_KEY_GEN = new DummyKeyGen();
076:
077: public CopyDatabaseBean() {
078: }
079:
080: public Properties getSrcProps() {
081: return srcProps;
082: }
083:
084: public void setSrcProps(Properties srcProps) {
085: this .srcProps = srcProps;
086: }
087:
088: public String getDatastore() {
089: return datastore;
090: }
091:
092: public void setDatastore(String datastore) {
093: this .datastore = datastore;
094: }
095:
096: public Properties getDestProps() {
097: return destProps;
098: }
099:
100: public void setDestProps(Properties destProps) {
101: this .destProps = destProps;
102: }
103:
104: public String getDb() {
105: return db;
106: }
107:
108: public void setDb(String db) {
109: this .db = db;
110: }
111:
112: public String getUrl() {
113: return url;
114: }
115:
116: public void setUrl(String url) {
117: this .url = url;
118: }
119:
120: public String getDriver() {
121: return driver;
122: }
123:
124: public void setDriver(String driver) {
125: this .driver = driver;
126: }
127:
128: public String getUser() {
129: return user;
130: }
131:
132: public void setUser(String user) {
133: this .user = user;
134: }
135:
136: public String getPassword() {
137: return password;
138: }
139:
140: public void setPassword(String password) {
141: this .password = password;
142: }
143:
144: public String getProperties() {
145: return properties;
146: }
147:
148: public void setProperties(String properties) {
149: this .properties = properties;
150: }
151:
152: public boolean isCreateTables() {
153: return createTables;
154: }
155:
156: public void setCreateTables(boolean createTables) {
157: this .createTables = createTables;
158: }
159:
160: public boolean isDropTables() {
161: return dropTables;
162: }
163:
164: public void setDropTables(boolean dropTables) {
165: this .dropTables = dropTables;
166: }
167:
168: public PrintStream getOut() {
169: return out;
170: }
171:
172: public void setOut(PrintStream out) {
173: this .out = out;
174: }
175:
176: public boolean isStopFlag() {
177: return stopFlag;
178: }
179:
180: public void setStopFlag(boolean stopFlag) {
181: this .stopFlag = stopFlag;
182: }
183:
184: public String getLogEvents() {
185: return logEvents;
186: }
187:
188: public void setLogEvents(String logEvents) {
189: this .logEvents = logEvents;
190: }
191:
192: public boolean isLogEventsToSysOut() {
193: return logEventsToSysOut;
194: }
195:
196: public void setLogEventsToSysOut(boolean logEventsToSysOut) {
197: this .logEventsToSysOut = logEventsToSysOut;
198: }
199:
200: public ClassLoader getLoader() {
201: return loader;
202: }
203:
204: public void setLoader(ClassLoader loader) {
205: this .loader = loader;
206: }
207:
208: public int getRowsPerTransaction() {
209: return rowsPerTransaction;
210: }
211:
212: public void setRowsPerTransaction(int rowsPerTransaction) {
213: this .rowsPerTransaction = rowsPerTransaction;
214: }
215:
216: private void checkStopFlag() {
217: if (stopFlag)
218: throw new StoppedException();
219: }
220:
221: /**
222: * Check properties and get ready to start.
223: */
224: private void prepare() {
225: if (srcProps == null) {
226: throw BindingSupportImpl.getInstance().illegalArgument(
227: "srcProps property not set");
228: }
229:
230: if (destProps == null)
231: destProps = (Properties) srcProps.clone();
232:
233: if (db != null)
234: destProps.setProperty(ConfigParser.STORE_DB, db);
235: if (driver != null) {
236: destProps.setProperty(ConfigParser.STD_CON_DRIVER_NAME,
237: driver);
238: }
239: if (url != null) {
240: destProps.setProperty(ConfigParser.STD_CON_URL, url);
241: }
242: if (user != null) {
243: destProps.setProperty(ConfigParser.STD_CON_USER_NAME, user);
244: }
245: if (password != null) {
246: destProps.setProperty(ConfigParser.STD_CON_PASSWORD,
247: password);
248: }
249: if (properties != null) {
250: properties = properties.trim().replace('\n', ';');
251: destProps.setProperty(ConfigParser.STORE_PROPERTIES,
252: properties);
253: }
254:
255: outw = new PrintWriter(out, true);
256: }
257:
258: /**
259: * Copy datastore from src to dest.
260: */
261: public void copyDatabase() throws Exception {
262: prepare();
263: long start = System.currentTimeMillis();
264: out.println("Started copy at "
265: + DATE_FORMAT.format(new Date(start)) + "\n");
266: src = dest = null;
267: try {
268: dest = new MiniServer(destProps);
269: checkStopFlag();
270: src = new MiniServer(srcProps);
271: checkStopFlag();
272:
273: if (dropTables)
274: dest.dropAllTables();
275: if (createTables)
276: dest.createAllTables();
277:
278: ClassMetaData[] srcClasses = src.getStoreClasses();
279: ClassMetaData[] destClasses = dest.getStoreClasses();
280:
281: out
282: .println("=== Copying main table for each class (one dot = "
283: + rowsPerTransaction + " rows) ===\n");
284: long mainStart = System.currentTimeMillis();
285: int mainTotRows = 0;
286: int numClasses = srcClasses.length;
287: for (int i = 0; i < numClasses; i++) {
288: mainTotRows += copyClassMainTable(srcClasses[i],
289: destClasses[i]);
290: checkStopFlag();
291: }
292: int mainTime = (int) (System.currentTimeMillis() - mainStart);
293: out.println();
294:
295: out
296: .println("=== Copying link tables for each class (one dot = "
297: + rowsPerTransaction + " rows) ===\n");
298: long linkStart = System.currentTimeMillis();
299: int linkTotRows = 0;
300: for (int i = 0; i < numClasses; i++) {
301: ClassMetaData sc = srcClasses[i];
302: for (int j = 0; j < sc.fields.length; j++) {
303: FieldMetaData sfmd = sc.fields[j];
304: if (sfmd.storeField instanceof JdbcLinkCollectionField) {
305: JdbcLinkCollectionField srcField = (JdbcLinkCollectionField) sfmd.storeField;
306: if (!srcField.readOnly) {
307: linkTotRows += copyLinkTable(
308: srcField,
309: (JdbcLinkCollectionField) destClasses[i].fields[j].storeField);
310: checkStopFlag();
311: }
312: }
313: }
314: }
315: int linkTime = (int) (System.currentTimeMillis() - linkStart);
316: out.println();
317:
318: if (createTables) {
319: dest.createAllIndexes();
320: dest.createAllConstraints();
321: }
322:
323: out.println("Copied " + mainTotRows + " main table rows "
324: + "in " + fmtSec(mainTime) + " secs ("
325: + perSec(mainTotRows, mainTime) + " per second)");
326:
327: out.println("Copied " + linkTotRows + " link table rows "
328: + "in " + fmtSec(linkTime) + " secs ("
329: + perSec(linkTotRows, linkTime) + " per second)");
330:
331: } catch (StoppedException x) {
332: out.println("\n*** Copy aborted ***");
333: } finally {
334: if (src != null)
335: src.shutdown();
336: if (dest != null)
337: dest.shutdown();
338: src = dest = null;
339: out.println("\nFinished at "
340: + DATE_FORMAT.format(new Date()));
341: }
342: }
343:
344: private String fmtSec(int ms) {
345: StringBuffer s = new StringBuffer();
346: s.append(ms / 1000);
347: s.append('.');
348: ms = ms % 1000;
349: if (ms < 10)
350: s.append("00");
351: if (ms < 100)
352: s.append('0');
353: s.append(ms);
354: return s.toString();
355: }
356:
357: private String perSec(int rows, int ms) {
358: float f = Math.round(rows * 10000.0f / ms) / 10;
359: return String.valueOf(f);
360: }
361:
362: private int copyClassMainTable(ClassMetaData srcClass,
363: ClassMetaData destClass) throws Exception {
364: out.println(((JdbcClass) srcClass.storeClass).table.name
365: + " - " + srcClass.qname);
366:
367: JdbcKeyGenerator keygen = ((JdbcClass) destClass.storeClass).jdbcKeyGenerator;
368: boolean autoinc = keygen != null
369: && keygen.isPostInsertGenerator();
370: ((JdbcClass) destClass.storeClass).jdbcKeyGenerator = DUMMY_KEY_GEN;
371:
372: JdbcQueryResult rs = null;
373: try {
374: Connection destCon = dest.pool.getConnection(false, false);
375: if (autoinc) {
376: dest.sqlDriver.enableIdentityInsert(destCon,
377: ((JdbcClass) destClass.storeClass).table.name,
378: true);
379: }
380:
381: PersistGraph pg = new PersistGraph(dest.getJmd(),
382: rowsPerTransaction);
383:
384: QueryDetails qp = new QueryDetails();
385: qp.setCandidateClass(srcClass.cls);
386: qp.setSubClasses(false);
387: qp.setFetchGroupIndex(srcClass
388: .getFetchGroup(FetchGroup.ALL_COLS_NAME).index);
389: qp.setResultBatchSize(rowsPerTransaction);
390: JdbcCompiledQuery cq = (JdbcCompiledQuery) src.getSm()
391: .compileQuery(qp);
392: cq.setCacheable(false);
393:
394: int tot = 0;
395: int dots = 0;
396: rs = (JdbcQueryResult) src.getSm().executeQuery(
397: DummyApplicationContext.INSTANCE, null, cq, null)
398: .getRunningQuery();
399: StatesReturned container = new StatesReturned(
400: DummyApplicationContext.INSTANCE);
401: for (;;) {
402: int c = 0;
403: boolean more = false;
404: for (; c < rowsPerTransaction && (more = rs.next(0)); c++) {
405: JdbcGenericOID oid = (JdbcGenericOID) rs
406: .getResultOID();
407: JdbcGenericState state = (JdbcGenericState) rs
408: .getResultState(false, container);
409: oid.setCmd(destClass);
410: state.setClassMetaData(destClass);
411: NewObjectOID noid = destClass.createNewObjectOID();
412: noid.realOID = oid;
413: pg.add(noid, null, state);
414: container.clear();
415: }
416: if (c > 0) {
417: dest.getSm().begin(false);
418: dest.getSm().persistPass1(pg);
419: dest.getSm().commit();
420: tot += c;
421: if (++dots == DOTS_PER_LINE) {
422: out.println("." + tot);
423: dots = 0;
424: } else {
425: out.print('.');
426: }
427: pg.clear();
428: }
429: if (!more)
430: break;
431: checkStopFlag();
432: }
433: if (dots > 0)
434: out.println(tot);
435:
436: if (autoinc) {
437: dest.sqlDriver.enableIdentityInsert(destCon,
438: ((JdbcClass) destClass.storeClass).table.name,
439: false);
440: }
441:
442: return tot;
443: } finally {
444: if (rs != null) {
445: try {
446: rs.close();
447: } catch (Exception e) {
448: // ignore
449: }
450: }
451: if (src.getSm().isActive()) {
452: src.getSm().rollback();
453: }
454: if (dest.getSm().isActive()) {
455: dest.getSm().rollback();
456: }
457: }
458: }
459:
460: private void close(ResultSet rs) {
461: if (rs == null)
462: return;
463: try {
464: rs.close();
465: } catch (SQLException e) {
466: // ignore
467: }
468: }
469:
470: private void close(Statement stat) {
471: if (stat == null)
472: return;
473: try {
474: stat.close();
475: } catch (SQLException e) {
476: // ignore
477: }
478: }
479:
480: private int copyLinkTable(JdbcLinkCollectionField srcField,
481: JdbcLinkCollectionField destField) throws Exception {
482: out.println(srcField.linkTable.name + " - "
483: + srcField.fmd.getQName());
484:
485: Connection srcCon = null;
486: Connection destCon = null;
487: ResultSet rs = null;
488: PreparedStatement ps = null;
489: Statement stat = null;
490: try {
491: srcCon = src.getAutoCommitCon();
492: destCon = dest.getSm().con();
493:
494: String insSql = srcField
495: .getInsertLinkTableRowSql(new CharBuf());
496: boolean batch = dest.getSm().isUseBatchInsert();
497:
498: stat = srcCon.createStatement();
499: rs = stat.executeQuery(srcField.getFetchAllRowsSql(src
500: .getSm()));
501: if (src.getSm().getSqlDriver().isFetchSizeSupported()) {
502: rs.setFetchSize(rowsPerTransaction);
503: }
504:
505: JdbcLinkCollectionField.LinkRow row = new JdbcLinkCollectionField.LinkRow();
506:
507: int tot = 0;
508: int dots = 0;
509: for (;;) {
510: ps = destCon.prepareStatement(insSql);
511: int c = 0;
512: boolean more = false;
513: for (; c < rowsPerTransaction && (more = rs.next()); c++) {
514: srcField.readRow(rs, row);
515: destField.writeRow(ps, row);
516: if (batch) {
517: ps.addBatch();
518: } else {
519: ps.execute();
520: }
521: }
522: if (c > 0) {
523: if (batch)
524: ps.executeBatch();
525: destCon.commit();
526: ps.close();
527: tot += c;
528: if (++dots == DOTS_PER_LINE) {
529: out.println("." + tot);
530: dots = 0;
531: } else {
532: out.print('.');
533: }
534: }
535: if (!more)
536: break;
537: checkStopFlag();
538: }
539: if (dots > 0)
540: out.println(tot);
541: return tot;
542: } finally {
543: close(rs);
544: close(stat);
545: close(ps);
546: src.closeAutoCommitCon(srcCon);
547: close(destCon);
548: }
549: }
550:
551: private void close(Connection con) {
552: if (con == null)
553: return;
554: try {
555: con.rollback();
556: } catch (SQLException e) {
557: // ignore
558: }
559: try {
560: con.close();
561: } catch (SQLException e) {
562: // ignore
563: }
564: }
565:
566: /**
567: * Stripped down JDO Genie server instance just so we can get at the
568: * database.
569: */
570: private class MiniServer {
571:
572: private final JdbcStorageManagerFactory factory;
573: private final JdbcStorageManager sm;
574: private final JdbcConnectionSource pool;
575: private final ModelMetaData jmd;
576: private final ConfigInfo config;
577: private final SqlDriver sqlDriver;
578:
579: public MiniServer(Properties props) throws Exception {
580:
581: // fudge and parse the config
582: props.put(ConfigParser.STORE_MAX_ACTIVE, "2");
583: props.put(ConfigParser.STORE_MAX_IDLE, "2");
584: props.put(ConfigParser.STORE_TEST_ON_ALLOC, "false");
585: props.put(ConfigParser.STORE_TEST_ON_RELEASE, "false");
586: props.put(ConfigParser.STORE_TEST_ON_EXCEPTION, "false");
587: props.put(ConfigParser.STORE_RETRY_COUNT, "-1");
588: ConfigParser p = new ConfigParser();
589: config = p.parse(props);
590: config.validate();
591: config.hyperdrive = false;
592:
593: // setup event log
594: LogEventStore pes = new LogEventStore();
595: pes.setLogEvents(logEvents);
596: pes.setLogEventsToSysOut(logEventsToSysOut);
597:
598: StorageManagerFactoryBuilder b = new StorageManagerFactoryBuilder();
599: b.setConfig(config);
600: b.setLoader(loader);
601: b.setFullInit(false);
602: factory = (JdbcStorageManagerFactory) b
603: .createStorageManagerFactory();
604: pool = factory.getConnectionSource();
605: jmd = factory.getModelMetaData();
606: sqlDriver = factory.getSqlDriver();
607: sm = (JdbcStorageManager) factory.getStorageManager();
608: }
609:
610: public JdbcStorageManager getSm() {
611: return sm;
612: }
613:
614: public ModelMetaData getJmd() {
615: return jmd;
616: }
617:
618: public void shutdown() {
619: factory.returnStorageManager(sm);
620: factory.destroy();
621: }
622:
623: public Connection getAutoCommitCon() throws Exception {
624: return pool.getConnection(false, true);
625: }
626:
627: public void closeAutoCommitCon(Connection con)
628: throws SQLException {
629: if (con == null)
630: return;
631: pool.returnConnection(con);
632: }
633:
634: /**
635: * Drop all the tables in store with names matching tables in the
636: * schema.
637: */
638: public void dropAllTables() throws Exception {
639: out.println("=== Dropping tables in: " + pool.getURL()
640: + " ===");
641: Connection con = null;
642: try {
643: con = getAutoCommitCon();
644: HashMap dbTableNames = sm.getDatabaseTableNames(con);
645: checkStopFlag();
646: ArrayList a = sm.getJdbcMetaData().getTables();
647: for (int i = 0; i < a.size(); i++) {
648: JdbcTable t = (JdbcTable) a.get(i);
649: String name = (String) dbTableNames.get(t.name
650: .toLowerCase());
651: if (name != null) {
652: out.println(" Dropping " + name);
653: sqlDriver.dropTable(con, name);
654: checkStopFlag();
655: }
656: }
657: } finally {
658: closeAutoCommitCon(con);
659: }
660: out.println();
661: }
662:
663: /**
664: * Create all tables but do not do indexes or constraints.
665: */
666: public void createAllTables() throws Exception {
667: out.println("=== Creating tables in: " + pool.getURL()
668: + " ===");
669: Connection con = null;
670: Statement stat = null;
671: try {
672: con = getAutoCommitCon();
673: stat = con.createStatement();
674: ArrayList tables = sm.getJdbcMetaData().getTables();
675: int n = tables.size();
676: for (int i = 0; i < n; i++) {
677: JdbcTable t = (JdbcTable) tables.get(i);
678: sqlDriver.generateCreateTable(t, stat, outw, false);
679: checkStopFlag();
680: }
681: } finally {
682: close(stat);
683: closeAutoCommitCon(con);
684: }
685: out.println();
686: }
687:
688: /**
689: * Create all indexes.
690: */
691: public void createAllIndexes() throws Exception {
692: out.println("=== Creating indexes in: " + pool.getURL()
693: + " ===");
694: Connection con = null;
695: Statement stat = null;
696: try {
697: con = getAutoCommitCon();
698: stat = con.createStatement();
699: ArrayList tables = sm.getJdbcMetaData().getTables();
700: int n = tables.size();
701: for (int i = 0; i < n; i++) {
702: JdbcTable t = (JdbcTable) tables.get(i);
703: if (t.indexes != null && t.indexes.length == 0)
704: continue;
705: sqlDriver.generateCreateIndexes(t, stat, outw,
706: false);
707: checkStopFlag();
708: }
709: } finally {
710: close(stat);
711: closeAutoCommitCon(con);
712: }
713: out.println();
714: }
715:
716: /**
717: * Create all constraints.
718: */
719: public void createAllConstraints() throws Exception {
720: out.println("=== Adding constraints in: " + pool.getURL()
721: + " ===");
722: Connection con = null;
723: Statement stat = null;
724: try {
725: con = getAutoCommitCon();
726: stat = con.createStatement();
727: ArrayList tables = sm.getJdbcMetaData().getTables();
728: int n = tables.size();
729: for (int i = 0; i < n; i++) {
730: JdbcTable t = (JdbcTable) tables.get(i);
731: if (t.constraints != null
732: && t.constraints.length == 0)
733: continue;
734: sqlDriver.generateConstraints(t, stat, outw, false);
735: checkStopFlag();
736: }
737: } finally {
738: close(stat);
739: closeAutoCommitCon(con);
740: }
741: out.println();
742: }
743:
744: private void close(Statement stat) {
745: if (stat == null)
746: return;
747: try {
748: stat.close();
749: } catch (SQLException e) {
750: // ignore
751: }
752: }
753:
754: /**
755: * Get all the classes in our store sorted in the correct order so
756: * we can do inserts without tripping constraints.
757: */
758: public ClassMetaData[] getStoreClasses() {
759: ArrayList a = new ArrayList();
760: for (int i = 0; i < jmd.classes.length; i++) {
761: ClassMetaData cmd = jmd.classes[i];
762: if (cmd.storeClass != null) {
763: a.add(cmd);
764: }
765: }
766: Collections.sort(a, new Comparator() {
767: public int compare(Object o1, Object o2) {
768: ClassMetaData a = (ClassMetaData) o1;
769: ClassMetaData b = (ClassMetaData) o2;
770: int diff = b.referenceGraphIndex
771: - a.referenceGraphIndex;
772: if (diff != 0)
773: return diff;
774: return b.index - a.index;
775: }
776: });
777: ClassMetaData[] x = new ClassMetaData[a.size()];
778: a.toArray(x);
779: return x;
780: }
781:
782: }
783:
784: /**
785: * Thrown by tasks when they detect that the stop flag is set.
786: */
787: private static class StoppedException extends RuntimeException {
788:
789: }
790:
791: /**
792: * Dummy key generator set on all destination classes so that new pks
793: * are not generated even for autoincrement classes.
794: */
795: private static class DummyKeyGen implements JdbcKeyGenerator {
796:
797: public void addKeyGenTables(HashSet set, JdbcMetaDataBuilder mdb) {
798: }
799:
800: public boolean isPostInsertGenerator() {
801: return false;
802: }
803:
804: public void init(String className, JdbcTable classTable,
805: Connection con) throws SQLException {
806: }
807:
808: public boolean isRequiresOwnConnection() {
809: return false;
810: }
811:
812: public void generatePrimaryKeyPre(String className,
813: JdbcTable classTable, int newObjectCount,
814: Object[] data, Connection con) throws SQLException {
815: }
816:
817: public String getPostInsertSQLSuffix(JdbcTable classTable) {
818: return null;
819: }
820:
821: public void generatePrimaryKeyPost(String className,
822: JdbcTable classTable, Object[] data, Connection con,
823: Statement stat) throws SQLException {
824: }
825: }
826: }
|