0001: /*
0002: * Copyright 2001 Sun Microsystems, Inc. All rights reserved.
0003: * PROPRIETARY/CONFIDENTIAL. Use of this product is subject to license terms.
0004: */
0005:
0006: package com.sun.portal.search.rdmgr;
0007:
0008: import com.sun.portal.search.util.*;
0009: import com.sun.portal.search.rdm.*;
0010: import com.sun.portal.search.soif.*;
0011: import com.sun.portal.search.db.*;
0012: import com.sun.portal.log.common.PortalLogger;
0013:
0014: import com.sun.kt.search.*;
0015:
0016: import java.util.*;
0017: import java.util.logging.Logger;
0018: import java.util.logging.Level;
0019: import java.util.logging.LogRecord;
0020: import java.io.*;
0021: import java.text.DateFormat;
0022:
0023: /**
0024: * Resource Description Manager - Used to
0025: * add/delete/update RD's. Runs as a cli.
0026: */
0027: public class RDMgr {
0028:
0029: public static final String COMMANDNAME = "rdmgr";
0030: public static final String COMMANDVERSION = "1.0";
0031:
0032: // upper limit for index batches XXX hardcode
0033: static final int MAX_RD_BATCH = 65520;
0034:
0035: boolean loggingStarted = false;
0036: String dbClass = "com.sun.portal.search.db.IndexedSOIFDb"; // XXX (rootdb/config)
0037:
0038: // admin commands - override all others
0039: boolean do_optimize = false;
0040: boolean do_purge = false;
0041: boolean do_reindex = false;
0042: boolean do_expire = false;
0043: boolean do_recover = false;
0044: boolean do_repartition = false;
0045: boolean do_drop = false;
0046: boolean do_dump = false;
0047: boolean do_select = false;
0048: boolean do_count = false;
0049:
0050: // debug/error output
0051: int dlevel = -1;
0052: String dfile = null;
0053:
0054: // input file
0055: String ifile;
0056:
0057: // Search Config
0058: SearchConfig searchcf = null;
0059:
0060: // for testing only - not documented
0061: static boolean loop = false;
0062: static boolean federated_search = false;
0063:
0064: /**
0065: * Run the rdmgr cli, does not throw an exception
0066: * @return status, 0 => success, !=0 => failure
0067: */
0068: public int run(String argv[]) {
0069:
0070: int status = 0;
0071:
0072: /* XXX Dummy access token and transaction */
0073: SToken st = new SToken(null, false, false, false);
0074:
0075: /* create a admin cli RDMSecurityManager */
0076: RDMSecurityManager secMgr = new RDMSecurityManager(true);
0077: RDMSecurityManager.setInstance(secMgr);
0078: RDMTransaction t = null;
0079:
0080: // default settings for command-line arguments
0081: boolean skip_alias_conf = false;
0082: boolean delete_on_exit = false;
0083: String oper = null;
0084: String type = null;
0085: String view = null;
0086: String csid = null;
0087: Set view_attr = null;
0088: String query = null;
0089:
0090: // Set defaults - overridden by config and header params
0091: String serverroot = null;
0092:
0093: String searchConfFile;
0094: RDMgrConfig rdmgrcf;
0095: RDSubmitRequest req;
0096:
0097: try {
0098:
0099: searchConfFile = SearchConfig.CONFDIR + File.separator
0100: + SearchConfig.SEARCH_CONF; // default location
0101:
0102: rdmgrcf = new RDMgrConfig();
0103: req = new RDSubmitRequest();
0104:
0105: // Process command line options
0106: String flags = "umgdnACSTDVNPGOXIREBULqvh?b:l:c:p:a:x:s:y:r:Q:j:i:o:YF";
0107: com.sun.portal.search.util.Getopt gopt = new com.sun.portal.search.util.Getopt(
0108: argv, flags);
0109: int c;
0110: while ((c = gopt.getopt()) != -1) {
0111: switch (c) {
0112: case 'u': // Update RDs
0113: oper = RDM.SUBMIT_INSERT;
0114: break;
0115: case 'm': // Modify RDs
0116: oper = RDM.SUBMIT_UPDATE;
0117: break;
0118: case 'g': // Retrieve RDs
0119: oper = RDM.SUBMIT_RETRIEVE;
0120: if (type == null)
0121: type = RDM.SUBMIT_MERGED; // merged data in retrieve mode
0122: break;
0123: case 'd': // Delete RDs
0124: if (type == null)
0125: type = RDM.SUBMIT_MERGED; // delete both N and P data
0126: oper = RDM.SUBMIT_DELETE;
0127: break;
0128: case 'n': // Count RDs
0129: if (type == null)
0130: type = RDM.SUBMIT_MERGED; // count both N and P data
0131: do_count = true;
0132: break;
0133: case 'Q': // Query RDs
0134: if (type == null)
0135: type = RDM.SUBMIT_MERGED; // merged data in query mode
0136: if (oper == null)
0137: oper = RDM.SUBMIT_RETRIEVE;
0138: if (req.maxhits == RDSubmitRequest.MAX_HITS)
0139: req.maxhits = RDSubmitRequest.MAX_QUERY_HITS;
0140: if (gopt.optArg != null)
0141: query = gopt.optArg;
0142: else
0143: return usage(1);
0144: break;
0145: case 'A':
0146: skip_alias_conf = true; // No alias mapping
0147: break;
0148: case 'C': // No create
0149: req.do_create = false;
0150: break;
0151: case 'S': // skip schema check
0152: req.schema_check = false;
0153: break;
0154: case 'T': // Taxonomy
0155: req.is_taxonomy = true;
0156: req.which_pass = RDSubmitRequest.RDM_SAGE; // XXX remove when tax db is ready
0157: break;
0158: case 'D': // Database only
0159: req.which_pass = RDSubmitRequest.RDM_CSDB;
0160: break;
0161: case 'V': // Index only
0162: req.which_pass = RDSubmitRequest.RDM_SAGE;
0163: break;
0164: case 'N':
0165: type = RDM.SUBMIT_NONPERSISTENT; // non-persistent data
0166: break;
0167: case 'P':
0168: type = RDM.SUBMIT_PERSISTENT; // persistent data
0169: break;
0170: case 'O':
0171: do_optimize = true; // optimize db
0172: break;
0173: case 'X':
0174: do_purge = true; // purge db
0175: break;
0176: case 'I':
0177: do_reindex = true; // reindex db
0178: if (oper == null)
0179: oper = RDM.SUBMIT_INSERT;
0180: //which_pass = RDMgrConfig.RDM_SAGE;
0181: break;
0182: case 'R':
0183: do_recover = true; // recover db
0184: break;
0185: case 'G':
0186: do_repartition = true; // repartition db
0187: break;
0188: case 'B':
0189: do_drop = true; // drop db
0190: break;
0191: case 'L':
0192: do_select = true; // dump db in select mode
0193: // fall through
0194: case 'U':
0195: if (type == null)
0196: type = RDM.SUBMIT_MERGED; // merged data in dump mode
0197: do_dump = true; // dump db
0198: break;
0199: case 'E':
0200: do_expire = true; // expire db
0201: if (type == null)
0202: type = RDM.SUBMIT_MERGED; // merged data in query mode
0203: if (oper == null)
0204: oper = RDM.SUBMIT_DELETE; // delete expired rds
0205: break;
0206: case 'q': // delete input file on exit
0207: delete_on_exit = true;
0208: break;
0209: case 'v': // version
0210: try {
0211: // Try Portal Server version info.
0212: System.out.println(getVersionInfo());
0213: } catch (Exception e) {
0214: // XXX Assume Compass Server.
0215: System.out.println("rdmgr version 6.0");
0216: }
0217: return 0;
0218: case 'h': // help
0219: return usage(0);
0220: case '?': // illegal/unknown option char or missing arg
0221: return usage(1);
0222: case 'b': // max rd batch
0223: if (gopt.optArg != null) {
0224: int i = Integer.parseInt(gopt.optArg);
0225: if (i > 0)
0226: req.max_rd_batch = i;
0227: } else
0228: return usage(1);
0229: break;
0230: case 'l': // loglevel
0231: if (gopt.optArg != null)
0232: dlevel = Integer.parseInt(gopt.optArg);
0233: else
0234: return usage(1);
0235: break;
0236: case 'c': // Search config
0237: if (gopt.optArg != null)
0238: searchConfFile = gopt.optArg;
0239: else
0240: return usage(1);
0241: break;
0242: case 'p': // log progress
0243: if (gopt.optArg != null) {
0244: boolean auto_flush = true;
0245: if (gopt.optArg.equals("stdout"))
0246: req.progress.progressWriter = new PrintWriter(
0247: System.out, auto_flush);
0248: else if (gopt.optArg.equals("stderr"))
0249: req.progress.progressWriter = new PrintWriter(
0250: System.err, auto_flush);
0251: else
0252: try {
0253: req.progress.progressWriter = new PrintWriter(
0254: new BufferedWriter(
0255: new FileWriter(
0256: gopt.optArg)),
0257: auto_flush);
0258: req.progress.closeProgressWriter = true;
0259: } catch (Exception e) {
0260: return usage(1);
0261: }
0262: } else
0263: return usage(1);
0264: break;
0265: case 'a': // attibute view list
0266: if (gopt.optArg != null)
0267: view = gopt.optArg;
0268: else
0269: return usage(1);
0270: break;
0271: case 'x': // CSID
0272: if (gopt.optArg != null)
0273: csid = gopt.optArg;
0274: else
0275: return usage(1);
0276: break;
0277: case 's': // overrides search.conf
0278: if (gopt.optArg != null)
0279: rdmgrcf.schemafn = gopt.optArg;
0280: else
0281: return usage(1);
0282: break;
0283: case 'y': // database name
0284: if (gopt.optArg != null)
0285: req.dbname = gopt.optArg;
0286: else
0287: return usage(1);
0288: break;
0289: case 'r': // report interval
0290: if (gopt.optArg != null)
0291: req.log_interval = Integer
0292: .parseInt(gopt.optArg);
0293: else
0294: return usage(1);
0295: break;
0296: case 'j': // max hits in query mode
0297: if (gopt.optArg != null)
0298: req.maxhits = Integer.parseInt(gopt.optArg);
0299: else
0300: return usage(1);
0301: break;
0302: case 'i': // input charset
0303: if (gopt.optArg != null)
0304: req.icharset = gopt.optArg;
0305: else
0306: return usage(1);
0307: break;
0308: case 'o': // output charset
0309: if (gopt.optArg != null)
0310: req.ocharset = gopt.optArg;
0311: else
0312: return usage(1);
0313: break;
0314: // testing cases
0315: case 'Y': // Loop for testing only, not documented
0316: loop = true;
0317: break;
0318: case 'F': // Allow federated search dbs (XXX should be default, but not ready)
0319: federated_search = true;
0320: break;
0321: default:
0322: System.err.println(RDMgrUtil.getLocalizedString(
0323: "invalidOption", new Object[] { ""
0324: + gopt.getOptLetter() }));
0325: return usage(1);
0326: }
0327: }
0328:
0329: // do we have a csid ?
0330: if ((argv.length - gopt.optInd) > 0)
0331: if (argv[gopt.optInd].startsWith("x-catalog"))
0332: csid = argv[gopt.optInd++];
0333:
0334: // do we have a file name?
0335: if (argv.length - gopt.optInd > 0)
0336: ifile = argv[gopt.optInd++];
0337:
0338: if (argv.length - gopt.optInd > 0)
0339: return usage(1);
0340:
0341: // Read config file
0342: try {
0343: SearchConfig.init(searchConfFile);
0344: searchcf = SearchConfig.getSearchConfig();
0345: } catch (Exception e) {
0346: System.err.println(RDMgrUtil.getLocalizedString(
0347: "configFailure", new Object[] { searchConfFile,
0348: e.getMessage() }));
0349: throw new RDMgrException();
0350: }
0351:
0352: // Start the logging
0353: try {
0354: SearchLogger.init(SearchConfig.RDMGR_LOGGER, searchcf
0355: .getValue(SearchConfig.SERVER_ROOT));
0356: } catch (Exception e) {
0357: System.err.println(RDMgrUtil.getLocalizedString(
0358: "loggingInitFailure", new Object[] { e
0359: .getMessage() }));
0360: throw new RDMgrException();
0361: }
0362:
0363: loggingStarted = true;
0364:
0365: // Nova logging
0366: String search_engine_log = searchcf
0367: .getValue(SearchConfig.SENGINE_LOGFN);
0368: if (search_engine_log != null) {
0369: try {
0370: // Search Engine logging. XXX no rotation
0371: Log log = Log.getLog();
0372: OutputStream os = new FileOutputStream(
0373: search_engine_log, true);
0374: log.setStream(os);
0375: log.setLevel(dlevel);
0376: if (!Build.debug)
0377: log.setLevel(Log.DEBUG, 0);
0378: } catch (Exception e) {
0379: if (SearchLogger.getLogger().isLoggable(
0380: Level.WARNING)) {
0381: LogRecord logRecord = new LogRecord(
0382: Level.WARNING, "PSSH_CSPSRDMR0001");
0383: logRecord
0384: .setParameters(new Object[] { search_engine_log });
0385: logRecord.setThrown(e);
0386: logRecord.setLoggerName(SearchLogger
0387: .getLogger().getName());
0388: SearchLogger.getLogger().log(logRecord);
0389: }
0390: throw new RDMgrException();
0391: }
0392: }
0393:
0394: if (csid == null)
0395: csid = searchcf.getDefault();
0396: searchcf.setDefault(csid);
0397: rdmgrcf.serverroot = serverroot;
0398:
0399: SearchLogger.getLogger().log(Level.INFO,
0400: "PSSH_CSPSRDMR0002");
0401:
0402: req.progress.init();
0403: if (oper != null)
0404: req.oper = oper;
0405: if (type != null)
0406: req.type = type;
0407: if (view != null)
0408: req.view = view;
0409: if (csid != null)
0410: req.csid = csid;
0411: if (view_attr != null)
0412: req.view_attr = view_attr;
0413: if (query != null)
0414: req.query = query;
0415:
0416: loadConfig(rdmgrcf, req);
0417: st.setDatabaseName(req.dbname); // function will set default dbname if not given from cli args
0418: req.dbname = st.getPhysicalDBName();
0419: int rds_processed = 0;
0420:
0421: if (do_recover) {
0422: // Don't start a db session
0423: req.progress.reportProgress("Recovering_database");
0424: SearchLogger.getLogger().log(Level.INFO,
0425: "PSSH_CSPSRDMR0003");
0426: boolean fatal = false;
0427: (new PartitionedDb()).recover(rdmgrcf.dbdir, fatal);
0428: SearchLogger.getLogger().log(Level.INFO,
0429: "PSSH_CSPSRDMR0004");
0430: rds_processed = -1;
0431: } else {
0432: // log - Starting db session
0433: SearchLogger.getLogger().log(Level.FINE,
0434: "PSSH_CSPSRDMR0005");
0435: dbopen(rdmgrcf, req, st, t);
0436: try {
0437: rds_processed = doRequest(rdmgrcf, req, st, t);
0438: } finally {
0439: dbclose(req, st);
0440: }
0441:
0442: if (rds_processed >= 0) {
0443: req.progress.reportProgress("Finished",
0444: rds_processed);
0445: // log - Finished {0} RDs
0446: SearchLogger.getLogger().log(Level.INFO,
0447: "PSSH_CSPSRDMR0006",
0448: Integer.toString(rds_processed));
0449: }
0450:
0451: if (delete_on_exit && ifile != null) {
0452: try {
0453: File f = new File(ifile);
0454: String cpath = f.getCanonicalPath();
0455: String dev_null_cpath = new File("/dev/null")
0456: .getCanonicalPath();
0457: if (!cpath.equals(dev_null_cpath)) {
0458: // log - Deleting input file: {0}
0459: SearchLogger.getLogger().log(Level.FINE,
0460: "PSSH_CSPSRDMR0007", ifile);
0461: f.delete();
0462: }
0463: } catch (Exception e) {
0464: if (SearchLogger.getLogger().isLoggable(
0465: Level.WARNING)) {
0466: // log - Error while deleting input file: {0}
0467: LogRecord logRecord = new LogRecord(
0468: Level.WARNING, "PSSH_CSPSRDMR0008");
0469: logRecord
0470: .setParameters(new Object[] { ifile });
0471: logRecord.setThrown(e);
0472: logRecord.setLoggerName(SearchLogger
0473: .getLogger().getName());
0474: SearchLogger.getLogger().log(logRecord);
0475: }
0476: throw new RDMgrException();
0477: }
0478: }
0479: }
0480:
0481: req.progress.finish();
0482:
0483: }
0484:
0485: catch (RDMgrException e) {
0486: // Cause of RDMgrException has already been logged (if appropriate)
0487: status = 1;
0488: }
0489:
0490: catch (Exception e) {
0491: // Uncaught exception - log it
0492: if (loggingStarted)
0493: // log - Uncaught exception:
0494: SearchLogger.getLogger().log(Level.WARNING,
0495: "PSSH_CSPSRDMR0009", e);
0496: else
0497: e.printStackTrace(System.err);
0498: status = 1;
0499: }
0500:
0501: finally {
0502: if (status != 0) {
0503: if (dfile != null && !dfile.equals("stderr")
0504: && !dfile.equals("stdout"))
0505: System.err.println(RDMgrUtil.getLocalizedString(
0506: "exiting_see", new Object[] { dfile }));
0507: else
0508: System.err.println(RDMgrUtil
0509: .getLocalizedString("exiting"));
0510: }
0511: }
0512:
0513: return status;
0514:
0515: }
0516:
0517: /**
0518: * Most of these commands are CLI only. RDSubmit.process_input() is common
0519: * to CLI and server. Some of these commands might make sense n the server,
0520: * but that is for the future. (hopefully not needed anyway if we switch protocols)
0521: * @return -1 if no rd processing was done, number of rds processed otherwise
0522: */
0523: public int doRequest(RDMgrConfig rdmgrcf, RDSubmitRequest req,
0524: SToken st, RDMTransaction t) throws Exception {
0525:
0526: int rds_processed = -1;
0527:
0528: // Operations that open the db
0529:
0530: if (do_count) {
0531: // This is returning nova count - use db_stat for BDB count
0532: req.progress
0533: .reportProgress("Counting_database", req.dbname);
0534: SearchLogger.getLogger().log(Level.INFO,
0535: "PSSH_CSPSRDMR0010", req.dbname);
0536: int n = req.db.count(st, t);
0537: System.out.println(RDMgrUtil.getLocalizedString(
0538: "Count_RDs", new Object[] { new Integer(n) }));
0539: SearchLogger.getLogger().log(Level.INFO,
0540: "PSSH_CSPSRDMR0011", Integer.toString(n));
0541: return -1;
0542: } else if (do_repartition) {
0543: req.progress.reportProgress("Repartitioning_database",
0544: req.dbname);
0545: SearchLogger.getLogger().log(Level.INFO,
0546: "PSSH_CSPSRDMR0012", req.dbname);
0547: req.db.close(st);
0548: req.db = null;
0549: String new_parts_str = SearchConfig
0550: .getValue(SearchConfig.DBPARTS);
0551: if (new_parts_str == null) {
0552: SearchLogger.getLogger().log(Level.WARNING,
0553: "PSSH_CSPSRDMR0013");
0554: throw new RDMgrException();
0555: }
0556: String[] new_parts = String2Array.string2Array(
0557: new_parts_str, ',', true);
0558: PartitionedDb pdb = new PartitionedDb();
0559: pdb.open(st, rdmgrcf.dbdir, req.dbname, RDMDb.WRITER, 0644); // create?
0560: int count = pdb.repartition(st, rdmgrcf.dbdir, req.dbname,
0561: new_parts);
0562: pdb.close(st);
0563: req.progress.reportProgress("Relocated_keys", count);
0564: // log - Relocated keys: {0}, Finished repartition
0565: SearchLogger.getLogger().log(Level.INFO,
0566: "PSSH_CSPSRDMR0014", Integer.toString(count));
0567: return -1;
0568: } else if (do_drop) {
0569: req.progress
0570: .reportProgress("Dropping_database", req.dbname);
0571: SearchLogger.getLogger().log(Level.INFO,
0572: "PSSH_CSPSRDMR0015", req.dbname);
0573: req.db.purge(st, null);
0574: req.db.close(st);
0575: req.db = null;
0576: PartitionedDb.drop(st, rdmgrcf.dbdir, req.dbname);
0577: SearchLogger.getLogger().log(Level.INFO,
0578: "PSSH_CSPSRDMR0016");
0579: return -1;
0580: } else if (do_optimize) {
0581: req.progress.reportProgress("Optimizing_database",
0582: req.dbname);
0583: SearchLogger.getLogger().log(Level.INFO,
0584: "PSSH_CSPSRDMR0017", req.dbname);
0585: req.db.optimize(st);
0586: SearchLogger.getLogger().log(Level.INFO,
0587: "PSSH_CSPSRDMR0018");
0588: return -1;
0589: } else if (do_purge) {
0590: req.progress.reportProgress("Purging_database", req.dbname);
0591: SearchLogger.getLogger().log(Level.INFO,
0592: "PSSH_CSPSRDMR0019", req.dbname);
0593: req.db.purge(st, null);
0594: SearchLogger.getLogger().log(Level.INFO,
0595: "PSSH_CSPSRDMR0020");
0596: return -1;
0597: } else if (do_dump) {
0598: req.progress.reportProgress("Dumping_database", req.dbname);
0599: SearchLogger.getLogger().log(Level.INFO,
0600: "PSSH_CSPSRDMR0021", req.dbname);
0601: if (req.is_taxonomy)
0602: req.sis = new SOIFInputStream(rdmgrcf.taxonomyfn,
0603: req.icharset);
0604: else
0605: req.sis = new SOIFDbStream(st, req.db, req.maxhits,
0606: req.view, req.type, t);
0607: SOIF s = null;
0608: if (!do_select) {
0609: SOIFOutputStream so = new SOIFOutputStream(System.out,
0610: req.ocharset);
0611: while ((s = req.sis.readSOIF()) != null)
0612: so.write(s);
0613: } else if (req.view != null) {
0614: if (req.ocharset.equalsIgnoreCase("UTF-16"))
0615: req.ocharset = "UTF-16BE"; // avoid byte marks
0616: while ((s = req.sis.readSOIF()) != null) {
0617: System.out.write((s.getURL() + "\n")
0618: .getBytes(req.ocharset));
0619: StringTokenizer tok = new StringTokenizer(req.view,
0620: ", ");
0621: while (tok.hasMoreTokens()) {
0622: String attr = tok.nextToken();
0623: AVPair avp = s.getAVPair(attr);
0624: if (avp != null) {
0625: for (int i = 0; i <= avp.getMaxIndex(); ++i) {
0626: if (avp.nthValid(i))
0627: System.out
0628: .write(("\t" + attr + "\t"
0629: + avp.getValue(i) + "\n")
0630: .getBytes(req.ocharset));
0631: }
0632: }
0633: }
0634:
0635: }
0636: }
0637: SearchLogger.getLogger().log(Level.INFO,
0638: "PSSH_CSPSRDMR0022");
0639: return -1;
0640: } else if (do_reindex) {
0641: // XXX could move this to db api, but it should really go away altogether
0642: req.progress.reportProgress("Reindexing_database",
0643: req.dbname);
0644: SearchLogger.getLogger().log(Level.INFO,
0645: "PSSH_CSPSRDMR0023", req.dbname);
0646: req.do_reindex = true;
0647: if (req.is_taxonomy)
0648: req.sis = new SOIFInputStream(rdmgrcf.taxonomyfn,
0649: req.icharset);
0650: else
0651: req.sis = new SOIFDbStream(st, req.db, req.maxhits,
0652: req.view, req.type, t);
0653: } else if (do_expire) {
0654: req.progress
0655: .reportProgress("Expiring_database", req.dbname);
0656: SearchLogger.getLogger().log(Level.INFO,
0657: "PSSH_CSPSRDMR0024", req.dbname);
0658: req.query = "rd-expires <= " + new Date();
0659: req.sis = new SOIFQueryStream(st, req.progress, req.db,
0660: req.query, req.maxhits, req.view, t);
0661: } else if (req.query != null) {
0662: req.progress.reportProgress("Query_for", req.query);
0663: SearchLogger.getLogger().log(Level.INFO,
0664: "PSSH_CSPSRDMR0025", req.query);
0665: req.sis = new SOIFQueryStream(st, req.progress, req.db,
0666: req.query, req.maxhits, req.view, t);
0667: }
0668:
0669: if (req.sis == null) {
0670: // No query stream - open the input file
0671: if (ifile == null || ifile.equals("-")
0672: || ifile.equals("stdin")) {
0673: req.sis = new SOIFInputStream(System.in, req.icharset);
0674: ifile = null;
0675: } else {
0676: try {
0677: req.sis = new SOIFInputStream(ifile, req.icharset);
0678: } catch (Exception e) {
0679: // log - Failed to open input stream: {0}
0680: SearchLogger.getLogger().log(Level.WARNING,
0681: "PSSH_CSPSRDMR0026", ifile);
0682: //throw e;
0683: throw new RDMgrException();
0684: }
0685: }
0686: }
0687:
0688: // Process the input
0689: if (req.sis != null) {
0690: //if (req.oper.equalsIgnoreCase(RDM.SUBMIT_RETRIEVE)) {
0691: // XXX where to send output?
0692: req.sos = new SOIFOutputStream(System.out, req.ocharset); // XXX no buffer?
0693: //}
0694: rds_processed = new RDSubmit().process_input(st, req, t);
0695: }
0696: return rds_processed;
0697:
0698: }
0699:
0700: class SOIFQueryStream extends SOIFInputStream {
0701: RDMResultSet rs;
0702: int curHit = 0;
0703: String query;
0704: RDMViewAttributes rdmView;
0705: SToken st;
0706: RDMTransaction t;
0707: RDMProgress p;
0708:
0709: SOIFQueryStream(SToken st, RDMProgress p, RDMDb db, String q,
0710: int nHits, String view, RDMTransaction t)
0711: throws Exception {
0712: super (new byte[] { 0 }); // dummy
0713: query = q;
0714: String viewOrder = null;
0715: if (view != null)
0716: rdmView = new RDMViewAttributes(view);
0717: // the view is filtered after fetch from BDB - needed for highlight, etc to work
0718: // XXX this should change
0719: //rs = db.search(st, query, nHits, rdmView, viewOrder, t);
0720: rs = db.search(st, query, nHits, null, viewOrder, t);
0721: p.reportProgress("results_out_of", new Object[] {
0722: new Long(rs.getResultCount()),
0723: new Long(rs.getHitCount()),
0724: new Long(rs.getDocCount()) });
0725: curHit = 0;
0726: this .st = st;
0727: this .t = t;
0728: this .p = p;
0729: }
0730:
0731: public SOIF readSOIF() throws IOException {
0732: if (curHit >= rs.getResultCount())
0733: return null;
0734: SOIF s = rs.getResult(curHit++, rdmView, null);
0735: if (s == null)
0736: throw new IOException("Error in search result stream");
0737: return s;
0738: }
0739: }
0740:
0741: class SOIFDbStream extends SOIFInputStream {
0742: Datum key, data;
0743: DbCursor cursor;
0744: RDMViewAttributes rdmView;
0745: int nHits = 0;
0746: int curHit = 0;
0747: String type;
0748: RDMDb db = null;
0749: SToken st;
0750: RDMTransaction t;
0751: boolean lookahead = false;
0752: boolean finished = false;
0753: boolean vdbViewAdded = false;
0754:
0755: SOIFDbStream(SToken st, RDMDb db, int nHits, String view,
0756: String type, RDMTransaction t) throws Exception {
0757: super (new byte[] { 0 }); // dummy
0758: key = new Datum();
0759: data = new Datum();
0760: cursor = new DbCursor((PartitionedDb) db, t, 0);
0761: if (view != null) {
0762: rdmView = new RDMViewAttributes(view);
0763: if (!rdmView.contains(st.getVirtualDBTagName())) {
0764: rdmView.add(st.getVirtualDBTagName());
0765: vdbViewAdded = true;
0766: }
0767: }
0768: this .db = db;
0769: this .nHits = nHits;
0770: this .type = type;
0771: this .st = st;
0772: this .t = t;
0773: }
0774:
0775: public SOIF readSOIF() throws IOException {
0776: if (finished)
0777: return null;
0778: if (++curHit > nHits) {
0779: try {
0780: cursor.close();
0781: finished = true;
0782: } catch (Exception e1) {
0783: SearchLogger.getLogger().log(Level.WARNING,
0784: "PSSH_CSPSRDMR0027", e1);
0785: }
0786: return null;
0787: }
0788: SOIF s = null;
0789: String keyval = null;
0790: try {
0791: do {
0792: s = null;
0793: while (s == null) {
0794: // XXX need a SOIFDb cursor to handle N/P data
0795:
0796: // grab the next RD or use lookahead
0797: if (!lookahead) {
0798: if (cursor.get(key, data, RDMDb.DB_NEXT) == RDMDb.DB_NOTFOUND) {
0799: cursor.close();
0800: return null; // EOS
0801: }
0802: } else
0803: lookahead = false;
0804: keyval = new String(key.get_data());
0805: SearchLogger.getLogger().log(Level.FINE,
0806: "PSSH_CSPSRDMR0028", keyval);
0807:
0808: // ignore unrequested P/NP types
0809: if (type.equals(RDM.SUBMIT_NONPERSISTENT)
0810: && !keyval.endsWith(".per\0")) {
0811: s = new SOIF(data.get_data(), rdmView);
0812: break;
0813: }
0814: if (type.equals(RDM.SUBMIT_PERSISTENT)
0815: && keyval.endsWith(".per\0")) {
0816: s = new SOIF(data.get_data(), rdmView);
0817: break;
0818: }
0819: // XXX ignore p-only RDs when MERGED - we should change this (in several places)
0820: if (type.equals(RDM.SUBMIT_MERGED)
0821: && !keyval.endsWith(".per\0")) {
0822: // we have the NP RD
0823: s = new SOIF(data.get_data(), rdmView);
0824: // now look for matching P RD
0825: if (cursor.get(key, data, RDMDb.DB_NEXT) == RDMDb.DB_NOTFOUND) {
0826: cursor.close();
0827: finished = true;
0828: break;
0829: }
0830: String nextkeyval = new String(key
0831: .get_data());
0832: SearchLogger.getLogger().log(Level.FINE,
0833: "PSSH_CSPSRDMR0029", nextkeyval);
0834: if (nextkeyval.regionMatches(0, keyval, 0,
0835: DbUtil.MD5_BYTES)
0836: && nextkeyval.endsWith(".per\0")) {
0837: SOIF p = new SOIF(data.get_data(),
0838: rdmView);
0839: s.merge(p);
0840: } else
0841: lookahead = true;
0842: break;
0843: }
0844: }
0845: } while (!st.validateVirtualDBTag(s));
0846: if (vdbViewAdded) {
0847: s.remove(st.getVirtualDBTagName());
0848: }
0849: SearchLogger.getLogger().log(Level.FINE,
0850: "PSSH_CSPSRDMR0030", s.getURL());
0851: } catch (Exception e) {
0852: String msg = "Database exception";
0853: if (e.getMessage() != null)
0854: msg += ": " + e.getMessage();
0855: if (keyval != null)
0856: msg += ": dbkey=" + keyval;
0857: if (SearchLogger.getLogger().isLoggable(Level.WARNING)) {
0858: LogRecord logRecord = new LogRecord(Level.WARNING,
0859: "PSSH_CSPSRDMR0031");
0860: logRecord.setParameters(new Object[] { msg });
0861: logRecord.setThrown(e);
0862: logRecord.setLoggerName(SearchLogger.getLogger()
0863: .getName());
0864: SearchLogger.getLogger().log(logRecord);
0865: }
0866: if (cursor != null) {
0867: try {
0868: cursor.close();
0869: finished = true;
0870: } catch (Exception e1) {
0871: SearchLogger.getLogger().log(Level.WARNING,
0872: "PSSH_CSPSRDMR0027", e1);
0873: }
0874: }
0875: throw new IOException("Error in database retrieval");
0876: }
0877: return s;
0878: }
0879: }
0880:
0881: public void dbopen(RDMgrConfig rdmgrcf, RDSubmitRequest req,
0882: SToken st, RDMTransaction t) throws RDMgrException {
0883: schema_init(rdmgrcf, req);
0884: try {
0885:
0886: // reject federated dbs (XXX for now)
0887: SOIF dbs = DbManager.getRootDbEntry(st, req.dbname);
0888: if (dbs != null) {
0889: String dbc = dbs.getValue("class");
0890: if (dbc != null
0891: && !dbc
0892: .equals("com.sun.portal.search.db.IndexedSOIFDb")) {
0893: // federated search db
0894: if (!federated_search) {
0895: // log - database class not supported: {0}
0896: SearchLogger.getLogger().log(Level.WARNING,
0897: "PSSH_CSPSRDMR0061", dbc);
0898: throw new RDMgrException();
0899: } else {
0900: // open federated search db using new mechanism
0901: int oflags = RDMDb.WRITER;
0902: if (req.do_create)
0903: oflags = RDMDb.WRCREAT;
0904: //req.db.open(st, rdmgrcf.dbdir, req.dbname, oflags, 0644);
0905: // XXX should oflag support search only?
0906: req.db = DbManager.dbOpen(st, req.dbname,
0907: oflags);
0908: return;
0909: }
0910: }
0911: }
0912:
0913: if (req.which_pass == RDSubmitRequest.RDM_SAGE
0914: || req.is_taxonomy) {
0915: req.db = new SearchOnlyDb();
0916: /*if (req.is_taxonomy) {
0917: req.db = new TaxonomyDb();
0918: } else if (req.which_pass == RDSubmitRequest.RDM_SAGE) {
0919: //req.db = new SearchOnlyDb();
0920: req.db = new NovaDb();
0921: } */
0922: } else {
0923: String p = SearchConfig.getValue(SearchConfig.DBCLASS);
0924: if (p != null)
0925: dbClass = p;
0926: req.db = (RDMDb) Class.forName(dbClass).newInstance();
0927:
0928: // Save content in field store for Nova only Db
0929: // XXX should do this for all db types, including BDB
0930: // to allow us to reference external content
0931: // XXX Should move to db api
0932: if (dbClass
0933: .equals("com.sun.portal.search.db.SearchOnlyDb"))
0934: NovaDb.setSaveData(true);
0935: }
0936:
0937: int oflags = RDMDb.WRITER;
0938: if (req.do_create)
0939: oflags = RDMDb.WRCREAT;
0940: // log - req.db_open({0}, {1}, {2})
0941: SearchLogger.getLogger().log(
0942: Level.FINE,
0943: "PSSH_CSPSRDMR0033",
0944: new Object[] { rdmgrcf.dbdir, req.dbname,
0945: Integer.toString(oflags) });
0946: req.db.open(st, rdmgrcf.dbdir, req.dbname, oflags, 0644);
0947: //req.db.setIndexBatchSize(st, req.max_rd_batch);
0948: } catch (Exception e) {
0949: SearchLogger.getLogger().log(Level.WARNING,
0950: "PSSH_CSPSRDMR0034",
0951: new Object[] { rdmgrcf.dbdir, req.dbname });
0952: throw new RDMgrException();
0953: }
0954: }
0955:
0956: void dbclose(RDSubmitRequest req, SToken st) throws RDMgrException {
0957: if (req.db != null) {
0958: try {
0959: // Close the session
0960: SearchLogger.getLogger().log(Level.FINE,
0961: "PSSH_CSPSRDMR0035");
0962: req.db.close(st);
0963: } catch (Exception e) {
0964: SearchLogger.getLogger().log(Level.WARNING,
0965: "PSSH_CSPSRDMR0036", e);
0966: throw new RDMgrException();
0967: }
0968: }
0969: }
0970:
0971: public void schema_init(RDMgrConfig rdmgrcf, RDSubmitRequest req)
0972: throws RDMgrException {
0973: SearchLogger.getLogger().log(Level.FINE, "PSSH_CSPSRDMR0037",
0974: rdmgrcf.schemafn);
0975: try {
0976: // XXX Should be using a RDMContext object here... (see RDMContext)
0977: SOIFInputStream ss = new SOIFInputStream(rdmgrcf.schemafn);
0978: SOIF s;
0979: RDMSchema schema;
0980: while ((s = ss.readSOIF()) != null) {
0981: schema = RDMSchema.register(new RDMSchema(s));
0982: SearchLogger.getLogger().log(Level.FINE,
0983: "PSSH_CSPSRDMR0038", schema.getName());
0984: }
0985: // XXX move to context/make explicit
0986: schema = RDMSchema.register(new RDMSchema(RDMSchema
0987: .getClassificationSchemaSOIF()));
0988: SearchLogger.getLogger().log(Level.FINE,
0989: "PSSH_CSPSRDMR0038", schema.getName());
0990: // XXX move to context/make explicit
0991: schema = RDMSchema.register(new RDMSchema(RDMSchema
0992: .getDatabaseSchemaSOIF()));
0993: SearchLogger.getLogger().log(Level.FINE,
0994: "PSSH_CSPSRDMR0038", schema.getName());
0995:
0996: if (!req.skip_alias_conf) {
0997: String aliasObj = "DOCUMENT"; // XXX Should not be restricted
0998: RDMSchema sch = RDMSchema.getSchema(aliasObj);
0999: if (sch == null) {
1000: // log - No schema for object type: {0}
1001: SearchLogger.getLogger().log(Level.WARNING,
1002: "PSSH_CSPSRDMR0039", aliasObj);
1003: req.skip_alias_conf = true;
1004: } else {
1005: req.alias = new Alias(sch);
1006: if (req.alias.size() < 1) {
1007: SearchLogger.getLogger().log(Level.FINE,
1008: "PSSH_CSPSRDMR0040");
1009: req.skip_alias_conf = true;
1010: }
1011: }
1012: }
1013:
1014: } catch (Exception e) {
1015: if (SearchLogger.getLogger().isLoggable(Level.WARNING)) {
1016: // log - Cannot parse schema: {0}
1017: LogRecord logRecord = new LogRecord(Level.WARNING,
1018: "PSSH_CSPSRDMR0041");
1019: logRecord
1020: .setParameters(new Object[] { rdmgrcf.schemafn });
1021: logRecord.setThrown(e);
1022: logRecord.setLoggerName(SearchLogger.getLogger()
1023: .getName());
1024: SearchLogger.getLogger().log(logRecord);
1025: }
1026: throw new RDMgrException();
1027: }
1028: }
1029:
1030: protected void loadConfig(RDMgrConfig rdmgrcf, RDSubmitRequest req)
1031: throws RDMgrException {
1032: String p = null;
1033: String tmpdir;
1034:
1035: if (req.dbname == null) {
1036: if ((p = searchcf.getValue(SearchConfig.DBNAME)) != null)
1037: req.dbname = p;
1038: }
1039: if (req.dbname == null) {
1040: SearchLogger.getLogger().log(
1041: Level.WARNING,
1042: "PSSH_CSPSRDMR0032",
1043: new Object[] { SearchConfig.DBNAME,
1044: searchcf.getDefault() });
1045: throw new RDMgrException();
1046: }
1047:
1048: if ((p = searchcf.getValue(SearchConfig.DBDIR)) != null)
1049: rdmgrcf.dbdir = p;
1050: else {
1051: SearchLogger.getLogger().log(
1052: Level.WARNING,
1053: "PSSH_CSPSRDMR0032",
1054: new Object[] { SearchConfig.DBDIR,
1055: searchcf.getDefault() });
1056: throw new RDMgrException();
1057: }
1058: SearchLogger.getLogger().log(Level.FINE, "PSSH_CSPSRDMR0042",
1059: rdmgrcf.dbdir);
1060:
1061: String taxdbname = searchcf.getValue(SearchConfig.TAX_DBNAME);
1062: rdmgrcf.taxonomyfn = searchcf.getValue(SearchConfig.TAX);
1063: if (req.is_taxonomy || req.dbname.equalsIgnoreCase(taxdbname)) {
1064: req.is_taxonomy = true;
1065: req.dbname = taxdbname;
1066: }
1067:
1068: // set the index batch size
1069: if (req.max_rd_batch == 0) {
1070: if ((p = searchcf.getValue(SearchConfig.MAX_INDEX_BATCH)) != null)
1071: req.max_rd_batch = Integer.parseInt(p);
1072: }
1073: if (req.max_rd_batch < 1 || req.max_rd_batch > MAX_RD_BATCH) {
1074: SearchLogger.getLogger().log(
1075: Level.WARNING,
1076: "PSSH_CSPSRDMR0043",
1077: new Object[] { SearchConfig.MAX_INDEX_BATCH,
1078: Integer.toString(MAX_RD_BATCH) });
1079: req.max_rd_batch = MAX_RD_BATCH;
1080: }
1081:
1082: // now multiply the batch size by the number of index threads,
1083: // since they share the batch between themselves
1084: int idx_threads = 1;
1085: if ((p = searchcf.getValue(SearchConfig.INDEX_THREADS)) != null) {
1086: try {
1087: idx_threads = Integer.parseInt(p);
1088: if (idx_threads < 1) {
1089: SearchLogger.getLogger().log(
1090: Level.WARNING,
1091: "PSSH_CSPSRDMR0044",
1092: new Object[] { SearchConfig.INDEX_THREADS,
1093: p });
1094: throw new RDMgrException();
1095: }
1096: } catch (Exception e) {
1097: SearchLogger.getLogger().log(Level.WARNING,
1098: "PSSH_CSPSRDMR0044",
1099: new Object[] { SearchConfig.INDEX_THREADS, p });
1100: throw new RDMgrException();
1101: }
1102: }
1103: req.max_rd_batch *= idx_threads;
1104:
1105: if (rdmgrcf.schemafn == null) // CLI can override
1106: rdmgrcf.schemafn = searchcf.getValue(SearchConfig.SCHEMA);
1107:
1108: // XXX hack - need proper dbname type mapping
1109: // XXX "rootdb" name should be configurable?
1110: if (req.dbname.equalsIgnoreCase(PartitionedDb.ROOTDB))
1111: req.is_rootdb = true;
1112:
1113: }
1114:
1115: protected int usage(int status) {
1116: System.err.println(RDMgrUtil.getLocalizedString("usage"));
1117: return status;
1118: }
1119:
1120: private String getVersionInfo() {
1121: ResourceBundle prodRB = PropertyResourceBundle
1122: .getBundle("PSversion");
1123: StringBuffer vinfo = new StringBuffer();
1124: vinfo.append(COMMANDNAME).append(" (").append(
1125: prodRB.getString("productname")).append(" ").append(
1126: prodRB.getString("productversion")).append(") ")
1127: .append(COMMANDVERSION);
1128: vinfo.append("\n").append(prodRB.getString("copyright"));
1129: return vinfo.toString();
1130: }
1131:
1132: public static void main(String argv[]) {
1133: int status;
1134: do
1135: status = new RDMgr().run(argv);
1136: while (loop);
1137: System.exit(status);
1138: }
1139:
1140: }
|