001: //=============================================================================
002: //=== Copyright (C) 2001-2007 Food and Agriculture Organization of the
003: //=== United Nations (FAO-UN), United Nations World Food Programme (WFP)
004: //=== and United Nations Environment Programme (UNEP)
005: //===
006: //=== This program is free software; you can redistribute it and/or modify
007: //=== it under the terms of the GNU General Public License as published by
008: //=== the Free Software Foundation; either version 2 of the License, or (at
009: //=== your option) any later version.
010: //===
011: //=== This program is distributed in the hope that it will be useful, but
012: //=== WITHOUT ANY WARRANTY; without even the implied warranty of
013: //=== MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
014: //=== General Public License for more details.
015: //===
016: //=== You should have received a copy of the GNU General Public License
017: //=== along with this program; if not, write to the Free Software
018: //=== Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
019: //===
020: //=== Contact: Jeroen Ticheler - FAO - Viale delle Terme di Caracalla 2,
021: //=== Rome - Italy. email: geonetwork@osgeo.org
022: //==============================================================================
023:
024: package org.fao.geonet.kernel.harvest.harvester;
025:
026: import java.lang.reflect.Method;
027: import java.sql.SQLException;
028: import java.util.HashMap;
029: import java.util.Map;
030: import jeeves.exceptions.BadInputEx;
031: import jeeves.exceptions.BadParameterEx;
032: import jeeves.exceptions.JeevesException;
033: import jeeves.exceptions.OperationAbortedEx;
034: import jeeves.interfaces.Logger;
035: import jeeves.resources.dbms.Dbms;
036: import jeeves.server.context.ServiceContext;
037: import jeeves.server.resources.ResourceManager;
038: import jeeves.utils.Log;
039: import org.fao.geonet.constants.Geonet;
040: import org.fao.geonet.kernel.DataManager;
041: import org.fao.geonet.kernel.harvest.Common.OperResult;
042: import org.fao.geonet.kernel.harvest.Common.Status;
043: import org.fao.geonet.kernel.harvest.harvester.csw.CswHarvester;
044: import org.fao.geonet.kernel.harvest.harvester.geonet.GeonetHarvester;
045: import org.fao.geonet.kernel.harvest.harvester.geonet20.Geonet20Harvester;
046: import org.fao.geonet.kernel.harvest.harvester.oaipmh.OaiPmhHarvester;
047: import org.fao.geonet.kernel.harvest.harvester.webdav.WebDavHarvester;
048: import org.fao.geonet.kernel.harvest.harvester.z3950.Z3950Harvester;
049: import org.fao.geonet.kernel.setting.SettingManager;
050: import org.fao.geonet.util.ISODate;
051: import org.jdom.Element;
052:
053: //=============================================================================
054:
055: public abstract class AbstractHarvester {
056: //---------------------------------------------------------------------------
057: //---
058: //--- Static API methods
059: //---
060: //---------------------------------------------------------------------------
061:
062: public static void staticInit(ServiceContext context)
063: throws Exception {
064: register(context, GeonetHarvester.class);
065: register(context, WebDavHarvester.class);
066: register(context, Geonet20Harvester.class);
067: register(context, CswHarvester.class);
068: register(context, Z3950Harvester.class);
069: register(context, OaiPmhHarvester.class);
070: }
071:
072: //---------------------------------------------------------------------------
073:
074: private static void register(ServiceContext context, Class harvester)
075: throws Exception {
076: try {
077: Method initMethod = harvester.getMethod("init", context
078: .getClass());
079: initMethod.invoke(null, context);
080:
081: AbstractHarvester ah = (AbstractHarvester) harvester
082: .newInstance();
083:
084: hsHarvesters.put(ah.getType(), harvester);
085: } catch (Exception e) {
086: throw new Exception("Cannot register harvester : "
087: + harvester, e);
088: }
089: }
090:
091: //---------------------------------------------------------------------------
092:
093: public static AbstractHarvester create(String type,
094: ServiceContext context, SettingManager sm, DataManager dm)
095: throws BadParameterEx, OperationAbortedEx {
096: //--- raises an exception if type is null
097:
098: if (type == null)
099: throw new BadParameterEx("type", type);
100:
101: Class c = hsHarvesters.get(type);
102:
103: if (c == null)
104: throw new BadParameterEx("type", type);
105:
106: try {
107: AbstractHarvester ah = (AbstractHarvester) c.newInstance();
108:
109: ah.context = context;
110: ah.settingMan = sm;
111: ah.dataMan = dm;
112:
113: return ah;
114: } catch (Exception e) {
115: throw new OperationAbortedEx(
116: "Cannot instantiate harvester", e);
117: }
118: }
119:
120: //--------------------------------------------------------------------------
121: //---
122: //--- API methods
123: //---
124: //--------------------------------------------------------------------------
125:
126: public void add(Dbms dbms, Element node) throws BadInputEx,
127: SQLException {
128: status = Status.INACTIVE;
129: executor = null;
130: error = null;
131: id = doAdd(dbms, node);
132: }
133:
134: //--------------------------------------------------------------------------
135:
136: public void init(Element node) throws BadInputEx {
137: id = node.getAttributeValue("id");
138: status = Status.parse(node.getChild("options").getChildText(
139: "status"));
140: executor = null;
141: error = null;
142:
143: //--- init harvester
144:
145: doInit(node);
146:
147: if (status == Status.ACTIVE) {
148: executor = new Executor(this );
149: executor.setTimeout(getParams().every);
150: executor.start();
151: }
152: }
153:
154: //--------------------------------------------------------------------------
155: /** Called when the harvesting entry is removed from the system.
156: * It is used to remove harvested metadata.
157: */
158:
159: public synchronized void destroy(Dbms dbms) throws Exception {
160: if (executor != null)
161: executor.terminate();
162:
163: executor = null;
164:
165: //--- remove all harvested metadata
166:
167: String getQuery = "SELECT id FROM Metadata WHERE harvestUuid=?";
168:
169: for (Object o : dbms.select(getQuery, getParams().uuid)
170: .getChildren()) {
171: Element el = (Element) o;
172: String id = (String) el.getChildText("id");
173:
174: dataMan.deleteMetadata(dbms, id);
175: dbms.commit();
176: }
177:
178: doDestroy(dbms);
179: }
180:
181: //--------------------------------------------------------------------------
182:
183: public synchronized OperResult start(Dbms dbms) throws SQLException {
184: if (status != Status.INACTIVE)
185: return OperResult.ALREADY_ACTIVE;
186:
187: settingMan.setValue(dbms, "harvesting/id:" + id
188: + "/options/status", Status.ACTIVE);
189:
190: status = Status.ACTIVE;
191: error = null;
192: executor = new Executor(this );
193: executor.setTimeout(getParams().every);
194: executor.start();
195:
196: return OperResult.OK;
197: }
198:
199: //--------------------------------------------------------------------------
200:
201: public synchronized OperResult stop(Dbms dbms) throws SQLException {
202: if (status != Status.ACTIVE)
203: return OperResult.ALREADY_INACTIVE;
204:
205: settingMan.setValue(dbms, "harvesting/id:" + id
206: + "/options/status", Status.INACTIVE);
207:
208: executor.terminate();
209: status = Status.INACTIVE;
210: executor = null;
211:
212: return OperResult.OK;
213: }
214:
215: //--------------------------------------------------------------------------
216:
217: public synchronized OperResult run() {
218: if (status == Status.INACTIVE)
219: return OperResult.INACTIVE;
220:
221: if (executor.isRunning())
222: return OperResult.ALREADY_RUNNING;
223:
224: executor.interrupt();
225:
226: return OperResult.OK;
227: }
228:
229: //--------------------------------------------------------------------------
230:
231: public synchronized void update(Dbms dbms, Element node)
232: throws BadInputEx, SQLException {
233: doUpdate(dbms, id, node);
234:
235: if (status == Status.ACTIVE) {
236: //--- stop executor
237: executor.terminate();
238:
239: //--- restart executor
240: error = null;
241: executor = new Executor(this );
242: executor.setTimeout(getParams().every);
243: executor.start();
244: }
245: }
246:
247: //--------------------------------------------------------------------------
248:
249: public String getID() {
250: return id;
251: }
252:
253: //--------------------------------------------------------------------------
254: /** Adds harvesting result information to each harvesting entry */
255:
256: public void addInfo(Element node) {
257: Element info = node.getChild("info");
258:
259: //--- 'running'
260:
261: boolean running = (status == Status.ACTIVE && executor
262: .isRunning());
263: info.addContent(new Element("running").setText(running + ""));
264:
265: //--- harvester specific info
266:
267: doAddInfo(node);
268:
269: //--- add error information
270:
271: if (error != null)
272: node.addContent(JeevesException.toElement(error));
273: }
274:
275: //---------------------------------------------------------------------------
276: /** Adds harvesting information to each metadata element. Some sites can generate
277: * url for thumbnails */
278:
279: public void addHarvestInfo(Element info, String id, String uuid) {
280: info.addContent(new Element("type").setText(getType()));
281: }
282:
283: //---------------------------------------------------------------------------
284: //---
285: //--- Package methods (called by Executor)
286: //---
287: //---------------------------------------------------------------------------
288:
289: void harvest() {
290: ResourceManager rm = new ResourceManager(context
291: .getProviderManager());
292:
293: Logger logger = Log.createLogger(Geonet.HARVESTER);
294:
295: String nodeName = getParams().name + " ("
296: + getClass().getSimpleName() + ")";
297:
298: error = null;
299:
300: try {
301: Dbms dbms = (Dbms) rm.open(Geonet.Res.MAIN_DB);
302:
303: //--- update lastRun
304:
305: String lastRun = new ISODate(System.currentTimeMillis())
306: .toString();
307: settingMan.setValue(dbms, "harvesting/id:" + id
308: + "/info/lastRun", lastRun);
309:
310: //--- proper harvesting
311:
312: logger.info("Started harvesting from node : " + nodeName);
313: doHarvest(logger, rm);
314: logger.info("Ended harvesting from node : " + nodeName);
315:
316: if (getParams().oneRunOnly)
317: stop(dbms);
318:
319: rm.close();
320: } catch (Throwable t) {
321: logger.warning("Raised exception while harvesting from : "
322: + nodeName);
323: logger.warning(" (C) Class : "
324: + t.getClass().getSimpleName());
325: logger.warning(" (C) Message : " + t.getMessage());
326:
327: error = t;
328: t.printStackTrace();
329:
330: try {
331: rm.abort();
332: } catch (Exception ex) {
333: logger.warning("CANNOT ABORT EXCEPTION");
334: logger.warning(" (C) Exc : " + ex);
335: }
336: }
337: }
338:
339: //---------------------------------------------------------------------------
340: //---
341: //--- Abstract methods that must be overridden
342: //---
343: //---------------------------------------------------------------------------
344:
345: public abstract String getType();
346:
347: public abstract AbstractParams getParams();
348:
349: protected abstract void doInit(Element entry) throws BadInputEx;
350:
351: protected abstract void doDestroy(Dbms dbms) throws SQLException;
352:
353: protected abstract String doAdd(Dbms dbms, Element node)
354: throws BadInputEx, SQLException;
355:
356: protected abstract void doUpdate(Dbms dbms, String id, Element node)
357: throws BadInputEx, SQLException;
358:
359: protected abstract void doAddInfo(Element node);
360:
361: protected abstract void doHarvest(Logger l, ResourceManager rm)
362: throws Exception;
363:
364: //---------------------------------------------------------------------------
365: //---
366: //--- Protected storage methods
367: //---
368: //---------------------------------------------------------------------------
369:
370: protected void storeNode(Dbms dbms, AbstractParams params,
371: String path) throws SQLException {
372: String siteId = settingMan.add(dbms, path, "site", "");
373: String optionsId = settingMan.add(dbms, path, "options", "");
374: String infoId = settingMan.add(dbms, path, "info", "");
375:
376: //--- setup site node ----------------------------------------
377:
378: settingMan.add(dbms, "id:" + siteId, "name", params.name);
379: settingMan.add(dbms, "id:" + siteId, "uuid", params.uuid);
380:
381: String useAccId = settingMan.add(dbms, "id:" + siteId,
382: "useAccount", params.useAccount);
383:
384: settingMan.add(dbms, "id:" + useAccId, "username",
385: params.username);
386: settingMan.add(dbms, "id:" + useAccId, "password",
387: params.password);
388:
389: //--- setup options node ---------------------------------------
390:
391: settingMan.add(dbms, "id:" + optionsId, "every", params.every);
392: settingMan.add(dbms, "id:" + optionsId, "oneRunOnly",
393: params.oneRunOnly);
394: settingMan.add(dbms, "id:" + optionsId, "status", status);
395:
396: //--- setup stats node ----------------------------------------
397:
398: settingMan.add(dbms, "id:" + infoId, "lastRun", "");
399:
400: //--- store privileges and categories ------------------------
401:
402: storePrivileges(dbms, params, path);
403: storeCategories(dbms, params, path);
404:
405: storeNodeExtra(dbms, params, path, siteId, optionsId);
406: }
407:
408: //---------------------------------------------------------------------------
409: /** Override this method with an empty body to avoid privileges storage */
410:
411: protected void storePrivileges(Dbms dbms, AbstractParams params,
412: String path) throws SQLException {
413: String privId = settingMan.add(dbms, path, "privileges", "");
414:
415: for (Privileges p : params.getPrivileges()) {
416: String groupId = settingMan.add(dbms, "id:" + privId,
417: "group", p.getGroupId());
418:
419: for (int oper : p.getOperations())
420: settingMan
421: .add(dbms, "id:" + groupId, "operation", oper);
422: }
423: }
424:
425: //---------------------------------------------------------------------------
426: /** Override this method with an empty body to avoid categories storage */
427:
428: protected void storeCategories(Dbms dbms, AbstractParams params,
429: String path) throws SQLException {
430: String categId = settingMan.add(dbms, path, "categories", "");
431:
432: for (String id : params.getCategories())
433: settingMan.add(dbms, "id:" + categId, "category", id);
434: }
435:
436: //---------------------------------------------------------------------------
437: /** Override this method to store harvesting node's specific settings */
438:
439: protected void storeNodeExtra(Dbms dbms, AbstractParams params,
440: String path, String siteId, String optionsId)
441: throws SQLException {
442: }
443:
444: //---------------------------------------------------------------------------
445:
446: protected void setValue(Map<String, Object> values, String path,
447: Element el, String name) {
448: if (el == null)
449: return;
450:
451: String value = el.getChildText(name);
452:
453: if (value != null)
454: values.put(path, value);
455: }
456:
457: //---------------------------------------------------------------------------
458:
459: protected void add(Element el, String name, int value) {
460: el.addContent(new Element(name)
461: .setText(Integer.toString(value)));
462: }
463:
464: //--------------------------------------------------------------------------
465: //---
466: //--- Variables
467: //---
468: //--------------------------------------------------------------------------
469:
470: private String id;
471: private Status status;
472:
473: private Executor executor;
474: private Throwable error;
475:
476: protected ServiceContext context;
477: protected SettingManager settingMan;
478: protected DataManager dataMan;
479:
480: private static Map<String, Class> hsHarvesters = new HashMap<String, Class>();
481: }
482:
483: //=============================================================================
|