001: /**********************************************************************************
002: * $URL: https://source.sakaiproject.org/svn/search/tags/sakai_2-4-1/search-impl/impl/src/java/org/sakaiproject/search/component/service/impl/SearchIndexBuilderImpl.java $
003: * $Id: SearchIndexBuilderImpl.java 29315 2007-04-20 14:28:12Z ajpoland@iupui.edu $
004: ***********************************************************************************
005: *
006: * Copyright (c) 2003, 2004, 2005, 2006 The Sakai Foundation.
007: *
008: * Licensed under the Educational Community License, Version 1.0 (the "License");
009: * you may not use this file except in compliance with the License.
010: * You may obtain a copy of the License at
011: *
012: * http://www.opensource.org/licenses/ecl1.php
013: *
014: * Unless required by applicable law or agreed to in writing, software
015: * distributed under the License is distributed on an "AS IS" BASIS,
016: * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
017: * See the License for the specific language governing permissions and
018: * limitations under the License.
019: *
020: **********************************************************************************/package org.sakaiproject.search.component.service.impl;
021:
022: import java.text.MessageFormat;
023: import java.util.ArrayList;
024: import java.util.Iterator;
025: import java.util.List;
026:
027: import org.apache.commons.logging.Log;
028: import org.apache.commons.logging.LogFactory;
029: import org.sakaiproject.component.api.ComponentManager;
030: import org.sakaiproject.event.api.Event;
031: import org.sakaiproject.event.api.Notification;
032: import org.sakaiproject.exception.IdUnusedException;
033: import org.sakaiproject.exception.PermissionException;
034: import org.sakaiproject.exception.TypeException;
035: import org.sakaiproject.search.api.EntityContentProducer;
036: import org.sakaiproject.search.api.SearchIndexBuilder;
037: import org.sakaiproject.search.api.SearchIndexBuilderWorker;
038: import org.sakaiproject.search.dao.SearchBuilderItemDao;
039: import org.sakaiproject.search.model.SearchBuilderItem;
040: import org.sakaiproject.search.model.SearchWriterLock;
041: import org.sakaiproject.site.api.Site;
042: import org.sakaiproject.site.api.ToolConfiguration;
043: import org.sakaiproject.site.cover.SiteService;
044:
045: /**
046: * Search index builder is expected to be registered in spring as org.sakaiproject.search.api.SearchIndexBuilder as a singleton. It receives resources which it adds to its list of pending documents to be indexed. A seperate thread then runs thtough the
047: * list of entities to be indexed, updating the index. Each time the index is updates an event is posted to force the Search components that are using the index to reload. Incremental updates to the Lucene index require that the searchers reload the index
048: * once the idex writer has been built.
049: *
050: * @author ieb
051: */
052:
053: public class SearchIndexBuilderImpl implements SearchIndexBuilder {
054:
055: private static Log log = LogFactory
056: .getLog(SearchIndexBuilderImpl.class);
057:
058: private SearchBuilderItemDao searchBuilderItemDao = null;
059:
060: private SearchIndexBuilderWorker searchIndexBuilderWorker = null;
061:
062: private List producers = new ArrayList();
063:
064: private boolean onlyIndexSearchToolSites = false;
065:
066: private boolean diagnostics = false;
067:
068: public void init() {
069: ComponentManager cm = org.sakaiproject.component.cover.ComponentManager
070: .getInstance();
071: searchIndexBuilderWorker = (SearchIndexBuilderWorker) load(cm,
072: SearchIndexBuilderWorker.class.getName());
073: try {
074:
075: } catch (Throwable t) {
076: log.error("Failed to init ", t);
077: }
078: log.info(this + " completed init()");
079: }
080:
081: private Object load(ComponentManager cm, String name) {
082: Object o = cm.get(name);
083: if (o == null) {
084: log.error("Cant find Spring component named " + name);
085: }
086: return o;
087: }
088:
089: /**
090: * register an entity content producer to provide content to the search engine {@inheritDoc}
091: */
092: public void registerEntityContentProducer(EntityContentProducer ecp) {
093: log.debug("register " + ecp);
094: producers.add(ecp);
095: }
096:
097: /**
098: * Add a resource to the indexing queue {@inheritDoc}
099: */
100: public void addResource(Notification notification, Event event) {
101: log.debug("Add resource " + notification + "::" + event);
102: String resourceName = event.getResource();
103: if (resourceName == null) {
104: // default if null
105: resourceName = "";
106: }
107: if (resourceName.length() > 255) {
108: log
109: .warn("Entity Reference is longer than 255 characters, not indexing. Reference="
110: + resourceName);
111: return;
112: }
113: EntityContentProducer ecp = newEntityContentProducer(event);
114: if (ecp.getSiteId(resourceName) == null) {
115: log.debug("Not indexing " + resourceName
116: + " as it has no context");
117: return;
118: }
119: if (onlyIndexSearchToolSites) {
120: try {
121: String siteId = ecp.getSiteId(resourceName);
122: Site s = SiteService.getSite(siteId);
123: ToolConfiguration t = s
124: .getToolForCommonId("sakai.search");
125: if (t == null) {
126: log.debug("Not indexing " + resourceName
127: + " as it has no search tool");
128: return;
129: }
130: } catch (Exception ex) {
131: log.debug("Not indexing " + resourceName
132: + " as it has no site", ex);
133: return;
134:
135: }
136: }
137: Integer action = ecp.getAction(event);
138: try {
139: SearchBuilderItem sb = searchBuilderItemDao
140: .findByName(resourceName);
141: if (sb == null) {
142: // new
143: sb = searchBuilderItemDao.create();
144: sb.setSearchaction(action);
145: sb.setName(resourceName);
146: String siteId = ecp.getSiteId(resourceName);
147: if (siteId == null || siteId.length() == 0) {
148: // default if null should neve happen
149: siteId = "none";
150: }
151: sb.setContext(siteId);
152: sb.setSearchstate(SearchBuilderItem.STATE_PENDING);
153: } else {
154: sb.setSearchaction(action);
155: String siteId = ecp.getSiteId(resourceName);
156: if (siteId == null || siteId.length() == 0) {
157: // default if null, should never happen
158: siteId = "none";
159: }
160: sb.setContext(siteId);
161: sb.setName(resourceName);
162: sb.setSearchstate(SearchBuilderItem.STATE_PENDING);
163: }
164: searchBuilderItemDao.update(sb);
165: log.debug("SEARCHBUILDER: Added Resource " + action + " "
166: + sb.getName());
167: } catch (Throwable t) {
168: log.debug("In trying to register resource " + resourceName
169: + " in search engine this resource will"
170: + " not be indexed untill it is modified");
171: }
172: searchIndexBuilderWorker.incrementActivity();
173: restartBuilder();
174: }
175:
176: /**
177: * refresh the index from the current stored state {@inheritDoc}
178: */
179: public void refreshIndex() {
180:
181: SearchBuilderItem sb = searchBuilderItemDao
182: .findByName(SearchBuilderItem.GLOBAL_MASTER);
183: if (sb == null) {
184: log.debug("Created NEW " + SearchBuilderItem.GLOBAL_MASTER);
185: sb = searchBuilderItemDao.create();
186: }
187:
188: if ((SearchBuilderItem.STATE_COMPLETED.equals(sb
189: .getSearchstate()))
190: || (!SearchBuilderItem.ACTION_REBUILD.equals(sb
191: .getSearchaction())
192: && !SearchBuilderItem.STATE_PENDING.equals(sb
193: .getSearchstate()) && !SearchBuilderItem.STATE_PENDING_2
194: .equals(sb.getSearchstate()))) {
195: sb.setSearchaction(SearchBuilderItem.ACTION_REFRESH);
196: sb.setName(SearchBuilderItem.GLOBAL_MASTER);
197: sb.setContext(SearchBuilderItem.GLOBAL_CONTEXT);
198: sb.setSearchstate(SearchBuilderItem.STATE_PENDING);
199: searchBuilderItemDao.update(sb);
200: log.debug("SEARCHBUILDER: REFRESH ALL "
201: + sb.getSearchaction() + " " + sb.getName());
202: restartBuilder();
203: } else {
204: log.debug("SEARCHBUILDER: REFRESH ALL IN PROGRESS "
205: + sb.getSearchaction() + " " + sb.getName());
206: }
207: }
208:
209: public void destroy() {
210: searchIndexBuilderWorker.destroy();
211: }
212:
213: /*
214: * List l = searchBuilderItemDao.getAll(); for (Iterator i = l.iterator(); i.hasNext();) { SearchBuilderItemImpl sbi = (SearchBuilderItemImpl) i.next(); sbi.setSearchstate(SearchBuilderItem.STATE_PENDING);
215: * sbi.setSearchaction(SearchBuilderItem.ACTION_ADD); try { log.info(" Updating " + sbi.getName()); searchBuilderItemDao.update(sbi); } catch (Exception ex) { try { sbi = (SearchBuilderItemImpl) searchBuilderItemDao .findByName(sbi.getName()); if (sbi !=
216: * null) { sbi.setSearchstate(SearchBuilderItem.STATE_PENDING); sbi.setSearchaction(SearchBuilderItem.ACTION_ADD); searchBuilderItemDao.update(sbi); } } catch (Exception e) { log.warn("Failed to update on second attempt " + e.getMessage()); } } }
217: * restartBuilder(); }
218: */
219: /**
220: * Rebuild the index from the entities own stored state {@inheritDoc}
221: */
222: public void rebuildIndex() {
223:
224: try {
225: SearchBuilderItem sb = searchBuilderItemDao
226: .findByName(SearchBuilderItem.GLOBAL_MASTER);
227: if (sb == null) {
228: sb = searchBuilderItemDao.create();
229: }
230: sb.setSearchaction(SearchBuilderItem.ACTION_REBUILD);
231: sb.setName(SearchBuilderItem.GLOBAL_MASTER);
232: sb.setContext(SearchBuilderItem.GLOBAL_CONTEXT);
233: sb.setSearchstate(SearchBuilderItem.STATE_PENDING);
234: searchBuilderItemDao.update(sb);
235: log.debug("SEARCHBUILDER: REBUILD ALL "
236: + sb.getSearchaction() + " " + sb.getName());
237:
238: } catch (Exception ex) {
239: log.warn(" rebuild index encountered a problme "
240: + ex.getMessage());
241: }
242: restartBuilder();
243: }
244:
245: /*
246: * for (Iterator i = producers.iterator(); i.hasNext();) { EntityContentProducer ecp = (EntityContentProducer) i.next(); List contentList = ecp.getAllContent(); for (Iterator ci = contentList.iterator(); ci.hasNext();) { String resourceName = (String)
247: * ci.next(); } } }
248: */
249: /**
250: * This adds and event to the list and if necessary starts a processing thread The method is syncronised with removeFromList
251: *
252: * @param e
253: */
254: private void restartBuilder() {
255: searchIndexBuilderWorker.checkRunning();
256: }
257:
258: /**
259: * Generates a SearchableEntityProducer
260: *
261: * @param ref
262: * @return
263: * @throws PermissionException
264: * @throws IdUnusedException
265: * @throws TypeException
266: */
267: public EntityContentProducer newEntityContentProducer(String ref) {
268: log.debug(" new entitycontent producer");
269: for (Iterator i = producers.iterator(); i.hasNext();) {
270: EntityContentProducer ecp = (EntityContentProducer) i
271: .next();
272: if (ecp.matches(ref)) {
273: return ecp;
274: }
275: }
276: return null;
277: }
278:
279: /**
280: * get hold of an entity content producer using the event
281: *
282: * @param event
283: * @return
284: */
285: public EntityContentProducer newEntityContentProducer(Event event) {
286: log.debug(" new entitycontent producer");
287: for (Iterator i = producers.iterator(); i.hasNext();) {
288: EntityContentProducer ecp = (EntityContentProducer) i
289: .next();
290: if (ecp.matches(event)) {
291: log.debug(" Matched Entity Content Producer for event "
292: + event + " with " + ecp);
293: return ecp;
294: } else {
295: log.debug("Skipped ECP " + ecp);
296: }
297: }
298: log
299: .debug("Failed to match any Entity Content Producer for event "
300: + event);
301: return null;
302: }
303:
304: /**
305: * @return Returns the searchBuilderItemDao.
306: */
307: public SearchBuilderItemDao getSearchBuilderItemDao() {
308: return searchBuilderItemDao;
309: }
310:
311: /**
312: * @param searchBuilderItemDao
313: * The searchBuilderItemDao to set.
314: */
315: public void setSearchBuilderItemDao(
316: SearchBuilderItemDao searchBuilderItemDao) {
317: this .searchBuilderItemDao = searchBuilderItemDao;
318: }
319:
320: /**
321: * return true if the queue is empty
322: *
323: * @{inheritDoc}
324: */
325: public boolean isBuildQueueEmpty() {
326: int n = searchBuilderItemDao.countPending();
327: log.debug("Queue has " + n);
328: return (n == 0);
329: }
330:
331: /**
332: * get all the producers registerd, as a clone to avoid concurrent modification exceptions
333: *
334: * @return
335: */
336: public List getContentProducers() {
337: return new ArrayList(producers);
338: }
339:
340: public int getPendingDocuments() {
341: return searchBuilderItemDao.countPending();
342: }
343:
344: /**
345: * Rebuild the index from the entities own stored state {@inheritDoc}, just the supplied siteId
346: */
347: public void rebuildIndex(String currentSiteId) {
348:
349: try {
350: if (currentSiteId == null || currentSiteId.length() == 0) {
351: currentSiteId = "none";
352: }
353: String siteMaster = MessageFormat.format(
354: SearchBuilderItem.SITE_MASTER_FORMAT,
355: new Object[] { currentSiteId });
356: SearchBuilderItem sb = searchBuilderItemDao
357: .findByName(siteMaster);
358: if (sb == null) {
359: sb = searchBuilderItemDao.create();
360: }
361: sb.setSearchaction(SearchBuilderItem.ACTION_REBUILD);
362: sb.setName(siteMaster);
363: sb.setContext(currentSiteId);
364: sb.setSearchstate(SearchBuilderItem.STATE_PENDING);
365: searchBuilderItemDao.update(sb);
366: log.debug("SEARCHBUILDER: REBUILD CONTEXT "
367: + sb.getSearchaction() + " " + sb.getName());
368:
369: } catch (Exception ex) {
370: log.warn(" rebuild index encountered a problme "
371: + ex.getMessage());
372: }
373: restartBuilder();
374: }
375:
376: /**
377: * Refresh the index fo the supplied site.
378: */
379: public void refreshIndex(String currentSiteId) {
380: if (currentSiteId == null || currentSiteId.length() == 0) {
381: currentSiteId = "none";
382: }
383: String siteMaster = MessageFormat.format(
384: SearchBuilderItem.SITE_MASTER_FORMAT,
385: new Object[] { currentSiteId });
386: SearchBuilderItem sb = searchBuilderItemDao
387: .findByName(siteMaster);
388: if (sb == null) {
389: log.debug("Created NEW " + siteMaster);
390: sb = searchBuilderItemDao.create();
391: sb.setContext(currentSiteId);
392: sb.setName(siteMaster);
393: sb.setSearchstate(SearchBuilderItem.STATE_COMPLETED);
394: sb.setSearchaction(SearchBuilderItem.ACTION_REFRESH);
395: }
396: if ((SearchBuilderItem.STATE_COMPLETED.equals(sb
397: .getSearchstate()))
398: || (!SearchBuilderItem.ACTION_REBUILD.equals(sb
399: .getSearchaction())
400: && !SearchBuilderItem.STATE_PENDING.equals(sb
401: .getSearchstate()) && !SearchBuilderItem.STATE_PENDING_2
402: .equals(sb.getSearchstate()))) {
403: sb.setSearchaction(SearchBuilderItem.ACTION_REFRESH);
404: sb.setName(siteMaster);
405: sb.setContext(currentSiteId);
406: sb.setSearchstate(SearchBuilderItem.STATE_PENDING);
407: searchBuilderItemDao.update(sb);
408: log.debug("SEARCHBUILDER: REFRESH CONTEXT "
409: + sb.getSearchaction() + " " + sb.getName());
410:
411: restartBuilder();
412: } else {
413: log.debug("SEARCHBUILDER: REFRESH CONTEXT IN PROGRESS "
414: + sb.getSearchaction() + " " + sb.getName());
415: }
416: }
417:
418: public List getAllSearchItems() {
419:
420: return searchBuilderItemDao.getAll();
421: }
422:
423: public List getGlobalMasterSearchItems() {
424: return searchBuilderItemDao.getGlobalMasters();
425: }
426:
427: public List getSiteMasterSearchItems() {
428: return searchBuilderItemDao.getSiteMasters();
429: }
430:
431: public SearchWriterLock getCurrentLock() {
432: return searchIndexBuilderWorker.getCurrentLock();
433: }
434:
435: public List getNodeStatus() {
436: return searchIndexBuilderWorker.getNodeStatus();
437: }
438:
439: public boolean removeWorkerLock() {
440: return searchIndexBuilderWorker.removeWorkerLock();
441: }
442:
443: public String getLastDocument() {
444: return searchIndexBuilderWorker.getLastDocument();
445: }
446:
447: public String getLastElapsed() {
448: return searchIndexBuilderWorker.getLastElapsed();
449: }
450:
451: public String getCurrentDocument() {
452: return searchIndexBuilderWorker.getCurrentDocument();
453: }
454:
455: public String getCurrentElapsed() {
456: return searchIndexBuilderWorker.getCurrentElapsed();
457: }
458:
459: /**
460: * @return the onlyIndexSearchToolSites
461: */
462: public boolean isOnlyIndexSearchToolSites() {
463: return onlyIndexSearchToolSites;
464: }
465:
466: /**
467: * @param onlyIndexSearchToolSites the onlyIndexSearchToolSites to set
468: */
469: public void setOnlyIndexSearchToolSites(
470: boolean onlyIndexSearchToolSites) {
471: this .onlyIndexSearchToolSites = onlyIndexSearchToolSites;
472: }
473:
474: /* (non-Javadoc)
475: * @see org.sakaiproject.search.api.SearchIndexBuilder#isLocalLock()
476: */
477: public boolean isLocalLock() {
478: return searchIndexBuilderWorker.isLocalLock();
479: }
480:
481: /* (non-Javadoc)
482: * @see org.sakaiproject.search.api.Diagnosable#disableDiagnostics()
483: */
484: public void disableDiagnostics() {
485: diagnostics = false;
486: searchIndexBuilderWorker.enableDiagnostics();
487: }
488:
489: /* (non-Javadoc)
490: * @see org.sakaiproject.search.api.Diagnosable#enableDiagnostics()
491: */
492: public void enableDiagnostics() {
493: diagnostics = true;
494: searchIndexBuilderWorker.disableDiagnostics();
495: }
496:
497: /* (non-Javadoc)
498: * @see org.sakaiproject.search.api.Diagnosable#hasDiagnostics()
499: */
500: public boolean hasDiagnostics() {
501: return diagnostics;
502: }
503: }
|