001: /*
002: * Copyright 2004-2006 the original author or authors.
003: *
004: * Licensed under the Apache License, Version 2.0 (the "License");
005: * you may not use this file except in compliance with the License.
006: * You may obtain a copy of the License at
007: *
008: * http://www.apache.org/licenses/LICENSE-2.0
009: *
010: * Unless required by applicable law or agreed to in writing, software
011: * distributed under the License is distributed on an "AS IS" BASIS,
012: * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013: * See the License for the specific language governing permissions and
014: * limitations under the License.
015: */
016:
017: package org.compass.core.lucene.engine.store;
018:
019: import java.io.IOException;
020: import java.io.InputStream;
021: import java.util.ArrayList;
022: import java.util.Arrays;
023: import java.util.HashMap;
024: import java.util.HashSet;
025: import java.util.List;
026: import java.util.Map;
027: import java.util.Properties;
028: import java.util.concurrent.ConcurrentHashMap;
029:
030: import org.apache.commons.logging.Log;
031: import org.apache.commons.logging.LogFactory;
032: import org.apache.lucene.analysis.standard.StandardAnalyzer;
033: import org.apache.lucene.index.IndexReader;
034: import org.apache.lucene.index.IndexWriter;
035: import org.apache.lucene.index.LuceneUtils;
036: import org.apache.lucene.store.Directory;
037: import org.apache.lucene.store.DirectoryWrapper;
038: import org.apache.lucene.store.LockFactory;
039: import org.apache.lucene.store.NativeFSLockFactory;
040: import org.apache.lucene.store.NoLockFactory;
041: import org.apache.lucene.store.SimpleFSLockFactory;
042: import org.apache.lucene.store.SingleInstanceLockFactory;
043: import org.compass.core.config.CompassConfigurable;
044: import org.compass.core.config.CompassEnvironment;
045: import org.compass.core.config.CompassSettings;
046: import org.compass.core.config.ConfigurationException;
047: import org.compass.core.engine.SearchEngine;
048: import org.compass.core.engine.SearchEngineException;
049: import org.compass.core.engine.event.SearchEngineEventManager;
050: import org.compass.core.lucene.LuceneEnvironment;
051: import org.compass.core.lucene.engine.LuceneSearchEngineFactory;
052: import org.compass.core.lucene.engine.store.localcache.LocalDirectoryCacheManager;
053: import org.compass.core.lucene.engine.store.wrapper.DirectoryWrapperProvider;
054: import org.compass.core.mapping.CompassMapping;
055: import org.compass.core.mapping.ResourceMapping;
056: import org.compass.core.util.ClassUtils;
057: import org.compass.core.util.StringUtils;
058:
059: /**
060: * @author kimchy
061: */
062: public class DefaultLuceneSearchEngineStore implements
063: LuceneSearchEngineStore {
064:
065: private static Log log = LogFactory
066: .getLog(DefaultLuceneSearchEngineStore.class);
067:
068: private CompassSettings settings;
069:
070: private DirectoryStore directoryStore;
071:
072: private Map<String, List<String>> aliasesBySubIndex = new HashMap<String, List<String>>();
073:
074: private Map<String, List<String>> subIndexesByAlias = new HashMap<String, List<String>>();
075:
076: private String defaultSubContext;
077:
078: private String[] subIndexes;
079:
080: private String connectionString;
081:
082: private DirectoryWrapperProvider[] directoryWrapperProviders;
083:
084: private LocalDirectoryCacheManager localDirectoryCacheManager;
085:
086: private Map<String, Map<String, Directory>> dirs;
087:
088: private volatile boolean closed = false;
089:
090: public void configure(
091: LuceneSearchEngineFactory searchEngineFactory,
092: CompassSettings settings, CompassMapping mapping) {
093: this .settings = settings;
094: this .connectionString = settings
095: .getSetting(CompassEnvironment.CONNECTION);
096: this .dirs = new ConcurrentHashMap<String, Map<String, Directory>>();
097:
098: this .defaultSubContext = settings.getSetting(
099: CompassEnvironment.CONNECTION_SUB_CONTEXT, "index");
100:
101: // setup the directory store
102: String connection = settings
103: .getSetting(CompassEnvironment.CONNECTION);
104: if (connection.startsWith(RAMDirectoryStore.PROTOCOL)) {
105: directoryStore = new RAMDirectoryStore();
106: } else if (connection.startsWith(FSDirectoryStore.PROTOCOL)) {
107: directoryStore = new FSDirectoryStore();
108: } else if (connection.startsWith(MMapDirectoryStore.PROTOCOL)) {
109: directoryStore = new MMapDirectoryStore();
110: } else if (connection.startsWith(JdbcDirectoryStore.PROTOCOL)) {
111: directoryStore = new JdbcDirectoryStore();
112: } else if (connection.indexOf("://") > -1) {
113: String pluggableStore = connection.substring(0, connection
114: .indexOf("://"));
115: InputStream is = LuceneSearchEngineStore.class
116: .getResourceAsStream("/META-INF/compass/store-"
117: + pluggableStore + ".properties");
118: Properties props;
119: try {
120: props = new Properties();
121: props.load(is);
122: } catch (Exception e) {
123: try {
124: is.close();
125: } catch (Exception e1) {
126: // ignore
127: }
128: throw new SearchEngineException(
129: "Failed to create store [" + connection + "]",
130: e);
131: }
132: String className = props.getProperty("type");
133: try {
134: directoryStore = (DirectoryStore) ClassUtils.forName(
135: className, settings.getClassLoader())
136: .newInstance();
137: } catch (Exception e) {
138: throw new SearchEngineException(
139: "Failed to create connection [" + connection
140: + "]", e);
141: }
142: } else {
143: directoryStore = new FSDirectoryStore();
144: }
145: if (directoryStore instanceof CompassConfigurable) {
146: ((CompassConfigurable) directoryStore).configure(settings);
147: }
148:
149: // setup sub indexes and aliases
150: HashSet<String> subIndexesSet = new HashSet<String>();
151: for (ResourceMapping resourceMapping : mapping
152: .getRootMappings()) {
153: String alias = resourceMapping.getAlias();
154: String[] tempSubIndexes = resourceMapping.getSubIndexHash()
155: .getSubIndexes();
156: for (String subIndex : tempSubIndexes) {
157: subIndexesSet.add(subIndex.intern());
158:
159: List<String> list = subIndexesByAlias.get(alias);
160: if (list == null) {
161: list = new ArrayList<String>();
162: subIndexesByAlias.put(alias, list);
163: }
164: list.add(subIndex);
165:
166: list = aliasesBySubIndex.get(subIndex);
167: if (aliasesBySubIndex.get(subIndex) == null) {
168: list = new ArrayList<String>();
169: aliasesBySubIndex.put(subIndex, list);
170: }
171: list.add(alias);
172: }
173: }
174: subIndexes = subIndexesSet.toArray(new String[subIndexesSet
175: .size()]);
176:
177: // set up directory wrapper providers
178: Map<String, CompassSettings> dwSettingGroups = settings
179: .getSettingGroups(LuceneEnvironment.DirectoryWrapper.PREFIX);
180: if (dwSettingGroups.size() > 0) {
181: ArrayList<DirectoryWrapperProvider> dws = new ArrayList<DirectoryWrapperProvider>();
182: for (Map.Entry<String, CompassSettings> entry : dwSettingGroups
183: .entrySet()) {
184: String dwName = entry.getKey();
185: if (log.isInfoEnabled()) {
186: log.info("Building directory wrapper [" + dwName
187: + "]");
188: }
189: CompassSettings dwSettings = entry.getValue();
190: String dwType = dwSettings
191: .getSetting(LuceneEnvironment.DirectoryWrapper.TYPE);
192: if (dwType == null) {
193: throw new ConfigurationException(
194: "Directory wrapper ["
195: + dwName
196: + "] has no type associated with it");
197: }
198: DirectoryWrapperProvider dw;
199: try {
200: dw = (DirectoryWrapperProvider) ClassUtils.forName(
201: dwType, settings.getClassLoader())
202: .newInstance();
203: } catch (Exception e) {
204: throw new ConfigurationException(
205: "Failed to create directory wrapper ["
206: + dwName + "]", e);
207: }
208: if (dw instanceof CompassConfigurable) {
209: ((CompassConfigurable) dw).configure(dwSettings);
210: }
211: dws.add(dw);
212: }
213: directoryWrapperProviders = dws
214: .toArray(new DirectoryWrapperProvider[dws.size()]);
215: }
216:
217: this .localDirectoryCacheManager = new LocalDirectoryCacheManager(
218: searchEngineFactory);
219: localDirectoryCacheManager.configure(settings);
220: }
221:
222: public void close() {
223: if (closed) {
224: return;
225: }
226: closed = true;
227: localDirectoryCacheManager.close();
228: closeDirectories();
229: }
230:
231: private void closeDirectories() {
232: for (Map<String, Directory> subIndexsDirs : dirs.values()) {
233: synchronized (subIndexsDirs) {
234: for (Directory dir : subIndexsDirs.values()) {
235: try {
236: dir.close();
237: } catch (IOException e) {
238: log
239: .debug(
240: "Failed to close directory while shutting down, ignoring",
241: e);
242: }
243: }
244: }
245: }
246: dirs.clear();
247: }
248:
249: public void performScheduledTasks() {
250: for (Map.Entry<String, Map<String, Directory>> entry : dirs
251: .entrySet()) {
252: String subContext = entry.getKey();
253: synchronized (entry.getValue()) {
254: for (Map.Entry<String, Directory> entry2 : entry
255: .getValue().entrySet()) {
256: String subIndex = entry2.getKey();
257: Directory dir = entry2.getValue();
258: directoryStore.performScheduledTasks(
259: unwrapDir(dir), subContext, subIndex);
260: }
261: }
262: }
263: }
264:
265: public String[] getAliasesBySubIndex(String subIndex) {
266: List<String> aliasesPerSubIndex = aliasesBySubIndex
267: .get(subIndex);
268: return aliasesPerSubIndex.toArray(new String[aliasesPerSubIndex
269: .size()]);
270: }
271:
272: public int getNumberOfAliasesBySubIndex(String subIndex) {
273: return (aliasesBySubIndex.get(subIndex)).size();
274: }
275:
276: public String[] getSubIndexes() {
277: return subIndexes;
278: }
279:
280: public String[] calcSubIndexes(String[] subIndexes, String[] aliases) {
281: if (aliases == null) {
282: if (subIndexes == null) {
283: return getSubIndexes();
284: }
285: return subIndexes;
286: }
287: HashSet<String> ret = new HashSet<String>();
288: for (String aliase : aliases) {
289: List<String> subIndexesList = subIndexesByAlias.get(aliase);
290: if (subIndexesList == null) {
291: throw new IllegalArgumentException(
292: "No sub-index is mapped to alias [" + aliase
293: + "]");
294: }
295: for (String subIndex : subIndexesList) {
296: ret.add(subIndex);
297: }
298: }
299: if (subIndexes != null) {
300: ret.addAll(Arrays.asList(subIndexes));
301: }
302: return ret.toArray(new String[ret.size()]);
303: }
304:
305: public Directory openDirectory(String subIndex)
306: throws SearchEngineException {
307: return openDirectory(defaultSubContext, subIndex);
308: }
309:
310: public Directory openDirectory(String subContext, String subIndex)
311: throws SearchEngineException {
312: Map<String, Directory> subContextDirs = dirs.get(subContext);
313: if (subContextDirs == null) {
314: subContextDirs = new ConcurrentHashMap<String, Directory>();
315: dirs.put(subContext, subContextDirs);
316: }
317: Directory dir = subContextDirs.get(subIndex);
318: if (dir != null) {
319: return dir;
320: }
321: synchronized (subContextDirs) {
322: dir = subContextDirs.get(subIndex);
323: if (dir != null) {
324: return dir;
325: }
326: dir = directoryStore.open(subContext, subIndex);
327: String lockFactoryType = settings
328: .getSetting(LuceneEnvironment.LockFactory.TYPE);
329: if (lockFactoryType != null) {
330: String path = settings
331: .getSetting(LuceneEnvironment.LockFactory.PATH);
332: if (path != null) {
333: path = StringUtils.replace(path, "#subindex#",
334: subIndex);
335: path = StringUtils.replace(path, "#subContext#",
336: subContext);
337: }
338: LockFactory lockFactory;
339: if (LuceneEnvironment.LockFactory.Type.NATIVE_FS
340: .equalsIgnoreCase(lockFactoryType)) {
341: String lockDir = path;
342: if (lockDir == null) {
343: lockDir = connectionString + "/" + subContext
344: + "/" + subIndex;
345: if (lockDir
346: .startsWith(FSDirectoryStore.PROTOCOL)) {
347: lockDir = lockDir
348: .substring(FSDirectoryStore.PROTOCOL
349: .length());
350: }
351: }
352: try {
353: lockFactory = new NativeFSLockFactory(lockDir);
354: } catch (IOException e) {
355: throw new SearchEngineException(
356: "Failed to create native fs lock factory with lock dir ["
357: + lockDir + "]", e);
358: }
359: if (log.isDebugEnabled()) {
360: log
361: .debug("Using native fs lock for sub index ["
362: + subIndex
363: + "] and lock directory ["
364: + lockDir + "]");
365: }
366: } else if (LuceneEnvironment.LockFactory.Type.SIMPLE_FS
367: .equalsIgnoreCase(lockFactoryType)) {
368: String lockDir = path;
369: if (lockDir == null) {
370: lockDir = connectionString + "/" + subContext
371: + "/" + subIndex;
372: if (lockDir
373: .startsWith(FSDirectoryStore.PROTOCOL)) {
374: lockDir = lockDir
375: .substring(FSDirectoryStore.PROTOCOL
376: .length());
377: }
378: }
379: try {
380: lockFactory = new SimpleFSLockFactory(lockDir);
381: } catch (IOException e) {
382: throw new SearchEngineException(
383: "Failed to create simple fs lock factory with lock dir ["
384: + lockDir + "]", e);
385: }
386: if (log.isDebugEnabled()) {
387: log
388: .debug("Using simple fs lock for sub index ["
389: + subIndex
390: + "] and lock directory ["
391: + lockDir + "]");
392: }
393:
394: } else if (LuceneEnvironment.LockFactory.Type.SINGLE_INSTANCE
395: .equalsIgnoreCase(lockFactoryType)) {
396: lockFactory = new SingleInstanceLockFactory();
397: } else if (LuceneEnvironment.LockFactory.Type.NO_LOCKING
398: .equalsIgnoreCase(lockFactoryType)) {
399: lockFactory = new NoLockFactory();
400: } else {
401: Object temp;
402: try {
403: temp = ClassUtils.forName(lockFactoryType,
404: settings.getClassLoader())
405: .newInstance();
406: } catch (Exception e) {
407: throw new SearchEngineException(
408: "Failed to create lock type ["
409: + lockFactoryType + "]", e);
410: }
411: if (temp instanceof LockFactory) {
412: lockFactory = (LockFactory) temp;
413: } else if (temp instanceof LockFactoryProvider) {
414: lockFactory = ((LockFactoryProvider) temp)
415: .createLockFactory(path, subIndex,
416: settings);
417: } else {
418: throw new SearchEngineException(
419: "No specific type of lock factory");
420: }
421:
422: if (lockFactory instanceof CompassConfigurable) {
423: ((CompassConfigurable) lockFactory)
424: .configure(settings);
425: }
426: }
427: dir.setLockFactory(lockFactory);
428: }
429: if (directoryWrapperProviders != null) {
430: for (DirectoryWrapperProvider directoryWrapperProvider : directoryWrapperProviders) {
431: dir = directoryWrapperProvider.wrap(subIndex, dir);
432: }
433: }
434: if (!closed) {
435: dir = localDirectoryCacheManager.createLocalCache(
436: subContext, subIndex, dir);
437: }
438: subContextDirs.put(subIndex, dir);
439: }
440: return dir;
441: }
442:
443: public boolean indexExists() throws SearchEngineException {
444: for (String subIndex : subIndexes) {
445: if (!indexExists(subIndex)) {
446: return false;
447: }
448: }
449: return true;
450: }
451:
452: public boolean indexExists(String subIndex)
453: throws SearchEngineException {
454: return indexExists(defaultSubContext, subIndex);
455: }
456:
457: public boolean indexExists(String subContext, String subIndex)
458: throws SearchEngineException {
459: boolean closeDir = !directoryExists(subContext, subIndex);
460: Directory dir = openDirectory(subContext, subIndex);
461: Boolean retVal = directoryStore.indexExists(unwrapDir(dir));
462: if (retVal != null) {
463: return retVal;
464: }
465: try {
466: retVal = IndexReader.indexExists(dir);
467: } catch (IOException e) {
468: return false;
469: }
470: if (closeDir) {
471: closeDirectory(dir, subContext, subIndex);
472: }
473: return retVal;
474: }
475:
476: public void createIndex() throws SearchEngineException {
477: for (String subIndex : subIndexes) {
478: createIndex(subIndex);
479: }
480: }
481:
482: public void createIndex(String subIndex)
483: throws SearchEngineException {
484: createIndex(defaultSubContext, subIndex);
485: }
486:
487: public void createIndex(String subContext, String subIndex)
488: throws SearchEngineException {
489: Directory dir = openDirectory(subContext, subIndex);
490: try {
491: IndexWriter indexWriter = new IndexWriter(dir,
492: new StandardAnalyzer(), true);
493: indexWriter.close();
494: } catch (IOException e) {
495: throw new SearchEngineException(
496: "Failed to create index for sub index [" + subIndex
497: + "]", e);
498: }
499: }
500:
501: public void deleteIndex() throws SearchEngineException {
502: for (String subIndex : subIndexes) {
503: deleteIndex(subIndex);
504: }
505: }
506:
507: public void deleteIndex(String subIndex)
508: throws SearchEngineException {
509: deleteIndex(defaultSubContext, subIndex);
510: }
511:
512: public void deleteIndex(String subContext, String subIndex)
513: throws SearchEngineException {
514: Directory dir = openDirectory(subContext, subIndex);
515: directoryStore
516: .deleteIndex(unwrapDir(dir), subContext, subIndex);
517: closeDirectory(dir, subContext, subIndex);
518: }
519:
520: public boolean verifyIndex() throws SearchEngineException {
521: boolean createdIndex = false;
522: for (String subIndex : subIndexes) {
523: if (verifyIndex(subIndex)) {
524: createdIndex = true;
525: }
526: }
527: return createdIndex;
528: }
529:
530: public boolean verifyIndex(String subIndex)
531: throws SearchEngineException {
532: return verifyIndex(defaultSubContext, subIndex);
533: }
534:
535: public boolean verifyIndex(String subContext, String subIndex)
536: throws SearchEngineException {
537: if (!indexExists(subContext, subIndex)) {
538: createIndex(subContext, subIndex);
539: return true;
540: }
541: return false;
542: }
543:
544: public void cleanIndex(String subIndex)
545: throws SearchEngineException {
546: cleanIndex(defaultSubContext, subIndex);
547: }
548:
549: public void cleanIndex(String subContext, String subIndex)
550: throws SearchEngineException {
551: Directory dir = directoryStore.open(subContext, subIndex);
552:
553: Directory unwrapDir = unwrapDir(dir);
554: directoryStore.cleanIndex(unwrapDir, subContext, subIndex);
555:
556: closeDirectory(dir, subContext, subIndex);
557: createIndex(subContext, subIndex);
558: }
559:
560: public boolean isLocked() throws SearchEngineException {
561: for (String subIndex : getSubIndexes()) {
562: if (isLocked(subIndex)) {
563: return true;
564: }
565: }
566: return false;
567: }
568:
569: public boolean isLocked(String subIndex)
570: throws SearchEngineException {
571: return isLocked(defaultSubContext, subIndex);
572: }
573:
574: public boolean isLocked(String subContext, String subIndex)
575: throws SearchEngineException {
576: try {
577: return IndexReader.isLocked(openDirectory(subContext,
578: subIndex));
579: } catch (IOException e) {
580: throw new SearchEngineException(
581: "Failed to check if index is locked for sub context ["
582: + subContext + "] and sub index ["
583: + subIndex + "]", e);
584: }
585: }
586:
587: public void releaseLocks() throws SearchEngineException {
588: for (String subIndex : subIndexes) {
589: releaseLock(subIndex);
590: }
591: }
592:
593: public void releaseLock(String subIndex)
594: throws SearchEngineException {
595: releaseLock(defaultSubContext, subIndex);
596: }
597:
598: public void releaseLock(String subContext, String subIndex)
599: throws SearchEngineException {
600: try {
601: IndexReader.unlock(openDirectory(subContext, subIndex));
602: } catch (IOException e) {
603: throw new SearchEngineException(
604: "Failed to unlock index for sub context ["
605: + subContext + "] and sub index ["
606: + subIndex + "]", e);
607: }
608: }
609:
610: public void copyFrom(String subIndex,
611: LuceneSearchEngineStore searchEngineStore)
612: throws SearchEngineException {
613: copyFrom(defaultSubContext, subIndex, searchEngineStore);
614: }
615:
616: public void copyFrom(String subContext, String subIndex,
617: LuceneSearchEngineStore searchEngineStore)
618: throws SearchEngineException {
619: // clear any possible wrappers
620: Directory dir = openDirectory(subContext, subIndex);
621: Directory unwrappedDir = unwrapDir(dir);
622: if (dir instanceof DirectoryWrapper) {
623: try {
624: ((DirectoryWrapper) dir).clearWrapper();
625: } catch (IOException e) {
626: throw new SearchEngineException(
627: "Failed to clear wrapper for sub index ["
628: + subIndex + "]", e);
629: }
630: }
631: CopyFromHolder holder = directoryStore.beforeCopyFrom(
632: subContext, subIndex, unwrappedDir);
633: final byte[] buffer = new byte[32768];
634: try {
635: Directory dest = openDirectory(subContext, subIndex);
636: // no need to pass the sub context to the given search engine store, it has its own sub context
637: Directory src = searchEngineStore.openDirectory(subIndex);
638: LuceneUtils.copy(src, dest, buffer);
639: // in case the index does not container anything, create an empty index
640: if (!IndexReader.indexExists(dest)) {
641: if (log.isDebugEnabled()) {
642: log
643: .debug("Copy From sub context ["
644: + subContext
645: + "] and sub index ["
646: + subIndex
647: + "] does not contain data, creating empty index");
648: }
649: IndexWriter writer = new IndexWriter(dest,
650: new StandardAnalyzer(), true);
651: writer.close();
652: }
653: } catch (Exception e) {
654: directoryStore.afterFailedCopyFrom(subContext, subIndex,
655: holder);
656: if (e instanceof SearchEngineException) {
657: throw (SearchEngineException) e;
658: }
659: throw new SearchEngineException("Failed to copy from "
660: + searchEngineStore, e);
661: }
662: directoryStore.afterSuccessfulCopyFrom(subContext, subIndex,
663: holder);
664: }
665:
666: public void registerEventListeners(SearchEngine searchEngine,
667: SearchEngineEventManager eventManager) {
668: directoryStore.registerEventListeners(searchEngine,
669: eventManager);
670: }
671:
672: public String getDefaultSubContext() {
673: return this .defaultSubContext;
674: }
675:
676: private boolean directoryExists(String subContext, String subIndex)
677: throws SearchEngineException {
678: Map<String, Directory> subContextDirs = dirs.get(subContext);
679: return subContextDirs != null
680: && subContextDirs.containsKey(subIndex);
681: }
682:
683: private void closeDirectory(Directory dir, String subContext,
684: String subIndex) throws SearchEngineException {
685: directoryStore.closeDirectory(dir, subContext, subIndex);
686: Map<String, Directory> subContextDirs = dirs.get(subContext);
687: if (subContextDirs != null) {
688: subContextDirs.remove(subIndex);
689: }
690: }
691:
692: private Directory unwrapDir(Directory dir) {
693: while (dir instanceof DirectoryWrapper) {
694: dir = ((DirectoryWrapper) dir).getWrappedDirectory();
695: }
696: return dir;
697: }
698:
699: public String toString() {
700: return "store [" + connectionString + "][" + defaultSubContext
701: + "] sub-indexes ["
702: + StringUtils.arrayToCommaDelimitedString(subIndexes)
703: + "]";
704: }
705: }
|