001: package org.apache.lucene;
002:
003: // Intentionally not in org.apache.lucene.index, to assert
004: // that we do not require any package private access.
005:
006: /**
007: * Licensed to the Apache Software Foundation (ASF) under one or more
008: * contributor license agreements. See the NOTICE file distributed with
009: * this work for additional information regarding copyright ownership.
010: * The ASF licenses this file to You under the Apache License, Version 2.0
011: * (the "License"); you may not use this file except in compliance with
012: * the License. You may obtain a copy of the License at
013: *
014: * http://www.apache.org/licenses/LICENSE-2.0
015: *
016: * Unless required by applicable law or agreed to in writing, software
017: * distributed under the License is distributed on an "AS IS" BASIS,
018: * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
019: * See the License for the specific language governing permissions and
020: * limitations under the License.
021: */
022:
023: import java.util.Iterator;
024: import java.util.Collection;
025: import java.io.File;
026: import java.io.IOException;
027:
028: import org.apache.lucene.document.Document;
029: import org.apache.lucene.document.Field;
030: import org.apache.lucene.store.Directory;
031: import org.apache.lucene.store.FSDirectory;
032: import org.apache.lucene.store.IndexInput;
033: import org.apache.lucene.store.MockRAMDirectory;
034: import org.apache.lucene.analysis.standard.StandardAnalyzer;
035: import org.apache.lucene.index.IndexCommitPoint;
036: import org.apache.lucene.index.KeepOnlyLastCommitDeletionPolicy;
037: import org.apache.lucene.index.IndexWriter;
038: import org.apache.lucene.index.TestIndexWriter;
039: import org.apache.lucene.index.SnapshotDeletionPolicy;
040:
041: import org.apache.lucene.util.LuceneTestCase;
042: import org.apache.lucene.util._TestUtil;
043:
044: //
045: // This was developed for Lucene In Action,
046: // http://lucenebook.com
047: //
048:
049: public class TestSnapshotDeletionPolicy extends LuceneTestCase {
050: public static final String INDEX_PATH = "test.snapshots";
051:
052: public void testSnapshotDeletionPolicy() throws IOException {
053: File dir = new File(System.getProperty("tempDir"), INDEX_PATH);
054: try {
055: Directory fsDir = FSDirectory.getDirectory(dir);
056: runTest(fsDir);
057: fsDir.close();
058: } finally {
059: _TestUtil.rmDir(dir);
060: }
061:
062: MockRAMDirectory dir2 = new MockRAMDirectory();
063: runTest(dir2);
064: }
065:
066: private void runTest(Directory dir) throws IOException {
067: // Run for ~7 seconds
068: final long stopTime = System.currentTimeMillis() + 7000;
069:
070: SnapshotDeletionPolicy dp = new SnapshotDeletionPolicy(
071: new KeepOnlyLastCommitDeletionPolicy());
072: final IndexWriter writer = new IndexWriter(dir, true,
073: new StandardAnalyzer(), dp);
074:
075: // Force frequent commits
076: writer.setMaxBufferedDocs(2);
077:
078: final Thread t = new Thread() {
079: public void run() {
080: Document doc = new Document();
081: doc.add(new Field("content", "aaa", Field.Store.YES,
082: Field.Index.TOKENIZED,
083: Field.TermVector.WITH_POSITIONS_OFFSETS));
084: while (System.currentTimeMillis() < stopTime) {
085: for (int i = 0; i < 27; i++) {
086: try {
087: writer.addDocument(doc);
088: } catch (IOException cie) {
089: RuntimeException re = new RuntimeException(
090: "addDocument failed");
091: re.initCause(cie);
092: throw re;
093: }
094: }
095: try {
096: Thread.sleep(1);
097: } catch (InterruptedException ie) {
098: Thread.currentThread().interrupt();
099: }
100: }
101: }
102: };
103:
104: t.start();
105:
106: // While the above indexing thread is running, take many
107: // backups:
108: while (System.currentTimeMillis() < stopTime) {
109: backupIndex(dir, dp);
110: try {
111: Thread.sleep(20);
112: } catch (InterruptedException ie) {
113: Thread.currentThread().interrupt();
114: }
115: if (!t.isAlive())
116: break;
117: }
118:
119: try {
120: t.join();
121: } catch (InterruptedException ie) {
122: Thread.currentThread().interrupt();
123: }
124:
125: // Add one more document to force writer to commit a
126: // final segment, so deletion policy has a chance to
127: // delete again:
128: Document doc = new Document();
129: doc.add(new Field("content", "aaa", Field.Store.YES,
130: Field.Index.TOKENIZED,
131: Field.TermVector.WITH_POSITIONS_OFFSETS));
132: writer.addDocument(doc);
133:
134: // Make sure we don't have any leftover files in the
135: // directory:
136: writer.close();
137: TestIndexWriter.assertNoUnreferencedFiles(dir,
138: "some files were not deleted but should have been");
139: }
140:
141: /** Example showing how to use the SnapshotDeletionPolicy
142: * to take a backup. This method does not really do a
143: * backup; instead, it reads every byte of every file
144: * just to test that the files indeed exist and are
145: * readable even while the index is changing. */
146: public void backupIndex(Directory dir, SnapshotDeletionPolicy dp)
147: throws IOException {
148:
149: // To backup an index we first take a snapshot:
150: IndexCommitPoint cp = dp.snapshot();
151: try {
152:
153: // While we hold the snapshot, and nomatter how long
154: // we take to do the backup, the IndexWriter will
155: // never delete the files in the snapshot:
156: Collection files = cp.getFileNames();
157: Iterator it = files.iterator();
158: while (it.hasNext()) {
159: final String fileName = (String) it.next();
160: // NOTE: in a real backup you would not use
161: // readFile; you would need to use something else
162: // that copies the file to a backup location. This
163: // could even be a spawned shell process (eg "tar",
164: // "zip") that takes the list of files and builds a
165: // backup.
166: readFile(dir, fileName);
167: }
168:
169: } finally {
170: // Make sure to release the snapshot, otherwise these
171: // files will never be deleted during this IndexWriter
172: // session:
173: dp.release();
174: }
175: }
176:
177: byte[] buffer = new byte[4096];
178:
179: private void readFile(Directory dir, String name)
180: throws IOException {
181: IndexInput input = dir.openInput(name);
182: try {
183: long size = dir.fileLength(name);
184: long bytesLeft = size;
185: while (bytesLeft > 0) {
186: final int numToRead;
187: if (bytesLeft < buffer.length)
188: numToRead = (int) bytesLeft;
189: else
190: numToRead = buffer.length;
191: input.readBytes(buffer, 0, numToRead, false);
192: bytesLeft -= numToRead;
193: }
194: // Don't do this in your real backups! This is just
195: // to force a backup to take a somewhat long time, to
196: // make sure we are exercising the fact that the
197: // IndexWriter should not delete this file even when I
198: // take my time reading it.
199: try {
200: Thread.sleep(1);
201: } catch (InterruptedException ie) {
202: Thread.currentThread().interrupt();
203: }
204: } finally {
205: input.close();
206: }
207: }
208: }
|