001: /*
002:
003: Derby - Class org.apache.derby.impl.store.access.sort.MergeInserter
004:
005: Licensed to the Apache Software Foundation (ASF) under one or more
006: contributor license agreements. See the NOTICE file distributed with
007: this work for additional information regarding copyright ownership.
008: The ASF licenses this file to you under the Apache License, Version 2.0
009: (the "License"); you may not use this file except in compliance with
010: the License. You may obtain a copy of the License at
011:
012: http://www.apache.org/licenses/LICENSE-2.0
013:
014: Unless required by applicable law or agreed to in writing, software
015: distributed under the License is distributed on an "AS IS" BASIS,
016: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
017: See the License for the specific language governing permissions and
018: limitations under the License.
019:
020: */
021:
022: package org.apache.derby.impl.store.access.sort;
023:
024: import java.util.Vector;
025: import org.apache.derby.iapi.services.sanity.SanityManager;
026: import org.apache.derby.iapi.services.io.Storable;
027: import org.apache.derby.iapi.error.StandardException;
028: import org.apache.derby.iapi.store.access.conglomerate.TransactionManager;
029: import org.apache.derby.iapi.store.access.ColumnOrdering;
030: import org.apache.derby.iapi.store.access.ConglomerateController;
031: import org.apache.derby.iapi.store.access.Qualifier;
032: import org.apache.derby.iapi.store.access.ScanController;
033: import org.apache.derby.iapi.store.access.SortController;
034: import org.apache.derby.iapi.store.access.SortInfo;
035: import org.apache.derby.iapi.store.access.TransactionController;
036:
037: import org.apache.derby.iapi.types.DataValueDescriptor;
038:
039: import org.apache.derby.iapi.types.RowLocation;
040:
041: /**
042:
043:
044: **/
045:
046: public final class MergeInserter implements SortController {
047: /**
048: The sort this inserter is for.
049: **/
050: protected MergeSort sort = null;
051:
052: /**
053: The transaction this inserter is in.
054: **/
055: protected TransactionManager tran;
056:
057: /**
058: A vector of the conglomerate ids of the merge runs.
059: **/
060: Vector mergeRuns = null;
061:
062: /**
063: An in-memory ordered set that is used to sort rows
064: before they're sent to merge runs.
065: **/
066: SortBuffer sortBuffer = null;
067:
068: /**
069: Information about memory usage to dynamically tune the
070: in-memory sort buffer size.
071: */
072: long beginFreeMemory;
073: long beginTotalMemory;
074: long estimatedMemoryUsed;
075: boolean avoidMergeRun; // try to avoid merge run if possible
076: int runSize;
077: int totalRunSize;
078:
079: protected String stat_sortType;
080: protected int stat_numRowsInput;
081: protected int stat_numRowsOutput;
082: protected int stat_numMergeRuns;
083: protected Vector stat_mergeRunsSize;
084:
085: /*
086: * Methods of SortController
087: */
088:
089: /**
090: Insert a row into the sort.
091: @see SortController#insert
092: **/
093: public void insert(DataValueDescriptor[] row)
094: throws StandardException {
095: if (SanityManager.DEBUG) {
096: // If the sort is null, probably the caller forgot
097: // to call initialize.
098: SanityManager.ASSERT(sort != null);
099: }
100:
101: // Check that the inserted row is of the correct type
102: sort.checkColumnTypes(row);
103:
104: // Insert the row into the sort buffer, which will
105: // sort it into the right order with the rest of the
106: // rows and remove any duplicates.
107: int insertResult = sortBuffer.insert(row);
108: stat_numRowsInput++;
109: if (insertResult != SortBuffer.INSERT_DUPLICATE)
110: stat_numRowsOutput++;
111: if (insertResult == SortBuffer.INSERT_FULL) {
112: if (avoidMergeRun) {
113: Runtime jvm = Runtime.getRuntime();
114: if (SanityManager.DEBUG) {
115: if (SanityManager.DEBUG_ON("SortTuning")) {
116: jvm.gc();
117: jvm.gc();
118: jvm.gc();
119: }
120: }
121:
122: long currentFreeMemory = jvm.freeMemory();
123: long currentTotalMemory = jvm.totalMemory();
124:
125: // before we create an external sort, which is expensive, see if
126: // we can use up more in-memory sort buffer
127: // we see how much memory has been used between now and the
128: // beginning of the sort. Not all of this memory is used by
129: // the sort and GC may have kicked in and release some memory.
130: // But it is a rough guess.
131: estimatedMemoryUsed = (currentTotalMemory - currentFreeMemory)
132: - (beginTotalMemory - beginFreeMemory);
133:
134: if (SanityManager.DEBUG) {
135: if (SanityManager.DEBUG_ON("SortTuning")) {
136: SanityManager
137: .DEBUG(
138: "SortTuning",
139: "Growing sortBuffer dynamically, "
140: + "current sortBuffer capacity= "
141: + sortBuffer.capacity()
142: + " estimatedMemoryUsed = "
143: + estimatedMemoryUsed
144: + " currentTotalMemory = "
145: + currentTotalMemory
146: + " currentFreeMemory = "
147: + currentFreeMemory
148: + " numcolumn = "
149: + row.length
150: + " real per row memory = "
151: + (estimatedMemoryUsed / sortBuffer
152: .capacity()));
153: }
154: }
155:
156: // we want to double the sort buffer size if that will result
157: // in the sort to use up no more than 1/2 of all the free
158: // memory (including the sort memory)
159: // or if GC is so effective we are now using less memory than before
160: // or if we are using less than 1Meg of memory and the jvm is
161: // using < 5 meg of memory (this indicates that the JVM can
162: // afford to be more bloated ?)
163: if (estimatedMemoryUsed < 0
164: || ((2 * estimatedMemoryUsed) < (estimatedMemoryUsed + currentFreeMemory) / 2)
165: || (2 * estimatedMemoryUsed < ExternalSortFactory.DEFAULT_MEM_USE && currentTotalMemory < (5 * 1024 * 1024))) {
166: // ok, double the sort buffer size
167: sortBuffer.grow(100);
168:
169: if (sortBuffer.insert(row) != SortBuffer.INSERT_FULL)
170: return;
171: }
172:
173: avoidMergeRun = false; // once we did it, too late to do in
174: // memory sort
175: }
176:
177: // The sort buffer became full. Empty it into a
178: // merge run, and add the merge run to the vector
179: // of merge runs.
180: stat_sortType = "external";
181: long conglomid = sort.createMergeRun(tran, sortBuffer);
182: if (mergeRuns == null)
183: mergeRuns = new Vector();
184: mergeRuns.addElement(new Long(conglomid));
185:
186: stat_numMergeRuns++;
187: // calculate size of this merge run
188: // buffer was too full for last row
189: runSize = stat_numRowsInput - totalRunSize - 1;
190: totalRunSize += runSize;
191: stat_mergeRunsSize.addElement(new Integer(runSize));
192:
193: // Re-insert the row into the sort buffer.
194: // This is guaranteed to work since the sort
195: // buffer has just been emptied.
196: sortBuffer.insert(row);
197: }
198: }
199:
200: /**
201: Close this sort controller. Closing the sort controller
202: means the caller is done inserting rows. This method
203: must not throw any exceptions since it's called during
204: error processing.
205:
206: @see SortController#close
207: **/
208:
209: public void close() {
210: // Tell the sort that we're closed, and hand off
211: // the sort buffer and the vector of merge runs.
212: if (sort != null)
213: sort.doneInserting(this , sortBuffer, mergeRuns);
214:
215: // if this is an external sort, there will actually
216: // be one last merge run with the contents of the
217: // current sortBuffer. It will be created when the user
218: // reads the result of the sort using openSortScan
219: if (stat_sortType == "external") {
220: stat_numMergeRuns++;
221: stat_mergeRunsSize.addElement(new Integer(stat_numRowsInput
222: - totalRunSize));
223: }
224:
225: // close the SortController in the transaction.
226: tran.closeMe(this );
227:
228: // Clean up.
229: sort = null;
230: tran = null;
231: mergeRuns = null;
232: sortBuffer = null;
233: }
234:
235: /*
236: * Methods of MergeInserter. Arranged alphabetically.
237: */
238:
239: /**
240: * Return SortInfo object which contains information about the current
241: * sort.
242: * <p>
243: *
244: * @see SortInfo
245: *
246: * @return The SortInfo object which contains info about current sort.
247: *
248: * @exception StandardException Standard exception policy.
249: **/
250: public SortInfo getSortInfo() throws StandardException {
251: return (new MergeSortInfo(this ));
252: }
253:
254: /**
255: Initialize this inserter.
256: @return true if initialization was successful
257: **/
258: boolean initialize(MergeSort sort, TransactionManager tran) {
259: Runtime jvm = Runtime.getRuntime();
260: if (SanityManager.DEBUG) {
261: if (SanityManager.DEBUG_ON("SortTuning")) {
262: jvm.gc();
263: jvm.gc();
264: jvm.gc();
265: }
266: }
267:
268: beginFreeMemory = jvm.freeMemory();
269: beginTotalMemory = jvm.totalMemory();
270: estimatedMemoryUsed = 0;
271: avoidMergeRun = true; // not an external sort
272: stat_sortType = "internal";
273: stat_numMergeRuns = 0;
274: stat_numRowsInput = 0;
275: stat_numRowsOutput = 0;
276: stat_mergeRunsSize = new Vector();
277: runSize = 0;
278: totalRunSize = 0;
279:
280: if (SanityManager.DEBUG) {
281: if (SanityManager.DEBUG_ON("testSort")) {
282: avoidMergeRun = false;
283: }
284: }
285:
286: this .sort = sort;
287: this .tran = tran;
288: sortBuffer = new SortBuffer(sort);
289: if (sortBuffer.init() == false)
290: return false;
291: return true;
292: }
293:
294: }
|