001: /*
002: This source file is part of Smyle, a database library.
003: For up-to-date information, see http://www.drjava.de/smyle
004: Copyright (C) 2001 Stefan Reich (doc@drjava.de)
005:
006: This library is free software; you can redistribute it and/or
007: modify it under the terms of the GNU Lesser General Public
008: License as published by the Free Software Foundation; either
009: version 2.1 of the License, or (at your option) any later version.
010:
011: This library is distributed in the hope that it will be useful,
012: but WITHOUT ANY WARRANTY; without even the implied warranty of
013: MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
014: Lesser General Public License for more details.
015:
016: You should have received a copy of the GNU Lesser General Public
017: License along with this library; if not, write to the Free Software
018: Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
019:
020: For full license text, see doc/license/lgpl.txt in this distribution
021: */
022:
023: package drjava.smyle.tests;
024:
025: import java.io.*;
026: import java.util.*;
027: import java.util.zip.*;
028: import drjava.util.*;
029: import drjava.smyle.*;
030: import drjava.smyle.core.*;
031:
032: public class CompressionBench extends Benchmark {
033: Disk disk;
034: DefaultChunkManager chunkManager;
035: ArrayList<byte[]> chunks = new ArrayList<byte[]>();
036: int totalBytes, totalCompressed;
037: byte[] chunk;
038: int l; // length of chunk
039: int[] optimal, used, power, saved;
040:
041: public CompressionBench() {
042: disk = new FileSystemDisk(new File("etc/compression"), true);
043: //System.out.println("Master: "+disk.getMasterFile());
044: chunkManager = new DefaultChunkManager(disk, 281);
045: chunkManager.getMasterChunk(); // needed to read chunk table
046:
047: for (Iterator<ChunkRef> i = chunkManager.chunks1(); i.hasNext();) {
048: byte[] b = chunkManager.readChunk(i.next()).toByteArray();
049: chunks.add(b);
050: totalBytes += b.length;
051: }
052: disk.release();
053:
054: setDescription("Compress " + chunks.size() + " chunks ("
055: + totalBytes + " bytes)");
056: }
057:
058: int deflate() {
059: byte[] buf = new byte[l + 512];
060: Deflater deflater = new Deflater();
061: deflater.setInput(chunk);
062: deflater.finish();
063: int compressed = deflater.deflate(buf);
064: if (compressed >= buf.length)
065: throw new RuntimeException("Buffer overflow");
066: return compressed;
067: }
068:
069: int gzip() {
070: try {
071: ByteArrayOutputStream baos = new ByteArrayOutputStream();
072: GZIPOutputStream out = new GZIPOutputStream(baos);
073: out.write(chunk);
074: out.close();
075: return baos.size();
076: } catch (IOException e) {
077: throw new RuntimeException(e.toString());
078: }
079: }
080:
081: int noCompression() {
082: return l;
083: }
084:
085: /** looks for sequences of zeros and non-zeros */
086: int countZeros() {
087: int n = 0;
088: for (int i = 0; i < l;) {
089: int zeros = 0, nonzeros = 0;
090: while (zeros < 3 && i + zeros < l && chunk[i + zeros] == 0)
091: ++zeros;
092: i += zeros;
093:
094: // basic (stop at first zero)
095: while (nonzeros < 63 && i + nonzeros < l
096: && chunk[i + nonzeros] != 0)
097: ++nonzeros;
098:
099: // advanced (skip one zero)
100: /*boolean z = false;
101: while (nonzeros < 63 && i+nonzeros < l)
102: if (chunk[i+nonzeros] != 0) ++nonzeros;
103: else if (z) break; // found 2nd zero
104: else z = true; // found 1st zero
105: // push back first zero if the two zeros were consecutive
106: // and we're not at the end of the stream
107: if (nonzeros > 0 && i+nonzeros < l && chunk[i+nonzeros-1] == 0) --nonzeros;*/
108:
109: i += nonzeros;
110: n += 1 + nonzeros;
111: }
112: return n;
113: }
114:
115: /*int highestBit() {
116: int bits = 0;
117: for (int i = 0; i < l; i++) {
118: bits += chunk[i] < 0 ? 9 : 7;
119: }
120: return (bits+7)/8;
121: }*/
122:
123: int diff(int diffBits) {
124: int bits = 8;
125: for (int i = 1; i < l; i++) {
126: int d = chunk[i] - chunk[i - 1];
127: bits += (d == (d & ((1 << diffBits) - 1))) ? 1 + diffBits
128: : 1 + 8;
129: }
130: return (bits + 7) / 8;
131: }
132:
133: /** combines one or more algorithms and chooses the best method
134: for each chunk (including no compression).
135: needs an additional byte to record which method was chosen */
136: int chooseBest(int[] results) {
137: if (optimal == null) {
138: optimal = new int[results.length];
139: used = new int[results.length];
140: power = new int[results.length];
141: saved = new int[results.length];
142: }
143: int best = Integer.MAX_VALUE;
144: for (int i = 0; i < results.length; i++)
145: if (results[i] < best)
146: best = results[i];
147: for (int i = 0; i < results.length; i++)
148: if (results[i] == best) {
149: ++optimal[i];
150: power[i] += l - best - 1;
151: }
152: for (int i = 0; i < results.length; i++)
153: if (results[i] == best) {
154: ++used[i];
155: saved[i] += l - best - 1;
156: break;
157: }
158: return best + 1;
159: }
160:
161: protected void action() {
162: totalCompressed = 0;
163: optimal = used = saved = null;
164: for (int i = 0; i < chunks.size(); i++) {
165: chunk = chunks.get(i);
166: l = chunk.length;
167:
168: int compressed = chooseBest(new int[] {
169: noCompression(),
170: Compression.estimateHighNibble(chunk, 0, 2, 6),
171: ///highNibble(0, 2, 7),
172: Compression.estimateHighNibble(chunk, 0, 6, 7),
173: Compression.estimateHighNibble(chunk, 2, 6, 7),
174: Compression.estimateZeros(chunk),
175: Compression.estimateDoubleZeros(chunk),
176: //countZeros(),
177: //highestBit(),
178: //diff(1),
179: //diff(2),
180: //diff(3),
181: Compression.estimateXOR(chunk, 1),
182: //xor(2),
183: //xor(3),
184: //xor(4),
185: Compression.estimateXOR(chunk, 5),
186: ///deflate(),
187: ///gzip(), // it seems gzip is always worse than deflate
188: });
189:
190: //System.out.println(chunk.length+" -> "+compressed);
191: totalCompressed += compressed;
192: }
193: }
194:
195: void printInfo() {
196: System.out.println("Compressed size: " + totalCompressed + " ("
197: + (totalCompressed * 1000L / totalBytes) * 0.1f + "%)");
198: for (int i = 0; i < optimal.length; i++)
199: System.out.println(optimal[i] + " " + used[i] + " " + " "
200: + power[i] + " " + saved[i]);
201: }
202:
203: public static void main(String[] args) {
204: CompressionBench bench = new CompressionBench();
205: for (int i = 0; i < 1; i++)
206: bench.runAndPrint();
207: bench.printInfo();
208: //System.out.println("Records/s: "+(long) (n/(bench.totalTime()*0.001)));
209: //System.gc(); // for memprofile; store is closed, but still referenced
210: }
211: }
|