001: /*
002: * GeoTools - OpenSource mapping toolkit
003: * http://geotools.org
004: * (C) 2002-2006, Geotools Project Managment Committee (PMC)
005: * (C) 2002, Centre for Computational Geography
006: *
007: * This library is free software; you can redistribute it and/or
008: * modify it under the terms of the GNU Lesser General Public
009: * License as published by the Free Software Foundation; either
010: * version 2.1 of the License, or (at your option) any later version.
011: *
012: * This library is distributed in the hope that it will be useful,
013: * but WITHOUT ANY WARRANTY; without even the implied warranty of
014: * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
015: * Lesser General Public License for more details.
016: */
017: package org.geotools.data.shapefile.shp;
018:
019: import java.io.EOFException;
020: import java.io.FileInputStream;
021: import java.io.IOException;
022: import java.nio.ByteBuffer;
023: import java.nio.ByteOrder;
024: import java.nio.MappedByteBuffer;
025: import java.nio.channels.FileChannel;
026: import java.nio.channels.ReadableByteChannel;
027:
028: import org.geotools.data.DataSourceException;
029: import org.geotools.data.shapefile.Lock;
030: import org.geotools.data.shapefile.StreamLogging;
031: import org.geotools.resources.NIOUtilities;
032:
033: /**
034: * The general use of this class is: <CODE><PRE>
035: *
036: * FileChannel in = new FileInputStream("thefile.dbf").getChannel();
037: * ShapefileReader r = new ShapefileReader( in ) while (r.hasNext()) { Geometry
038: * shape = (Geometry) r.nextRecord().shape() // do stuff } r.close();
039: *
040: * </PRE></CODE> You don't have to immediately ask for the shape from the record. The
041: * record will contain the bounds of the shape and will only read the shape when
042: * the shape() method is called. This ShapefileReader.Record is the same object
043: * every time, so if you need data from the Record, be sure to copy it.
044: *
045: * @author jamesm
046: * @author aaime
047: * @author Ian Schneider
048: * @source $URL: http://svn.geotools.org/geotools/tags/2.4.1/modules/plugin/shapefile/src/main/java/org/geotools/data/shapefile/shp/ShapefileReader.java $
049: */
050: public class ShapefileReader {
051:
052: /**
053: * The reader returns only one Record instance in its lifetime. The record
054: * contains the current record information.
055: */
056: public final class Record {
057: int length;
058:
059: int number = 0;
060:
061: int offset; // Relative to the whole file
062:
063: int start = 0; // Relative to the current loaded buffer
064:
065: /** The minimum X value. */
066: public double minX;
067:
068: /** The minimum Y value. */
069: public double minY;
070:
071: /** The maximum X value. */
072: public double maxX;
073:
074: /** The maximum Y value. */
075: public double maxY;
076:
077: ShapeType type;
078:
079: int end = 0; // Relative to the whole file
080:
081: Object shape = null;
082:
083: /** Fetch the shape stored in this record. */
084: public Object shape() {
085: if (shape == null) {
086: buffer.position(start);
087: buffer.order(ByteOrder.LITTLE_ENDIAN);
088: shape = handler.read(buffer, type);
089: }
090: return shape;
091: }
092:
093: public int offset() {
094: return offset;
095: }
096:
097: /** A summary of the record. */
098: public String toString() {
099: return "Record " + number + " length " + length
100: + " bounds " + minX + "," + minY + " " + maxX + ","
101: + maxY;
102: }
103: }
104:
105: private ShapeHandler handler;
106:
107: private ShapefileHeader header;
108:
109: private ReadableByteChannel channel;
110:
111: ByteBuffer buffer;
112:
113: private ShapeType fileShapeType = ShapeType.UNDEFINED;
114:
115: private ByteBuffer headerTransfer;
116:
117: private final Record record = new Record();
118:
119: private final boolean randomAccessEnabled;
120:
121: private Lock lock;
122:
123: private boolean useMemoryMappedBuffer;
124:
125: private long currentOffset = 0L;
126: private StreamLogging streamLogger = new StreamLogging(
127: "Shapefile Reader");
128:
129: /**
130: * Creates a new instance of ShapeFile.
131: *
132: * @param channel
133: * The ReadableByteChannel this reader will use.
134: * @param strict
135: * True to make the header parsing throw Exceptions if the
136: * version or magic number are incorrect.
137: * @throws IOException
138: * If problems arise.
139: * @throws ShapefileException
140: * If for some reason the file contains invalid records.
141: */
142: public ShapefileReader(ReadableByteChannel channel, boolean strict,
143: boolean useMemoryMapped, Lock lock) throws IOException,
144: ShapefileException {
145: this .channel = channel;
146: this .useMemoryMappedBuffer = useMemoryMapped;
147: streamLogger.open();
148: randomAccessEnabled = channel instanceof FileChannel;
149: this .lock = lock;
150: lock.lockRead();
151: init(strict);
152: }
153:
154: /**
155: * Default constructor. Calls ShapefileReader(channel,true).
156: *
157: * @param channel
158: * @throws IOException
159: * @throws ShapefileException
160: */
161: public ShapefileReader(ReadableByteChannel channel, Lock lock)
162: throws IOException, ShapefileException {
163: this (channel, true, true, lock);
164: }
165:
166: // convenience to peak at a header
167: /**
168: * A short cut for reading the header from the given channel.
169: *
170: * @param channel
171: * The channel to read from.
172: * @param strict
173: * True to make the header parsing throw Exceptions if the
174: * version or magic number are incorrect.
175: * @throws IOException
176: * If problems arise.
177: * @return A ShapefileHeader object.
178: */
179: public static ShapefileHeader readHeader(
180: ReadableByteChannel channel, boolean strict)
181: throws IOException {
182: ByteBuffer buffer = ByteBuffer.allocateDirect(100);
183: if (fill(buffer, channel) == -1) {
184: throw new EOFException("Premature end of header");
185: }
186: buffer.flip();
187: ShapefileHeader header = new ShapefileHeader();
188: header.read(buffer, strict);
189: NIOUtilities.clean(buffer);
190: return header;
191: }
192:
193: // ensure the capacity of the buffer is of size by doubling the original
194: // capacity until it is big enough
195: // this may be naiive and result in out of MemoryError as implemented...
196: public static ByteBuffer ensureCapacity(ByteBuffer buffer,
197: int size, boolean useMemoryMappedBuffer) {
198: // This sucks if you accidentally pass is a MemoryMappedBuffer of size
199: // 80M
200: // like I did while messing around, within moments I had 1 gig of
201: // swap...
202: if (buffer.isReadOnly() || useMemoryMappedBuffer) {
203: return buffer;
204: }
205:
206: int limit = buffer.limit();
207: while (limit < size) {
208: limit *= 2;
209: }
210: if (limit != buffer.limit()) {
211: // if (record.ready) {
212: buffer = ByteBuffer.allocateDirect(limit);
213: // }
214: // else {
215: // throw new IllegalArgumentException("next before hasNext");
216: // }
217: }
218: return buffer;
219: }
220:
221: // for filling a ReadableByteChannel
222: public static int fill(ByteBuffer buffer,
223: ReadableByteChannel channel) throws IOException {
224: int r = buffer.remaining();
225: // channel reads return -1 when EOF or other error
226: // because they a non-blocking reads, 0 is a valid return value!!
227: while (buffer.remaining() > 0 && r != -1) {
228: r = channel.read(buffer);
229: }
230: if (r == -1) {
231: buffer.limit(buffer.position());
232: }
233: return r;
234: }
235:
236: private void init(boolean strict) throws IOException,
237: ShapefileException {
238: header = readHeader(channel, strict);
239: fileShapeType = header.getShapeType();
240: handler = fileShapeType.getShapeHandler();
241:
242: // recordHeader = ByteBuffer.allocateDirect(8);
243: // recordHeader.order(ByteOrder.BIG_ENDIAN);
244:
245: if (handler == null) {
246: throw new IOException("Unsuported shape type:"
247: + fileShapeType);
248: }
249:
250: if (channel instanceof FileChannel && useMemoryMappedBuffer) {
251: FileChannel fc = (FileChannel) channel;
252: buffer = fc
253: .map(FileChannel.MapMode.READ_ONLY, 0, fc.size());
254: buffer.position(100);
255: this .currentOffset = 0;
256: } else {
257: // force useMemoryMappedBuffer to false
258: this .useMemoryMappedBuffer = false;
259: // start with 8K buffer
260: buffer = ByteBuffer.allocateDirect(8 * 1024);
261: fill(buffer, channel);
262: buffer.flip();
263: this .currentOffset = 100;
264: }
265:
266: headerTransfer = ByteBuffer.allocate(8);
267: headerTransfer.order(ByteOrder.BIG_ENDIAN);
268:
269: // make sure the record end is set now...
270: record.end = this .toFileOffset(buffer.position());
271: }
272:
273: /**
274: * Get the header. Its parsed in the constructor.
275: *
276: * @return The header that is associated with this file.
277: */
278: public ShapefileHeader getHeader() {
279: return header;
280: }
281:
282: // do important cleanup stuff.
283: // Closes channel !
284: /**
285: * Clean up any resources. Closes the channel.
286: *
287: * @throws IOException
288: * If errors occur while closing the channel.
289: */
290: public void close() throws IOException {
291: lock.unlockRead();
292: if (channel.isOpen()) {
293: channel.close();
294: streamLogger.close();
295: }
296: if (buffer instanceof MappedByteBuffer) {
297: NIOUtilities.clean(buffer);
298: }
299: channel = null;
300: header = null;
301: }
302:
303: public boolean supportsRandomAccess() {
304: return randomAccessEnabled;
305: }
306:
307: /**
308: * If there exists another record. Currently checks the stream for the
309: * presence of 8 more bytes, the length of a record. If this is true and the
310: * record indicates the next logical record number, there exists more
311: * records.
312: *
313: * @throws IOException
314: * @return True if has next record, false otherwise.
315: */
316: public boolean hasNext() throws IOException {
317: return this .hasNext(true);
318: }
319:
320: /**
321: * If there exists another record. Currently checks the stream for the
322: * presence of 8 more bytes, the length of a record. If this is true and the
323: * record indicates the next logical record number (if checkRecord == true),
324: * there exists more records.
325: *
326: * @param checkRecno
327: * If true then record number is checked
328: * @throws IOException
329: * @return True if has next record, false otherwise.
330: */
331: private boolean hasNext(boolean checkRecno) throws IOException {
332: // mark current position
333: int position = buffer.position();
334:
335: // ensure the proper position, regardless of read or handler behavior
336: buffer.position(this .toBufferOffset(record.end));
337:
338: // no more data left
339: if (buffer.remaining() < 8)
340: return false;
341:
342: // looks good
343: boolean hasNext = true;
344: if (checkRecno) {
345: // record headers in big endian
346: buffer.order(ByteOrder.BIG_ENDIAN);
347: hasNext = buffer.getInt() == record.number + 1;
348: }
349:
350: // reset things to as they were
351: buffer.position(position);
352:
353: return hasNext;
354: }
355:
356: /**
357: * Transfer (by bytes) the data at the current record to the
358: * ShapefileWriter.
359: *
360: * @param bounds
361: * double array of length four for transfering the bounds into
362: * @return The length of the record transfered in bytes
363: */
364: public int transferTo(ShapefileWriter writer, int recordNum,
365: double[] bounds) throws IOException {
366:
367: buffer.position(this .toBufferOffset(record.end));
368: buffer.order(ByteOrder.BIG_ENDIAN);
369:
370: buffer.getInt(); // record number
371: int rl = buffer.getInt();
372: int mark = buffer.position();
373: int len = rl * 2;
374:
375: buffer.order(ByteOrder.LITTLE_ENDIAN);
376: ShapeType recordType = ShapeType.forID(buffer.getInt());
377:
378: if (recordType.isMultiPoint()) {
379: for (int i = 0; i < 4; i++) {
380: bounds[i] = buffer.getDouble();
381: }
382: } else if (recordType != ShapeType.NULL) {
383: bounds[0] = bounds[1] = buffer.getDouble();
384: bounds[2] = bounds[3] = buffer.getDouble();
385: }
386:
387: // write header to shp and shx
388: headerTransfer.position(0);
389: headerTransfer.putInt(recordNum).putInt(rl).position(0);
390: writer.shpChannel.write(headerTransfer);
391: headerTransfer.putInt(0, writer.offset).position(0);
392: writer.offset += rl + 4;
393: writer.shxChannel.write(headerTransfer);
394:
395: // reset to mark and limit at end of record, then write
396: buffer.position(mark).limit(mark + len);
397: writer.shpChannel.write(buffer);
398: buffer.limit(buffer.capacity());
399:
400: record.end = this .toFileOffset(buffer.position());
401: record.number++;
402:
403: return len;
404: }
405:
406: /**
407: * Fetch the next record information.
408: *
409: * @throws IOException
410: * @return The record instance associated with this reader.
411: */
412: public Record nextRecord() throws IOException {
413:
414: // need to update position
415: buffer.position(this .toBufferOffset(record.end));
416:
417: // record header is big endian
418: buffer.order(ByteOrder.BIG_ENDIAN);
419:
420: // read shape record header
421: int recordNumber = buffer.getInt();
422: // silly ESRI say contentLength is in 2-byte words
423: // and ByteByffer uses bytes.
424: // track the record location
425: int recordLength = buffer.getInt() * 2;
426:
427: if (!buffer.isReadOnly() && !useMemoryMappedBuffer) {
428: // capacity is less than required for the record
429: // copy the old into the newly allocated
430: if (buffer.capacity() < recordLength + 8) {
431: this .currentOffset += buffer.position();
432: ByteBuffer old = buffer;
433: // ensure enough capacity for one more record header
434: buffer = ensureCapacity(buffer, recordLength + 8,
435: useMemoryMappedBuffer);
436: buffer.put(old);
437: NIOUtilities.clean(old);
438: fill(buffer, channel);
439: buffer.position(0);
440: } else
441: // remaining is less than record length
442: // compact the remaining data and read again,
443: // allowing enough room for one more record header
444: if (buffer.remaining() < recordLength + 8) {
445: this .currentOffset += buffer.position();
446: buffer.compact();
447: fill(buffer, channel);
448: buffer.position(0);
449: }
450: }
451:
452: // shape record is all little endian
453: buffer.order(ByteOrder.LITTLE_ENDIAN);
454:
455: // read the type, handlers don't need it
456: ShapeType recordType = ShapeType.forID(buffer.getInt());
457:
458: // this usually happens if the handler logic is bunk,
459: // but bad files could exist as well...
460: if (recordType != ShapeType.NULL && recordType != fileShapeType) {
461: throw new IllegalStateException(
462: "ShapeType changed illegally from " + fileShapeType
463: + " to " + recordType);
464: }
465:
466: // peek at bounds, then reset for handler
467: // many handler's may ignore bounds reading, but we don't want to
468: // second guess them...
469: buffer.mark();
470: if (recordType.isMultiPoint()) {
471: record.minX = buffer.getDouble();
472: record.minY = buffer.getDouble();
473: record.maxX = buffer.getDouble();
474: record.maxY = buffer.getDouble();
475: } else if (recordType != ShapeType.NULL) {
476: record.minX = record.maxX = buffer.getDouble();
477: record.minY = record.maxY = buffer.getDouble();
478: }
479: buffer.reset();
480:
481: record.offset = record.end;
482: // update all the record info.
483: record.length = recordLength;
484: record.type = recordType;
485: record.number = recordNumber;
486: // remember, we read one int already...
487: record.end = this .toFileOffset(buffer.position())
488: + recordLength - 4;
489: // mark this position for the reader
490: record.start = buffer.position();
491: // clear any cached shape
492: record.shape = null;
493:
494: return record;
495: }
496:
497: /**
498: * Needs better data, what is the requirements for offset?
499: *
500: * @param offset
501: * @throws IOException
502: * @throws UnsupportedOperationException
503: */
504: public void goTo(int offset) throws IOException,
505: UnsupportedOperationException {
506: if (randomAccessEnabled) {
507: if (this .useMemoryMappedBuffer) {
508: buffer.position(offset);
509: } else {
510: /*
511: * Check to see if requested offset is already loaded; ensure
512: * that record header is in the buffer
513: */
514: if (this .currentOffset <= offset
515: && this .currentOffset + buffer.limit() >= offset + 8) {
516: buffer.position(this .toBufferOffset(offset));
517: } else {
518: FileChannel fc = (FileChannel) this .channel;
519: fc.position(offset);
520: this .currentOffset = offset;
521: buffer.position(0);
522: fill(buffer, fc);
523: buffer.position(0);
524: }
525: }
526:
527: int oldRecordOffset = record.end;
528: record.end = offset;
529: try {
530: hasNext(false); // don't check for next logical record equality
531: } catch (IOException ioe) {
532: record.end = oldRecordOffset;
533: throw ioe;
534: }
535: } else {
536: throw new UnsupportedOperationException(
537: "Random Access not enabled");
538: }
539: }
540:
541: /**
542: * TODO needs better java docs!!! What is offset?
543: *
544: * @param offset
545: * @throws IOException
546: * @throws UnsupportedOperationException
547: */
548: public Object shapeAt(int offset) throws IOException,
549: UnsupportedOperationException {
550: if (randomAccessEnabled) {
551: this .goTo(offset);
552: return nextRecord().shape();
553: }
554: throw new UnsupportedOperationException(
555: "Random Access not enabled");
556: }
557:
558: /**
559: * Sets the current location of the byteStream to offset and returns the
560: * next record. Usually used in conjuctions with the shx file or some other
561: * index file.
562: *
563: * @param offset
564: * If using an shx file the offset would be: 2 *
565: * (index.getOffset(i))
566: * @return The record after the offset location in the bytestream
567: * @throws IOException
568: * thrown in a read error occurs
569: * @throws UnsupportedOperationException
570: * thrown if not a random access file
571: */
572:
573: public Record recordAt(int offset) throws IOException,
574: UnsupportedOperationException {
575: if (randomAccessEnabled) {
576: this .goTo(offset);
577: return nextRecord();
578: }
579: throw new UnsupportedOperationException(
580: "Random Access not enabled");
581: }
582:
583: /**
584: * Converts file offset to buffer offset
585: *
586: * @param offset
587: * The offset relative to the whole file
588: * @return The offset relative to the current loaded portion of the file
589: */
590: private int toBufferOffset(int offset) {
591: return (int) (offset - this .currentOffset);
592: }
593:
594: /**
595: * Converts buffer offset to file offset
596: *
597: * @param offset
598: * The offset relative to the buffer
599: * @return The offset relative to the whole file
600: */
601: private int toFileOffset(int offset) {
602: return (int) (this .currentOffset + offset);
603: }
604:
605: /**
606: * Parses the shpfile counting the records.
607: *
608: * @return the number of non-null records in the shapefile
609: */
610: public int getCount(int count) throws DataSourceException {
611: try {
612: if (channel == null)
613: return -1;
614: count = 0;
615:
616: for (int tmp = readRecord(); tmp != -1; tmp = readRecord())
617: count += tmp;
618:
619: } catch (IOException ioe) {
620: count = -1;
621: // What now? This seems arbitrarily appropriate !
622: throw new DataSourceException(
623: "Problem reading shapefile record", ioe);
624: }
625: return count;
626: }
627:
628: /**
629: * Reads a record and returns 1 if the record is not null.
630: *
631: * @param channel
632: * the io channel
633: * @param buffer
634: * @return 0 if null feature; 1 if valid feature; -1 if end of file reached.
635: * @throws IOException
636: */
637: private int readRecord() throws IOException {
638: if (!fillBuffer())
639: return -1;
640: // burn the record number
641: buffer.getInt();
642: if (!fillBuffer())
643: return -1;
644: int recordlength = buffer.getInt() * 2;
645: // Going to read the first 4 bytes of the record so
646: // subtract that from the record length
647: recordlength -= 4;
648: if (!fillBuffer())
649: return -1;
650:
651: // read record type (used to determine if record is a null record)
652: int type = buffer.getInt();
653: // go to end of record
654: while (buffer.limit() < buffer.position() + recordlength) {
655: recordlength -= buffer.limit() - buffer.position();
656: buffer.clear();
657: if (channel.read(buffer) < 1) {
658: return -1;
659: }
660: }
661: buffer.position(buffer.position() + recordlength);
662:
663: // return 0 if record is null. Null records should be counted.
664: if (type == 0) {
665: // this is a null feature
666: return 0;
667: }
668: return 1;
669: }
670:
671: /**
672: * Ensures that there is at least 1 integer (4 bytes) is in the buffer.
673: *
674: * @return true if there is data in the buffer, false less than a byte is in
675: * the buffer.
676: * @throws IOException
677: * if exception during reading occurs.
678: */
679: private boolean fillBuffer() throws IOException {
680: int result = 1;
681: if (buffer.limit() <= buffer.position() + 4) {
682: result = fill(buffer, channel);
683: }
684: return result > 0;
685: }
686:
687: public static void main(String[] args) throws Exception {
688: FileChannel channel = new FileInputStream(args[0]).getChannel();
689: ShapefileReader reader = new ShapefileReader(channel,
690: new Lock());
691: System.out.println(reader.getHeader());
692: while (reader.hasNext()) {
693: System.out.println(reader.nextRecord().shape());
694: }
695: reader.close();
696: }
697:
698: /**
699: * @param handler
700: * The handler to set.
701: */
702: public void setHandler(ShapeHandler handler) {
703: this.handler = handler;
704: }
705: }
|