diff --git a/pom.xml b/pom.xml index 439ddd0..aa6c575 100644 --- a/pom.xml +++ b/pom.xml @@ -37,6 +37,11 @@ + + it.unimi.dsi + fastutil + 8.5.15 + org.junit.jupiter junit-jupiter diff --git a/src/main/java/com/maxmind/db/BiInt2IntFunction.java b/src/main/java/com/maxmind/db/BiInt2IntFunction.java new file mode 100644 index 0000000..fa127e1 --- /dev/null +++ b/src/main/java/com/maxmind/db/BiInt2IntFunction.java @@ -0,0 +1,15 @@ +package com.maxmind.db; + +import java.util.function.ToIntBiFunction; + +@FunctionalInterface +interface BiInt2IntFunction extends ToIntBiFunction { + + @Override + default int applyAsInt(Integer integer, Integer integer2) { + return apply(integer, integer); + } + + int apply(int integer0, int integer1); + +} diff --git a/src/main/java/com/maxmind/db/BigByteBuffer.java b/src/main/java/com/maxmind/db/BigByteBuffer.java new file mode 100644 index 0000000..8439221 --- /dev/null +++ b/src/main/java/com/maxmind/db/BigByteBuffer.java @@ -0,0 +1,298 @@ +package com.maxmind.db; + +import it.unimi.dsi.fastutil.bytes.ByteBigList; +import it.unimi.dsi.fastutil.ints.Int2IntFunction; +import it.unimi.dsi.fastutil.longs.Long2LongFunction; +import java.nio.BufferUnderflowException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.Objects; + +/** + * This provides minimal functionality from {@link ByteBuffer} that is required for this library, + * but it is modified to work with files larger than Java's normal memory mapped file size limit. + */ +final class BigByteBuffer { + + public static final boolean NATIVE_IS_BIG_ENDIAN = + ByteOrder.nativeOrder() == ByteOrder.BIG_ENDIAN; + + public static BigByteBuffer wrap(ByteBigList bytes) { + return new BigByteBuffer(bytes, 0, bytes.size64(), bytes.size64()); + } + + // copied from Unsafe.java + private static long toUnsignedLong(byte n) { + return n & 0xffL; + } + + // copied from Unsafe.java + private static int toUnsignedInt(byte n) { + return n & 0xff; + } + + private static final BiInt2IntFunction PICK_POS = + NATIVE_IS_BIG_ENDIAN + ? (int top, int pos) -> top - pos + : (int top, int pos) -> pos; + + private static final Int2IntFunction CONV_ENDIAN_INT = + NATIVE_IS_BIG_ENDIAN + ? Int2IntFunction.identity() + : Integer::reverseBytes; + + private static final Long2LongFunction CONV_ENDIAN_LONG = + NATIVE_IS_BIG_ENDIAN + ? Long2LongFunction.identity() + : Long::reverseBytes; + + final ByteBigList bytes; + final long capacity; + long position; + long limit; + + private BigByteBuffer( + ByteBigList bytes, + long position, + long limit, + long capacity + ) { + this.bytes = bytes; + this.capacity = capacity; + this.limit = limit; + this.position = position; + } + + /** + * {@link ByteBuffer#get()} + */ + public byte get() { + return bytes.getByte(nextGetIndex(1)); + } + + /** + * {@link ByteBuffer#get(int)} + */ + public byte get(long index) { + return bytes.getByte(checkIndex(index)); + } + + /** + * {@link ByteBuffer#get(byte[])} + */ + public BigByteBuffer get(byte[] dst) { + return get(dst, 0, dst.length); + } + + /** + * {@link ByteBuffer#get(byte[], int, int)} )} + */ + public BigByteBuffer get(byte[] dst, int offset, int length) { + Objects.checkFromIndexSize(offset, length, dst.length); + long pos = position(); + if (length > limit() - pos) { + throw new BufferUnderflowException(); + } + + this.bytes.getElements(pos, dst, offset, length); + + position(pos + length); + return this; + } + + public long getLong() { + long index = nextGetIndex(8); + byte i0 = get(index); + byte i1 = get(++index); + byte i2 = get(++index); + byte i3 = get(++index); + byte i4 = get(++index); + byte i5 = get(++index); + byte i6 = get(++index); + byte i7 = get(++index); + return CONV_ENDIAN_LONG.applyAsLong(((toUnsignedLong(i0) << PICK_POS.apply(56, 0)) + | (toUnsignedLong(i1) << PICK_POS.apply(56, 8)) + | (toUnsignedLong(i2) << PICK_POS.apply(56, 16)) + | (toUnsignedLong(i3) << PICK_POS.apply(56, 24)) + | (toUnsignedLong(i4) << PICK_POS.apply(56, 32)) + | (toUnsignedLong(i5) << PICK_POS.apply(56, 40)) + | (toUnsignedLong(i6) << PICK_POS.apply(56, 48)) + | (toUnsignedLong(i7) << PICK_POS.apply(56, 56)))); + } + + public int getInt() { + long index = nextGetIndex(4); + byte i0 = get(index); + byte i1 = get(++index); + byte i2 = get(++index); + byte i3 = get(++index); + return CONV_ENDIAN_INT.applyAsInt(((toUnsignedInt(i0) << PICK_POS.apply(24, 0)) + | (toUnsignedInt(i1) << PICK_POS.apply(24, 8)) + | (toUnsignedInt(i2) << PICK_POS.apply(24, 16)) + | (toUnsignedInt(i3) << PICK_POS.apply(24, 24)))); + } + + /** + * {@link ByteBuffer#getDouble()} + */ + public double getDouble() { + return Double.longBitsToDouble(getLong()); + } + + /** + * {@link ByteBuffer#getFloat()} + */ + public float getFloat() { + return Float.intBitsToFloat(getInt()); + } + + /** + * {@link ByteBuffer#position()} + */ + public long position() { + return position; + } + + // copied from Buffer.java + /** + * {@link ByteBuffer#position(int)} + */ + public BigByteBuffer position(final long newPosition) { + if (newPosition > limit | newPosition < 0) { + throw createPositionException(newPosition); + } + position = newPosition; + return this; + } + + /** + * {@link ByteBuffer#limit()} + */ + public long limit() { + return limit; + } + + /** + * {@link ByteBuffer#limit(int)} + */ + public BigByteBuffer limit(final long newLimit) { + if (newLimit > capacity | newLimit < 0) { + throw createLimitException(newLimit); + } + limit = newLimit; + if (position > newLimit) { + position = newLimit; + } + return this; + } + + /** + * {@link ByteBuffer#capacity()} + */ + long capacity() { + return capacity; + } + + // copied from Buffer.java + /** + * Verify that {@code 0 < newPosition <= limit} + * + * @param newPosition + * The new position value + * + * @throws IllegalArgumentException + * If the specified position is out of bounds. + */ + private IllegalArgumentException createPositionException(long newPosition) { + String msg = null; + + if (newPosition > limit) { + msg = "newPosition > limit: (" + newPosition + " > " + limit + ")"; + } else { // assume negative + assert newPosition < 0 : "newPosition expected to be negative"; + msg = "newPosition < 0: (" + newPosition + " < 0)"; + } + + return new IllegalArgumentException(msg); + } + + // copied from Buffer.java + /** + * Verify that {@code 0 < newLimit <= capacity} + * + * @param newLimit + * The new limit value + * + * @throws IllegalArgumentException + * If the specified limit is out of bounds. + */ + private IllegalArgumentException createLimitException(long newLimit) { + String msg = null; + + if (newLimit > capacity) { + msg = "newLimit > capacity: (" + newLimit + " > " + capacity + ")"; + } else { // assume negative + assert newLimit < 0 : "newLimit expected to be negative"; + msg = "newLimit < 0: (" + newLimit + " < 0)"; + } + + return new IllegalArgumentException(msg); + } + + // copied from Buffer.java + /** + * {@link ByteBuffer#nextGetIndex(int)} + */ + long nextGetIndex(long nb) { // package-private + long p = position; + if (limit - p < nb) { + throw new BufferUnderflowException(); + } + position = p + nb; + return p; + } + + // copied from Buffer.java + /** + * Checks the given index against the limit, throwing an {@link + * IndexOutOfBoundsException} if it is not smaller than the limit + * or is smaller than zero. + */ + long checkIndex(long i) { // package-private + if ((i < 0) || (i >= limit)) { + throw new IndexOutOfBoundsException(); + } + return i; + } + + /** + * Get a {@link ByteBuffer} for the current buffer's position. {@link #position()} + * is forwarded by the number of bytes. + * + * @param limit is the number of bytes from {@link #position()} to put into the + * {@link ByteBuffer}. + * @throws BufferUnderflowException if there aren't enough bytes remaining + */ + public ByteBuffer getByteBuffer(int limit) { + /* + A more optimal solution might be to create a ByteBuffer implementation that + is backed by bytes.subList(position, limit). + */ + final long position = nextGetIndex(limit); + final byte[] bufferBytes = new byte[limit]; + bytes.getElements(position, bufferBytes, 0, limit); + return ByteBuffer.wrap(bufferBytes); + } + + /** + * {@link ByteBuffer#duplicate()} + */ + public BigByteBuffer duplicate() { + return new BigByteBuffer( + bytes, + position(), + limit(), + capacity()); + } + +} diff --git a/src/main/java/com/maxmind/db/BufferHolder.java b/src/main/java/com/maxmind/db/BufferHolder.java index dbbe8aa..cf18736 100644 --- a/src/main/java/com/maxmind/db/BufferHolder.java +++ b/src/main/java/com/maxmind/db/BufferHolder.java @@ -1,63 +1,68 @@ package com.maxmind.db; import com.maxmind.db.Reader.FileMode; -import java.io.ByteArrayOutputStream; +import it.unimi.dsi.fastutil.bytes.ByteArrayList; +import it.unimi.dsi.fastutil.bytes.ByteBigArrayBigList; +import it.unimi.dsi.fastutil.bytes.ByteBigList; +import it.unimi.dsi.fastutil.bytes.ByteBigLists; +import it.unimi.dsi.fastutil.bytes.ByteList; +import it.unimi.dsi.fastutil.bytes.ByteMappedBigList; import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.RandomAccessFile; -import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.nio.channels.FileChannel; import java.nio.channels.FileChannel.MapMode; final class BufferHolder { // DO NOT PASS OUTSIDE THIS CLASS. Doing so will remove thread safety. - private final ByteBuffer buffer; + private final BigByteBuffer buffer; BufferHolder(File database, FileMode mode) throws IOException { + final ByteBigList list; try ( final RandomAccessFile file = new RandomAccessFile(database, "r"); final FileChannel channel = file.getChannel() ) { + final ByteBigList mapped = + ByteMappedBigList.map(channel, ByteOrder.BIG_ENDIAN, MapMode.READ_ONLY); if (mode == FileMode.MEMORY) { - final ByteBuffer buf = ByteBuffer.wrap(new byte[(int) channel.size()]); - if (channel.read(buf) != buf.capacity()) { - throw new IOException("Unable to read " - + database.getName() - + " into memory. Unexpected end of stream."); - } - this.buffer = buf.asReadOnlyBuffer(); + list = ByteBigLists.unmodifiable(new ByteBigArrayBigList(mapped)); } else { - this.buffer = channel.map(MapMode.READ_ONLY, 0, channel.size()).asReadOnlyBuffer(); + list = mapped; } } + this.buffer = BigByteBuffer.wrap(list); } /** - * Construct a ThreadBuffer from the provided URL. + * Construct a {@link BufferHolder} from the provided {@link InputStream}. * * @param stream the source of my bytes. * @throws IOException if unable to read from your source. - * @throws NullPointerException if you provide a NULL InputStream + * @throws NullPointerException if you provide a {@code null} InputStream */ BufferHolder(InputStream stream) throws IOException { if (null == stream) { throw new NullPointerException("Unable to use a NULL InputStream"); } - final ByteArrayOutputStream baos = new ByteArrayOutputStream(); - final byte[] bytes = new byte[16 * 1024]; + final ByteBigArrayBigList bigList = new ByteBigArrayBigList(); + final byte[] bytesForStream = new byte[16 * 1024]; int br; - while (-1 != (br = stream.read(bytes))) { - baos.write(bytes, 0, br); + final ByteList bytesForBigList = ByteArrayList.wrap(bytesForStream); + while (-1 != (br = stream.read(bytesForStream))) { + bigList.addAll(bytesForBigList.subList(0, br)); } - this.buffer = ByteBuffer.wrap(baos.toByteArray()).asReadOnlyBuffer(); + bigList.trim(); + this.buffer = BigByteBuffer.wrap(ByteBigLists.unmodifiable(bigList)); } /* * Returns a duplicate of the underlying ByteBuffer. The returned ByteBuffer * should not be shared between threads. */ - ByteBuffer get() { + BigByteBuffer get() { // The Java API docs for buffer state: // // Buffers are not safe for use by multiple concurrent threads. If a buffer is to be @@ -75,6 +80,7 @@ ByteBuffer get() { // operations on the original buffer object, the risk of not synchronizing this call seems // relatively low and worth taking for the performance benefit when lookups are being done // from many threads. - return this.buffer.duplicate(); + return buffer.duplicate(); } + } diff --git a/src/main/java/com/maxmind/db/CacheKey.java b/src/main/java/com/maxmind/db/CacheKey.java index d62c084..81f7951 100644 --- a/src/main/java/com/maxmind/db/CacheKey.java +++ b/src/main/java/com/maxmind/db/CacheKey.java @@ -8,17 +8,17 @@ * @param the type of value */ public final class CacheKey { - private final int offset; + private final long offset; private final Class cls; private final java.lang.reflect.Type type; - CacheKey(int offset, Class cls, java.lang.reflect.Type type) { + CacheKey(long offset, Class cls, java.lang.reflect.Type type) { this.offset = offset; this.cls = cls; this.type = type; } - int getOffset() { + long getOffset() { return this.offset; } @@ -58,7 +58,7 @@ public boolean equals(Object o) { @Override public int hashCode() { - int result = offset; + int result = Long.hashCode(offset); result = 31 * result + (cls == null ? 0 : cls.hashCode()); result = 31 * result + (type == null ? 0 : type.hashCode()); return result; diff --git a/src/main/java/com/maxmind/db/CtrlData.java b/src/main/java/com/maxmind/db/CtrlData.java index c9bf03f..af7062a 100644 --- a/src/main/java/com/maxmind/db/CtrlData.java +++ b/src/main/java/com/maxmind/db/CtrlData.java @@ -3,10 +3,10 @@ final class CtrlData { private final Type type; private final int ctrlByte; - private final int offset; - private final int size; + private final long offset; + private final long size; - CtrlData(Type type, int ctrlByte, int offset, int size) { + CtrlData(Type type, int ctrlByte, long offset, long size) { this.type = type; this.ctrlByte = ctrlByte; this.offset = offset; @@ -21,11 +21,11 @@ public int getCtrlByte() { return this.ctrlByte; } - public int getOffset() { + public long getOffset() { return this.offset; } - public int getSize() { + public long getSize() { return this.size; } } diff --git a/src/main/java/com/maxmind/db/Decoder.java b/src/main/java/com/maxmind/db/Decoder.java index ab1bc5e..c156336 100644 --- a/src/main/java/com/maxmind/db/Decoder.java +++ b/src/main/java/com/maxmind/db/Decoder.java @@ -38,11 +38,11 @@ final class Decoder { private final CharsetDecoder utfDecoder = UTF_8.newDecoder(); - private final ByteBuffer buffer; + private final BigByteBuffer buffer; private final ConcurrentHashMap constructors; - Decoder(NodeCache cache, ByteBuffer buffer, long pointerBase) { + Decoder(NodeCache cache, BigByteBuffer buffer, long pointerBase) { this( cache, buffer, @@ -53,7 +53,7 @@ final class Decoder { Decoder( NodeCache cache, - ByteBuffer buffer, + BigByteBuffer buffer, long pointerBase, ConcurrentHashMap constructors ) { @@ -65,7 +65,7 @@ final class Decoder { private final NodeCache.Loader cacheLoader = this::decode; - public T decode(int offset, Class cls) throws IOException { + public T decode(long offset, Class cls) throws IOException { if (offset >= this.buffer.capacity()) { throw new InvalidDatabaseException( "The MaxMind DB file's data section contains bad data: " @@ -77,7 +77,7 @@ public T decode(int offset, Class cls) throws IOException { } private DecodedValue decode(CacheKey key) throws IOException { - int offset = key.getOffset(); + long offset = key.getOffset(); if (offset >= this.buffer.capacity()) { throw new InvalidDatabaseException( "The MaxMind DB file's data section contains bad data: " @@ -110,7 +110,7 @@ private DecodedValue decode(Class cls, java.lang.reflect.Type genericType } int targetOffset = (int) pointer; - int position = buffer.position(); + long position = buffer.position(); CacheKey key = new CacheKey(targetOffset, cls, genericType); DecodedValue o = cache.get(key, cacheLoader); @@ -196,9 +196,9 @@ private Object decodeByType( } private String decodeString(int size) throws CharacterCodingException { - int oldLimit = buffer.limit(); - buffer.limit(buffer.position() + size); - String s = utfDecoder.decode(buffer).toString(); + long oldLimit = buffer.limit(); + final ByteBuffer byteBuffer = buffer.getByteBuffer(size); + String s = utfDecoder.decode(byteBuffer).toString(); buffer.limit(oldLimit); return s; } @@ -231,7 +231,7 @@ private int decodeInteger(int base, int size) { return Decoder.decodeInteger(this.buffer, base, size); } - static int decodeInteger(ByteBuffer buffer, int base, int size) { + static int decodeInteger(BigByteBuffer buffer, int base, int size) { int integer = base; for (int i = 0; i < size; i++) { integer = (integer << 8) | (buffer.get() & 0xFF); @@ -426,7 +426,7 @@ private Object decodeMapIntoObject(int size, Class cls) Integer parameterIndex = parameterIndexes.get(key); if (parameterIndex == null) { - int offset = this.nextValueOffset(this.buffer.position(), 1); + long offset = this.nextValueOffset(this.buffer.position(), 1); this.buffer.position(offset); continue; } @@ -492,7 +492,7 @@ private static String getParameterName( + " is not annotated with MaxMindDbParameter."); } - private int nextValueOffset(int offset, int numberToSkip) + private long nextValueOffset(long offset, long numberToSkip) throws InvalidDatabaseException { if (numberToSkip == 0) { return offset; @@ -500,7 +500,7 @@ private int nextValueOffset(int offset, int numberToSkip) CtrlData ctrlData = this.getCtrlData(offset); int ctrlByte = ctrlData.getCtrlByte(); - int size = ctrlData.getSize(); + long size = ctrlData.getSize(); offset = ctrlData.getOffset(); Type type = ctrlData.getType(); @@ -525,7 +525,7 @@ private int nextValueOffset(int offset, int numberToSkip) return nextValueOffset(offset, numberToSkip - 1); } - private CtrlData getCtrlData(int offset) + private CtrlData getCtrlData(long offset) throws InvalidDatabaseException { if (offset >= this.buffer.capacity()) { throw new InvalidDatabaseException( @@ -578,7 +578,7 @@ private byte[] getByteArray(int length) { return Decoder.getByteArray(this.buffer, length); } - private static byte[] getByteArray(ByteBuffer buffer, int length) { + private static byte[] getByteArray(BigByteBuffer buffer, int length) { byte[] bytes = new byte[length]; buffer.get(bytes); return bytes; diff --git a/src/main/java/com/maxmind/db/Metadata.java b/src/main/java/com/maxmind/db/Metadata.java index 3c31f46..aa2d733 100644 --- a/src/main/java/com/maxmind/db/Metadata.java +++ b/src/main/java/com/maxmind/db/Metadata.java @@ -24,11 +24,11 @@ public final class Metadata { private final int nodeByteSize; - private final int nodeCount; + private final long nodeCount; private final int recordSize; - private final int searchTreeSize; + private final long searchTreeSize; /** * Constructs a {@code Metadata} object. @@ -71,7 +71,7 @@ public Metadata( this.languages = languages; this.description = description; this.ipVersion = ipVersion; - this.nodeCount = (int) nodeCount; + this.nodeCount = nodeCount; this.recordSize = recordSize; this.nodeByteSize = this.recordSize / 4; @@ -140,7 +140,7 @@ int getNodeByteSize() { /** * @return the number of nodes in the search tree. */ - int getNodeCount() { + long getNodeCount() { return this.nodeCount; } @@ -155,7 +155,7 @@ int getRecordSize() { /** * @return the searchTreeSize */ - int getSearchTreeSize() { + long getSearchTreeSize() { return this.searchTreeSize; } diff --git a/src/main/java/com/maxmind/db/Networks.java b/src/main/java/com/maxmind/db/Networks.java index b06dfff..2131b29 100644 --- a/src/main/java/com/maxmind/db/Networks.java +++ b/src/main/java/com/maxmind/db/Networks.java @@ -3,7 +3,6 @@ import java.io.IOException; import java.net.Inet4Address; import java.net.InetAddress; -import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Iterator; import java.util.Stack; @@ -19,7 +18,7 @@ public final class Networks implements Iterator> { private final Stack nodes; private NetworkNode lastNode; private final boolean includeAliasedNetworks; - private final ByteBuffer buffer; /* Stores the buffer for Next() calls */ + private final BigByteBuffer buffer; /* Stores the buffer for Next() calls */ private final Class typeParameterClass; /** diff --git a/src/main/java/com/maxmind/db/Reader.java b/src/main/java/com/maxmind/db/Reader.java index 1b8b84b..93be9bb 100644 --- a/src/main/java/com/maxmind/db/Reader.java +++ b/src/main/java/com/maxmind/db/Reader.java @@ -7,7 +7,6 @@ import java.net.Inet6Address; import java.net.InetAddress; import java.net.UnknownHostException; -import java.nio.ByteBuffer; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicReference; @@ -128,8 +127,8 @@ private Reader(BufferHolder bufferHolder, String name, NodeCache cache) throws I } this.cache = cache; - ByteBuffer buffer = bufferHolder.get(); - int start = this.findMetadataStart(buffer, name); + BigByteBuffer buffer = bufferHolder.get(); + long start = this.findMetadataStart(buffer, name); Decoder metadataDecoder = new Decoder(this.cache, buffer, start); this.metadata = metadataDecoder.decode(start, Metadata.class); @@ -176,8 +175,8 @@ public DatabaseRecord getRecord(InetAddress ipAddress, Class cls) int pl = traverseResult[1]; int record = traverseResult[0]; - int nodeCount = this.metadata.getNodeCount(); - ByteBuffer buffer = this.getBufferHolder().get(); + long nodeCount = this.metadata.getNodeCount(); + BigByteBuffer buffer = this.getBufferHolder().get(); T dataRecord = null; if (record > nodeCount) { // record is a data pointer @@ -264,7 +263,7 @@ private int startNode(int bitLength) { return 0; } - private int findIpV4StartNode(ByteBuffer buffer) + private int findIpV4StartNode(BigByteBuffer buffer) throws InvalidDatabaseException { if (this.metadata.getIpVersion() == 4) { return 0; @@ -337,10 +336,10 @@ public Networks networksWithin( */ private int[] traverseTree(byte[] ip, int bitCount) throws ClosedDatabaseException, InvalidDatabaseException { - ByteBuffer buffer = this.getBufferHolder().get(); + BigByteBuffer buffer = this.getBufferHolder().get(); int bitLength = ip.length * 8; int record = this.startNode(bitLength); - int nodeCount = this.metadata.getNodeCount(); + long nodeCount = this.metadata.getNodeCount(); int i = 0; for (; i < bitCount && record < nodeCount; i++) { @@ -355,11 +354,11 @@ record = this.readNode(buffer, record, bit); return new int[]{record, i}; } - int readNode(ByteBuffer buffer, int nodeNumber, int index) + int readNode(BigByteBuffer buffer, int nodeNumber, int index) throws InvalidDatabaseException { // index is the index of the record within the node, which // can either be 0 or 1. - int baseOffset = nodeNumber * this.metadata.getNodeByteSize(); + long baseOffset = ((long) nodeNumber) * this.metadata.getNodeByteSize(); switch (this.metadata.getRecordSize()) { case 24: @@ -389,11 +388,11 @@ int readNode(ByteBuffer buffer, int nodeNumber, int index) } T resolveDataPointer( - ByteBuffer buffer, + BigByteBuffer buffer, int pointer, Class cls ) throws IOException { - int resolved = (pointer - this.metadata.getNodeCount()) + long resolved = (pointer - this.metadata.getNodeCount()) + this.metadata.getSearchTreeSize(); if (resolved >= buffer.capacity()) { @@ -421,9 +420,9 @@ T resolveDataPointer( * are much faster algorithms (e.g., Boyer-Moore) for this if speed is ever * an issue, but I suspect it won't be. */ - private int findMetadataStart(ByteBuffer buffer, String databaseName) + private long findMetadataStart(BigByteBuffer buffer, String databaseName) throws InvalidDatabaseException { - int fileSize = buffer.capacity(); + long fileSize = buffer.capacity(); FILE: for (int i = 0; i < fileSize - METADATA_START_MARKER.length + 1; i++) { @@ -455,7 +454,7 @@ public Metadata getMetadata() { *

* If you are using FileMode.MEMORY_MAPPED, this will * not unmap the underlying file due to a limitation in Java's - * MappedByteBuffer. It will however set the reference to + * MappedBigByteBuffer. It will however set the reference to * the buffer to null, allowing the garbage collector to * collect it. *

diff --git a/src/main/java/module-info.java b/src/main/java/module-info.java index 3c5a681..a7067b4 100644 --- a/src/main/java/module-info.java +++ b/src/main/java/module-info.java @@ -1,3 +1,4 @@ module com.maxmind.db { + requires it.unimi.dsi.fastutil; exports com.maxmind.db; } diff --git a/src/test/java/com/maxmind/db/BigByteBufferTest.java b/src/test/java/com/maxmind/db/BigByteBufferTest.java new file mode 100644 index 0000000..43ef0f8 --- /dev/null +++ b/src/test/java/com/maxmind/db/BigByteBufferTest.java @@ -0,0 +1,122 @@ +package com.maxmind.db; + +import it.unimi.dsi.fastutil.bytes.ByteBigArrayBigList; +import it.unimi.dsi.fastutil.bytes.ByteMappedBigList; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Map; +import java.util.stream.IntStream; +import java.util.stream.LongStream; +import java.util.stream.Stream; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +public class BigByteBufferTest { + + // past 2 GiB + static final long SIZE = (long) Integer.MAX_VALUE + 1; + + static BigByteBuffer createLargeBuffer() throws IOException { + final ByteBuffer oneByte = ByteBuffer.wrap(new byte[] {1}); + final Path temp = Files.createTempFile(BigByteBufferTest.class.getName(), "createLargeBuffer"); + temp.toFile().deleteOnExit(); + try (FileChannel channel = FileChannel.open(temp, StandardOpenOption.SPARSE, StandardOpenOption.READ, StandardOpenOption.WRITE)) { + Assertions.assertEquals(0L, channel.size()); + channel.position(SIZE - oneByte.capacity()); + channel.write(oneByte); + Assertions.assertEquals(SIZE, channel.size()); + Assertions.assertTrue(channel.size() > Integer.MAX_VALUE); + channel.position(0L); + return BigByteBuffer.wrap(ByteMappedBigList.map(channel)); + } + } + + static Stream intsProvider() { + return IntStream.of( + Integer.MIN_VALUE, + Integer.MAX_VALUE, + 0, + 1, + -1 + ).mapToObj( + i -> { + byte[] bytes = new byte[4]; + ByteBuffer b = ByteBuffer.wrap(bytes); + b.putInt(i); + return Arguments.arguments(i, bytes); + } + ); + } + + static Stream longsProvider() { + return LongStream.of( + Long.MIN_VALUE, + Long.MAX_VALUE, + 0L, + 1L, + -1L + ).mapToObj( + i -> { + byte[] bytes = new byte[8]; + ByteBuffer b = ByteBuffer.wrap(bytes); + b.putLong(i); + return Arguments.arguments(i, bytes); + } + ); + } + + @Test + public void testLargeFile() throws IOException { + final BigByteBuffer buffer = createLargeBuffer(); + Assertions.assertEquals((byte) 0, buffer.get(SIZE - 2)); + Assertions.assertEquals((byte) 1, buffer.get(SIZE - 1)); + Assertions.assertThrows(IndexOutOfBoundsException.class, () -> buffer.get(SIZE)); + } + + @ParameterizedTest + @MethodSource("intsProvider") + public final void testGetInt(int i, byte[] bytes) { + final BigByteBuffer buffer = BigByteBuffer.wrap(new ByteBigArrayBigList(new byte[][] {bytes})); + final int actual = buffer.getInt(); + Assertions.assertEquals(i, actual); + } + + + @ParameterizedTest + @MethodSource("longsProvider") + public final void testGetLong(long i, byte[] bytes) { + final BigByteBuffer buffer = BigByteBuffer.wrap(new ByteBigArrayBigList(new byte[][] {bytes})); + final long actual = buffer.getLong(); + Assertions.assertEquals(i, actual); + } + + @Test + public final void testGetDouble() { + for (Map.Entry entry: DecoderTest.doubles().entrySet()) { + final BigByteBuffer buffer = BigByteBuffer.wrap(new ByteBigArrayBigList(new byte[][] {entry.getValue()})); + // skip the type byte + buffer.position(1); + final double actual = buffer.getDouble(); + Assertions.assertEquals(entry.getKey(), actual); + } + } + + @Test + public final void testGetFloat() { + for (Map.Entry entry: DecoderTest.floats().entrySet()) { + final BigByteBuffer buffer = BigByteBuffer.wrap(new ByteBigArrayBigList(new byte[][] {entry.getValue()})); + // skip the type byte and extended type byte + buffer.position(2); + final float actual = buffer.getFloat(); + Assertions.assertEquals(entry.getKey(), actual); + } + } + +} diff --git a/src/test/java/com/maxmind/db/BufferHolderTest.java b/src/test/java/com/maxmind/db/BufferHolderTest.java new file mode 100644 index 0000000..7659262 --- /dev/null +++ b/src/test/java/com/maxmind/db/BufferHolderTest.java @@ -0,0 +1,54 @@ +package com.maxmind.db; + +import com.maxmind.db.Reader.FileMode; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class BufferHolderTest { + + static Path createLargeFile() throws IOException { + final ByteBuffer oneByte = ByteBuffer.wrap(new byte[] {1}); + final Path temp = Files.createTempFile(BigByteBufferTest.class.getName(), "createLargeBuffer"); + temp.toFile().deleteOnExit(); + try (FileChannel channel = FileChannel.open(temp, StandardOpenOption.SPARSE, StandardOpenOption.READ, StandardOpenOption.WRITE)) { + Assertions.assertEquals(0, channel.size()); + channel.position(BigByteBufferTest.SIZE - oneByte.capacity()); + channel.write(oneByte); + channel.close(); + final long size = Files.size(temp); + Assertions.assertEquals(BigByteBufferTest.SIZE, size); + Assertions.assertTrue(size > Integer.MAX_VALUE); + return temp; + } + } + + @Test + public void testMemoryMap() throws IOException { + final Path temp = createLargeFile(); + final BufferHolder holder = new BufferHolder(temp.toFile(), FileMode.MEMORY_MAPPED); + final BigByteBuffer bigByteBuffer = holder.get(); + Assertions.assertEquals((byte) 0, bigByteBuffer.get()); + bigByteBuffer.position(BigByteBufferTest.SIZE - 1); + Assertions.assertEquals((byte) 1, bigByteBuffer.get()); + } + + @Test + public void testThreadSafe() throws IOException { + final BufferHolder holder = + new BufferHolder( + new ByteArrayInputStream(new byte[] {0, 1}) + ); + final BigByteBuffer bigByteBuffer0 = holder.get(); + bigByteBuffer0.get(); + final BigByteBuffer bigByteBuffer1 = holder.get(); + Assertions.assertNotEquals(bigByteBuffer0.position(), bigByteBuffer1.position()); + } + +} diff --git a/src/test/java/com/maxmind/db/DecoderTest.java b/src/test/java/com/maxmind/db/DecoderTest.java index 2668bd5..8d8267a 100644 --- a/src/test/java/com/maxmind/db/DecoderTest.java +++ b/src/test/java/com/maxmind/db/DecoderTest.java @@ -6,14 +6,13 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import it.unimi.dsi.fastutil.bytes.ByteMappedBigList; import java.io.File; import java.io.IOException; import java.io.RandomAccessFile; import java.math.BigInteger; import java.nio.ByteBuffer; -import java.nio.MappedByteBuffer; import java.nio.channels.FileChannel; -import java.nio.channels.FileChannel.MapMode; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashMap; @@ -204,7 +203,7 @@ private static void addTestString(Map tests, byte[] ctrl, tests.put(str, bytes); } - private static Map doubles() { + static Map doubles() { Map doubles = new HashMap<>(); doubles.put(0.0, new byte[] {0x68, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}); @@ -226,7 +225,7 @@ private static Map doubles() { return doubles; } - private static Map floats() { + static Map floats() { Map floats = new HashMap<>(); floats.put((float) 0.0, new byte[] {0x4, 0x8, 0x0, 0x0, 0x0, 0x0}); floats.put((float) 1.0, new byte[] {0x4, 0x8, 0x3F, (byte) 0x80, 0x0, @@ -406,7 +405,7 @@ public void testArrays() throws IOException { @Test public void testInvalidControlByte() throws IOException { try (FileChannel fc = DecoderTest.getFileChannel(new byte[] {0x0, 0xF})) { - MappedByteBuffer mmap = fc.map(MapMode.READ_ONLY, 0, fc.size()); + BigByteBuffer mmap = BigByteBuffer.wrap(ByteMappedBigList.map(fc)); Decoder decoder = new Decoder(new CHMCache(), mmap, 0); InvalidDatabaseException ex = assertThrows( @@ -427,7 +426,7 @@ private static void testTypeDecoding(Type type, Map tests) String desc = "decoded " + type.name() + " - " + expect; try (FileChannel fc = DecoderTest.getFileChannel(input)) { - MappedByteBuffer mmap = fc.map(MapMode.READ_ONLY, 0, fc.size()); + BigByteBuffer mmap = BigByteBuffer.wrap(ByteMappedBigList.map(fc)); Decoder decoder = new Decoder(cache, mmap, 0); decoder.pointerTestHack = true;