From 26e9d7651e1c39d0a4b7df8fd620412ab9d472ab Mon Sep 17 00:00:00 2001 From: Mohammad Kolahdouzan Date: Fri, 29 Jan 2016 03:02:00 +0000 Subject: [PATCH 1/2] use CharsetEncoder/Decoder, removed support for all encodings but utf-8 --- ChangeLog | 5 + pom.xml | 2 +- src/main/java/org/lwes/ArrayEvent.java | 122 +++++------ src/main/java/org/lwes/BaseType.java | 17 +- src/main/java/org/lwes/DefaultEvent.java | 20 +- src/main/java/org/lwes/Event.java | 19 +- src/main/java/org/lwes/EventFactory.java | 29 +-- src/main/java/org/lwes/MapEvent.java | 61 ++---- .../org/lwes/serializer/Deserializer.java | 40 +--- .../java/org/lwes/serializer/Serializer.java | 72 +++---- .../java/org/lwes/util/CharacterEncoding.java | 178 ---------------- .../java/org/lwes/util/EncodedString.java | 193 ++++++++++++------ src/main/java/org/lwes/util/NumberCodec.java | 55 ++--- .../org/lwes/ArrayEventPerformanceTest.java | 16 +- src/test/java/org/lwes/ArrayEventTest.java | 33 +-- src/test/java/org/lwes/BaseTypeTest.java | 46 ++++- src/test/java/org/lwes/EventFactoryTest.java | 6 - src/test/java/org/lwes/MapEventTest.java | 64 ++++++ src/test/java/org/lwes/RandomEventTest.java | 4 +- .../org/lwes/serializer/SerializerTest.java | 18 +- .../org/lwes/util/CharacterEncodingTest.java | 68 ------ .../java/org/lwes/util/EncodedStringTest.java | 118 ++++++++--- 22 files changed, 535 insertions(+), 651 deletions(-) delete mode 100644 src/main/java/org/lwes/util/CharacterEncoding.java delete mode 100644 src/test/java/org/lwes/util/CharacterEncodingTest.java diff --git a/ChangeLog b/ChangeLog index 65e841d..2db66e6 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,8 @@ +* Fri Jan 29 2016 Mohammad Kolahdouzan 3.0.0 +- replaced all String.getBytes() used for character encoding/decoding with + CharsetEncoder/CharsetDecoder +- removed support for all encodings except utf-8 + * Fri Nov 20 2015 Kenneth Kharma 2.2.0 - only emit System::Startup and System::Shutdown when heartbeats are enabled - added heartbeat support to emitter groups diff --git a/pom.xml b/pom.xml index 17a31d3..1590cdc 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,6 @@ org.lwes lwes-java jar - 2.2.0 lwes-java Lightweight event system, java implementation http://lwes.org @@ -336,4 +335,5 @@ + 3.0.0 diff --git a/src/main/java/org/lwes/ArrayEvent.java b/src/main/java/org/lwes/ArrayEvent.java index aca99f2..b028b1f 100644 --- a/src/main/java/org/lwes/ArrayEvent.java +++ b/src/main/java/org/lwes/ArrayEvent.java @@ -12,7 +12,6 @@ import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; @@ -25,10 +24,12 @@ import java.util.TreeSet; import org.apache.commons.lang3.mutable.MutableInt; +import org.lwes.MemoryPool.Buffer; import org.lwes.serializer.Deserializer; import org.lwes.serializer.DeserializerState; import org.lwes.serializer.Serializer; import org.lwes.util.EncodedString; +import org.lwes.util.Util; public final class ArrayEvent extends DefaultEvent { @@ -36,7 +37,6 @@ public final class ArrayEvent extends DefaultEvent { private byte[] bytes; private final DeserializerState tempState = new DeserializerState(); private int length = 3; - private short encoding = DEFAULT_ENCODING; private static Map STATS = new EnumMap(ArrayEventStats.class); @@ -57,10 +57,10 @@ public final class ArrayEvent extends DefaultEvent { public ArrayEvent() { bytes = new byte[MAX_MESSAGE_SIZE]; length = getValueListIndex(); - setEncoding(DEFAULT_ENCODING); + setEncoding(); updateCreationStats(); } - + /** * All constructors call this aux function once */ @@ -77,7 +77,7 @@ public ArrayEvent(String name) throws EventSystemException { setEventName(name); } - + /** * Creates a new event from the given byte array, copying it only if the copy flag is true. * @param bytes @@ -101,7 +101,7 @@ public ArrayEvent(final byte[] bytes, final int len, final boolean copy) { updateCreationStats(); resetCaches(); } - + /** * Creates a new event, making a copy of the given byte array into a newly allocated buffer * @param bytes @@ -113,7 +113,7 @@ public ArrayEvent(final byte[] bytes) { public ArrayEvent(final byte[] bytes, boolean copy) { this(bytes, bytes.length, copy); } - + private ArrayEvent(byte[] bytes, int offset, int length, int excess) { this.bytes = Arrays.copyOfRange(bytes, offset, offset + length + excess); this.length = length; @@ -121,13 +121,12 @@ private ArrayEvent(byte[] bytes, int offset, int length, int excess) { resetCaches(); } - private ArrayEvent(byte[] bytes, int length, short encoding) { + private ArrayEvent(byte[] bytes, int length) { this(); assert length <= bytes.length; assert length <= this.bytes.length; System.arraycopy(bytes, 0, this.bytes, 0, length); this.length = length; - this.encoding = encoding; } @Override @@ -141,7 +140,6 @@ public void reset() { Arrays.fill(bytes, (byte) 0); length = getValueListIndex(); tempState.reset(); - encoding = DEFAULT_ENCODING; } @Override @@ -160,25 +158,29 @@ public void clear(String key) { @Override public void setEventName(String name) { - checkShortStringLength(name, encoding, MAX_EVENT_NAME_SIZE); + checkShortStringLength(name, MAX_EVENT_NAME_SIZE); final String oldName = getEventName(); - final String defaultEncodingString = ENCODING_STRINGS[DEFAULT_ENCODING].getEncodingString(); - try { - final byte[] oldBytes = oldName.getBytes(defaultEncodingString); - final byte[] newBytes = name.getBytes(defaultEncodingString); - if (oldBytes != newBytes) { - final int numFields = getNumEventAttributes(); - final int oldValueListIndex = getValueListIndex(); - final int newValueListIndex = oldValueListIndex + newBytes.length - oldBytes.length; - Serializer.serializeUBYTE((short) newBytes.length, bytes, 0); - shiftTail(oldValueListIndex, newValueListIndex); - int offset = Serializer.serializeEVENTWORD(name, bytes, 0); - Serializer.serializeUINT16(numFields, bytes, offset); - } - } - catch (UnsupportedEncodingException e) { - throw new IllegalArgumentException("Unknown Encoding: " + defaultEncodingString); + + Buffer oldBytesBuffer = EncodedString.encode(oldName); + final byte[] oldBytes = oldBytesBuffer.getEncoderOutputBuffer().array(); + int oldBytesLen = oldBytesBuffer.getEncoderOutputBuffer().position(); + + Buffer newBytesBuffer = EncodedString.encode(name); + final byte[] newBytes = newBytesBuffer.getEncoderOutputBuffer().array(); + int newBytesLen = newBytesBuffer.getEncoderOutputBuffer().position(); + + if (!Util.compareByteArrays(oldBytes, oldBytesLen, newBytes, newBytesLen)) { + final int numFields = getNumEventAttributes(); + final int oldValueListIndex = getValueListIndex(); + final int newValueListIndex = oldValueListIndex + newBytesLen - oldBytesLen; + Serializer.serializeUBYTE((short) newBytesLen, bytes, 0); + shiftTail(oldValueListIndex, newValueListIndex); + int offset = Serializer.serializeEVENTWORD(name, bytes, 0); + Serializer.serializeUINT16(numFields, bytes, offset); } + + MemoryPool.putBack(newBytesBuffer); + MemoryPool.putBack(oldBytesBuffer); } /** @@ -187,10 +189,10 @@ public void setEventName(String name) { */ @Override public void set(String key, FieldType type, Object value) { - checkShortStringLength(key, encoding, MAX_FIELD_NAME_SIZE); + checkShortStringLength(key, MAX_FIELD_NAME_SIZE); if (ENCODING.equals(key)) { if (type == FieldType.INT16) { - setEncoding((Short) value); + setEncoding(); } else { throw new EventSystemException("Attempted to set " + ENCODING + " with type " @@ -201,7 +203,7 @@ public void set(String key, FieldType type, Object value) { else { if (type == FieldType.STRING || type == FieldType.STRING_ARRAY) { if (find(ENCODING) < 0) { - setEncoding(encoding); + setEncoding(); } } final int fieldIndex = find(key); @@ -211,7 +213,7 @@ public void set(String key, FieldType type, Object value) { final FieldType oldType = FieldType.byToken(bytes[tokenIndex]); if (oldType == type && type.isConstantSize()) { // Modify the value in place, requiring no shifts. - Serializer.serializeValue(type, value, encoding, bytes, tokenIndex + 1); + Serializer.serializeValue(type, value, bytes, tokenIndex + 1); return; } clear(key); @@ -235,7 +237,7 @@ private void appendField(String key, FieldType type, Object value) { try { length += Serializer.serializeATTRIBUTEWORD(key, bytes, length); length += Serializer.serializeBYTE(type.token, bytes, length); - length += Serializer.serializeValue(type, value, encoding, bytes, length); + length += Serializer.serializeValue(type, value, bytes, length); setNumEventAttributes(getNumEventAttributes() + 1); } catch (ArrayIndexOutOfBoundsException e) { @@ -248,28 +250,21 @@ private void appendField(String key, FieldType type, Object value) { } @Override - public void setEncoding(short encoding) { - if (encoding < 0 || encoding >= ENCODING_STRINGS.length) { - throw new IllegalArgumentException( - "Unable to set " + ENCODING + " to " + encoding + "; acceptable range is 0<=enc<" + - ENCODING_STRINGS.length); - } - - this.encoding = encoding; + public void setEncoding() { final int fieldCountIndex = getFieldCountIndex(); final int numFields = deserializeUINT16(fieldCountIndex); tempState.set(fieldCountIndex + 2); if (numFields == 0) { // We had no fields at all; just set ENCODING. - appendField(ENCODING, FieldType.INT16, encoding); + appendField(ENCODING, FieldType.INT16, UTF_8); return; } else { if (ENCODING.equals(Deserializer.deserializeATTRIBUTEWORD(tempState, bytes))) { if (FieldType.INT16.token == Deserializer.deserializeBYTE(tempState, bytes)) { // Encoding was already the first field and the right type. Just change the value. - Serializer.serializeINT16(encoding, bytes, tempState.currentIndex()); + Serializer.serializeINT16(UTF_8, bytes, tempState.currentIndex()); return; } else { // Encoding was the first field, but had the wrong type. Clear it and recreate below. @@ -284,7 +279,7 @@ public void setEncoding(short encoding) { shiftTail(index, index + SERIALIZED_ENCODING_LENGTH + 3); index += Serializer.serializeATTRIBUTEWORD(ENCODING, bytes, index); index += Serializer.serializeBYTE(FieldType.INT16.token, bytes, index); - index += Serializer.serializeINT16(encoding, bytes, index); + index += Serializer.serializeINT16(UTF_8, bytes, index); setNumEventAttributes(getNumEventAttributes() + 1); } @@ -359,12 +354,7 @@ private Object get(FieldType type, int valueIndex) { } private Object get(FieldType type, DeserializerState state) { - return Deserializer.deserializeValue(state, bytes, type, encoding); - } - - @Override - public short getEncoding() { - return encoding; + return Deserializer.deserializeValue(state, bytes, type); } /** @@ -372,8 +362,8 @@ public short getEncoding() { * this.encoding value. */ private short readEncoding() { - final Short encodingValue = getInt16(ENCODING); - return encodingValue == null ? DEFAULT_ENCODING : encodingValue; + getInt16(ENCODING); // ignore the encoding + return UTF_8; } @Override @@ -410,14 +400,14 @@ public void deserialize(ByteBuffer buffer, int length) { } private void resetCaches() { - this.encoding = readEncoding(); + readEncoding(); } @Override public int getBytesSize() { return length; } - + public int getCapacity() { return bytes.length; } @@ -425,18 +415,20 @@ public int getCapacity() { @Override public Event copy() { STATS.get(ArrayEventStats.COPIES).increment(); - return new ArrayEvent(bytes, length, encoding); + return new ArrayEvent(bytes, length); } private int find(String key) { int count = 0; + Buffer buffer = null; try { - final byte[] keyBytes = EncodedString.getBytes(key, ENCODING_STRINGS[DEFAULT_ENCODING]); + buffer = EncodedString.encode(key); for (tempState.set(getValueListIndex()); tempState.currentIndex() < length; ) { ++count; final int keyIndex = tempState.currentIndex(); final int keyLength = bytes[keyIndex] & 0xff; - if (arrayEquals(bytes, keyIndex + 1, keyLength, keyBytes, 0, keyBytes.length)) { + if (arrayEquals(bytes, keyIndex + 1, keyLength, buffer.getEncoderOutputBuffer().array(), + 0, buffer.getEncoderOutputBuffer().position())) { return keyIndex; } else { @@ -455,6 +447,8 @@ private int find(String key) { return -1; } finally { + // return the buffer back to the pool + MemoryPool.putBack(buffer); STATS.get(ArrayEventStats.FINDS).increment(); STATS.get(ArrayEventStats.PARSES).add(count); } @@ -481,7 +475,7 @@ public int getValueByteSize(FieldType type, int valueIndex) { } if (type.isArray()) { final FieldType componentType = type.getComponentType(); - + if (type.isNullableArray()) { // array_len + bitset_len + bitset + array DeserializerState ds = new DeserializerState(); @@ -498,7 +492,7 @@ public int getValueByteSize(FieldType type, int valueIndex) { } return ds.currentIndex() - valueIndex; } - + if (componentType.isConstantSize()) { return 2 + deserializeUINT16(valueIndex) * componentType.getConstantSize(); } else { @@ -543,7 +537,6 @@ public void copyFrom(Event event) { System.arraycopy(ae.bytes, 0, bytes, 0, ae.length); length = ae.length; tempState.reset(); - encoding = ae.encoding; } else { super.copyFrom(event); @@ -586,13 +579,10 @@ public void swap(ArrayEvent event) { } final byte[] tempBytes = bytes; final int tempLength = length; - final short tempEncoding = encoding; this.bytes = event.bytes; this.length = event.length; - this.encoding = event.encoding; event.bytes = tempBytes; event.length = tempLength; - event.encoding = tempEncoding; STATS.get(ArrayEventStats.SWAPS).increment(); } @@ -614,7 +604,7 @@ public ArrayEvent trim(int excess) { return new ArrayEvent(bytes, 0, length, excess); } - + private static boolean arrayEquals(final byte[] b1, int o1, final int l1, final byte[] b2, final int o2, final int l2) { if (l1 != l2) { return false; @@ -646,7 +636,7 @@ public String toStringDetailed() { buf.append(String.format("Event name: \"%s\"%n", getEventName())); buf.append(String.format("Serialized length: %d%n", length)); buf.append(String.format("tempState index: %d%n", tempState.currentIndex())); - buf.append(String.format("Encoding: %s%n", Event.ENCODING_STRINGS[encoding].getEncodingString())); + buf.append(String.format("Encoding: %s%n", UTF_8_NAME)); buf.append(String.format("Number of fields: %d%n", getNumEventAttributes())); final DeserializerState ds = new DeserializerState(); ds.set(getValueListIndex()); @@ -667,7 +657,7 @@ public String toStringDetailed() { throw new Exception("Error when reading field name: " + e.getMessage()); } try { - value = Deserializer.deserializeValue(ds, bytes, type, encoding); + value = Deserializer.deserializeValue(ds, bytes, type); } catch (Exception e) { throw new Exception("Error when reading field name: " + e.getMessage()); @@ -711,7 +701,7 @@ private final class ArrayEventFieldAccessor extends DefaultFieldAccessor { currentValueIndex = Integer.MIN_VALUE; public void advance() { - // Deserialize name,type eagerly; deserialize value lazily. + // Deserialize name,type eagerly; deserialize value lazily. currentFieldIndex = nextFieldIndex; accessorTempState.set(currentFieldIndex); setName(Deserializer.deserializeATTRIBUTEWORD(accessorTempState, bytes)); diff --git a/src/main/java/org/lwes/BaseType.java b/src/main/java/org/lwes/BaseType.java index 72d4d7e..16861d1 100644 --- a/src/main/java/org/lwes/BaseType.java +++ b/src/main/java/org/lwes/BaseType.java @@ -209,7 +209,7 @@ public void setSizeRestriction(Integer sizeRestriction) { this.sizeRestriction = sizeRestriction; } - public int getNullableArrayByteSize(short encoding) { + public int getNullableArrayByteSize() { // Size of the array (* n bytes) + 2 bytes for the length number int count = 2 + 2; // start with the length of the array + length of bitset int arrayLen = Array.getLength(typeObject); @@ -220,7 +220,7 @@ public int getNullableArrayByteSize(short encoding) { switch (type.getComponentType()) { case STRING: // length of each string in bytes + 2 for the length number - count += EncodedString.getBytes((String) o, Event.ENCODING_STRINGS[encoding]).length + 2; + count += EncodedString.getEncodedLength((String) o) + 2; break; default: count += type.getComponentType().getConstantSize(); @@ -232,9 +232,9 @@ public int getNullableArrayByteSize(short encoding) { return count; } - public int getByteSize(short encoding) { + public int getByteSize() { if (type.isNullableArray()) { - return getNullableArrayByteSize(encoding); + return getNullableArrayByteSize(); } else if (type.isConstantSize()) { return type.getConstantSize(); @@ -243,14 +243,13 @@ else if (type.isConstantSize()) { switch (type) { case STRING: /* add size of string plus two bytes for the length */ - return EncodedString.getBytes((String) typeObject, - Event.ENCODING_STRINGS[encoding]).length + 2; + return EncodedString.getEncodedLength((String) typeObject) + 2; case STRING_ARRAY: { int count = 2; // start with the length of the array String[] anArray = (String[]) typeObject; for (String s : anArray) { if (s != null) { - count += EncodedString.getBytes(s, Event.ENCODING_STRINGS[encoding]).length + 2; + count += EncodedString.getEncodedLength(s) + 2; } } return count; @@ -299,9 +298,9 @@ else if (type.isConstantSize()) { throw new IllegalArgumentException("Unknown size of BaseType " + type.name); } - public int bytesStoreSize(short encoding) { + public int bytesStoreSize() { /* add size of data plus size of token denoting data type */ - return getByteSize(encoding) + 1; + return getByteSize() + 1; } public Object parseFromString(String string) throws EventSystemException { diff --git a/src/main/java/org/lwes/DefaultEvent.java b/src/main/java/org/lwes/DefaultEvent.java index 6d0dd8f..4ba5aa4 100644 --- a/src/main/java/org/lwes/DefaultEvent.java +++ b/src/main/java/org/lwes/DefaultEvent.java @@ -19,6 +19,7 @@ import java.util.Iterator; import java.util.Set; import java.util.TreeSet; +import java.util.concurrent.ConcurrentHashMap; import org.lwes.util.EncodedString; import org.lwes.util.IPAddress; @@ -26,6 +27,11 @@ public abstract class DefaultEvent implements Event { private static final BigInteger UINT64_MASK = new BigInteger("ffffffffffffffff", 16); + // used to cache length of the encoded event names and keys + static protected ConcurrentHashMap encodedStringLengthCache = + new ConcurrentHashMap(); + + public void setInt16Array(String attributeName, short[] value) throws EventSystemException { set(attributeName, FieldType.INT16_ARRAY, value); } @@ -384,7 +390,7 @@ public boolean equals(Object o) { public abstract void set(String key, FieldType type, Object value); - public abstract void setEncoding(short encoding); + public abstract void setEncoding(); public abstract int getNumEventAttributes(); @@ -396,8 +402,6 @@ public boolean equals(Object o) { public abstract Object get(String attributeName); - public abstract short getEncoding(); - public abstract int serialize(byte[] bytes, int offset); public abstract int serialize(DataOutput output) throws IOException; @@ -453,9 +457,15 @@ else if (value.getClass().isArray()) { return sb.toString(); } - protected static void checkShortStringLength(String string, short encoding, int maxLength) + protected static void checkShortStringLength(String string, int maxLength) throws EventSystemException { - final int serializedLength = EncodedString.getBytes(string, Event.ENCODING_STRINGS[encoding]).length; + // try to find the length of the encoded string in the cache if it is + // already calculated before, otherwise calculate it and update the cache. + Integer serializedLength = encodedStringLengthCache.get(string); + if(serializedLength == null) { + serializedLength = EncodedString.getEncodedLength(string); + encodedStringLengthCache.putIfAbsent(string, serializedLength); + } if (serializedLength > maxLength) { throw new EventSystemException( "String " + string + " was longer than maximum length: " + serializedLength + " > " + maxLength); diff --git a/src/main/java/org/lwes/Event.java b/src/main/java/org/lwes/Event.java index bc3b639..a879006 100644 --- a/src/main/java/org/lwes/Event.java +++ b/src/main/java/org/lwes/Event.java @@ -21,12 +21,11 @@ import java.util.Iterator; import java.util.Set; -import org.lwes.util.CharacterEncoding; import org.lwes.util.IPAddress; public interface Event extends Iterable { - static final int MAX_EVENT_NAME_SIZE = 127; - static final int MAX_FIELD_NAME_SIZE = 255; + public static final int MAX_EVENT_NAME_SIZE = 127; + public static final int MAX_FIELD_NAME_SIZE = 255; public static final int MAX_MESSAGE_SIZE = 65507; /** @@ -37,13 +36,9 @@ public interface Event extends Iterable { static final String SENDER_IP = "SenderIP"; static final String SENDER_PORT = "SenderPort"; - /** - * Encoding variables - */ - static final short ISO_8859_1 = 0; - static final short UTF_8 = 1; - static final short DEFAULT_ENCODING = UTF_8; - static final CharacterEncoding[] ENCODING_STRINGS = {CharacterEncoding.ISO_8859_1, CharacterEncoding.UTF_8}; + // supported encoding + public static final short UTF_8 = 1; + public static final String UTF_8_NAME = "UTF-8"; // SETTERS @@ -123,7 +118,7 @@ public interface Event extends Iterable { void setFloatArray(String attributeName, Float[] value); - void setEncoding(short encoding); + void setEncoding(); // GETTERS @@ -209,8 +204,6 @@ public interface Event extends Iterable { IPAddress getIPAddressObj(String attributeName); - short getEncoding(); - // SERIALIZATION byte[] serialize(); diff --git a/src/main/java/org/lwes/EventFactory.java b/src/main/java/org/lwes/EventFactory.java index 3c0e711..f88b287 100644 --- a/src/main/java/org/lwes/EventFactory.java +++ b/src/main/java/org/lwes/EventFactory.java @@ -119,19 +119,7 @@ public void initialize() throws EventSystemException { * @throws EventSystemException if there is a problem creating the event */ public Event createEvent(String eventName) throws EventSystemException { - return createEvent(eventName, Event.DEFAULT_ENCODING); - } - - /** - * Create a validated event named eventName with specified encoding. - * - * @param eventName the name of the event - * @param encoding the encoding to use - * @return the Event object - * @throws EventSystemException if there is a problem creating the event - */ - public Event createEvent(String eventName, short encoding) throws EventSystemException { - return createEvent(eventName, true, encoding); + return createEvent(eventName, true); } /** @@ -143,23 +131,10 @@ public Event createEvent(String eventName, short encoding) throws EventSystemExc * @throws EventSystemException if there is a problem creating the event */ public Event createEvent(String eventName, boolean validate) throws EventSystemException { - return createEvent(eventName, validate, Event.DEFAULT_ENCODING); - } - - /** - * Create an event named eventName with optional validation and specified encoding - * - * @param eventName the name of the event - * @param validate whether or not to validate the event against the EventTemplateDB - * @param encoding the encoding to use - * @return the Event object - * @throws EventSystemException if there is a problem creating the event - */ - public Event createEvent(String eventName, boolean validate, short encoding) throws EventSystemException { if (validate && !eventTemplateDBInit) { throw new EventSystemException("Event template db not initialized"); } - return new MapEvent(eventName, validate, eventTemplateDB, encoding); + return new MapEvent(eventName, validate, eventTemplateDB); } /** diff --git a/src/main/java/org/lwes/MapEvent.java b/src/main/java/org/lwes/MapEvent.java index 0db5336..5c89756 100644 --- a/src/main/java/org/lwes/MapEvent.java +++ b/src/main/java/org/lwes/MapEvent.java @@ -34,7 +34,6 @@ public class MapEvent extends DefaultEvent { private final ConcurrentHashMap attributes = new ConcurrentHashMap(); private String name = null; private EventTemplateDB eventTemplateDB = null; - private short encoding = DEFAULT_ENCODING; /** * If this is set to true, types and attributes are validated against the EventTemplateDB @@ -91,26 +90,11 @@ public MapEvent(String eventName, EventTemplateDB eventTemplateDB) */ public MapEvent(String eventName, boolean validate, EventTemplateDB eventTemplateDB) throws EventSystemException { - this(eventName, validate, eventTemplateDB, DEFAULT_ENCODING); - } - - /** - * Create an event called eventName - * - * @param eventName the name of the event - * @param validate true if the EventTemplateDB should be checked for types before all mutations - * @param encoding the character encoding used by the event - * @throws NoSuchEventException if the Event does not exist in the EventTemplateDB - * @throws NoSuchAttributeException if an attribute does not exist in the EventTemplateDB - * @throws NoSuchAttributeTypeException if an attribute type does not exist in the EventTemplateDB - */ - public MapEvent(String eventName, boolean validate, EventTemplateDB eventTemplateDB, short encoding) - throws EventSystemException { - checkShortStringLength(eventName, encoding, MAX_EVENT_NAME_SIZE); + checkShortStringLength(eventName, MAX_EVENT_NAME_SIZE); setEventTemplateDB(eventTemplateDB); validating = validate; setEventName(eventName); - setEncoding(encoding); + setEncoding(); setDefaultValues(eventTemplateDB); } @@ -158,7 +142,6 @@ public void reset() { validating = false; eventTemplateDB = null; attributes.clear(); - encoding = DEFAULT_ENCODING; if (state != null) { state.reset(); } @@ -264,8 +247,8 @@ public synchronized String getEventName() { */ @Override public synchronized void setEventName(String name) { - checkShortStringLength(name, encoding, MAX_EVENT_NAME_SIZE); - + checkShortStringLength(name, MAX_EVENT_NAME_SIZE); + /* determine if we already have the name and are just resetting it */ if (this.name != null) { bytesStoreSize -= (this.name.length() + 1 + 2); @@ -276,27 +259,15 @@ public synchronized void setEventName(String name) { this.name = name; } - /** - * Get the character encoding for this event - * - * @return the encoding - */ - @Override - public short getEncoding() { - return this.encoding; - } - /** * Set the character encoding for event strings * - * @param encoding the character encoding * @throws NoSuchAttributeTypeException if the type for the encoding attribute does not exist * @throws NoSuchAttributeException if the encoding attribute does not exist */ @Override - public void setEncoding(short encoding) throws EventSystemException { - this.encoding = encoding; - setInt16(ENCODING, this.encoding); + public void setEncoding() throws EventSystemException { + setInt16(ENCODING, UTF_8); } /** @@ -320,7 +291,7 @@ public Object get(String attributeName) { public void clear(String attributeName) { final BaseType bt = attributes.remove(attributeName); if (bt != null) { - bytesStoreSize -= (attributeName.length() + 1) + bt.bytesStoreSize(encoding); + bytesStoreSize -= (attributeName.length() + 1) + bt.bytesStoreSize(); } } @@ -356,7 +327,7 @@ public void set(String attribute, FieldType type, Object value) throws EventSyst * @throws NoSuchAttributeTypeException if there is an attribute with an undefined type */ private void set(String attribute, BaseType bt) { - checkShortStringLength(attribute, encoding, MAX_FIELD_NAME_SIZE); + checkShortStringLength(attribute, MAX_FIELD_NAME_SIZE); if (isValidating() && getEventTemplateDB() != null) { if (getEventTemplateDB().checkForAttribute(name, attribute)) { @@ -374,16 +345,16 @@ private void set(String attribute, BaseType bt) { // Remove the existing value, and record the reduction in the serialized size. final BaseType oldObject = attributes.remove(attribute); if (oldObject != null) { - bytesStoreSize -= (attribute.length() + 1) + oldObject.bytesStoreSize(encoding); + bytesStoreSize -= (attribute.length() + 1) + oldObject.bytesStoreSize(); } if (bt.getTypeObject() != null) { - int newSize = bytesStoreSize + ((attribute.length() + 1) + bt.bytesStoreSize(encoding)); + int newSize = bytesStoreSize + ((attribute.length() + 1) + bt.bytesStoreSize()); if (newSize > MAX_MESSAGE_SIZE) { throw new EventSystemException("Event size limit is " + MAX_MESSAGE_SIZE + " bytes."); } - bytesStoreSize += (attribute.length() + 1) + bt.bytesStoreSize(encoding); + bytesStoreSize += (attribute.length() + 1) + bt.bytesStoreSize(); attributes.put(attribute, bt); } } @@ -407,7 +378,6 @@ public int serialize(byte[] bytes, int offset) { */ int pos = 0; int attributeCount = attributes.size(); - short encoding = DEFAULT_ENCODING; pos += Serializer.serializeEVENTWORD(name, bytes, pos); pos += Serializer.serializeUINT16((short) (attributeCount), bytes, pos); @@ -421,7 +391,7 @@ public int serialize(byte[] bytes, int offset) { FieldType encodingType = encodingBase.getType(); if (encodingObj != null) { if (encodingType == FieldType.INT16) { - encoding = (Short) encodingObj; + short encoding = UTF_8; // ignore encodingObj, always set to UTF-8 if (log.isTraceEnabled()) { log.trace("Character encoding: " + encoding); } @@ -458,7 +428,7 @@ public int serialize(byte[] bytes, int offset) { pos += Serializer.serializeATTRIBUTEWORD(key, bytes, pos); pos += Serializer.serializeBYTE(type.token, bytes, pos); - pos += Serializer.serializeValue(type, data, encoding, bytes, pos); + pos += Serializer.serializeValue(type, data, bytes, pos); if (log.isTraceEnabled()) { log.trace("Serialized attribute " + key); @@ -526,7 +496,8 @@ public void deserialize(byte[] bytes, int offset, int length) if (attribute != null) { if (i == 0 && attribute.equals(ENCODING)) { if (type == FieldType.INT16) { - setEncoding(Deserializer.deserializeINT16(state, bytes)); + Deserializer.deserializeINT16(state, bytes); + setEncoding(); // ignore the encoding specified, always set to utf-8 continue; } else { @@ -534,7 +505,7 @@ public void deserialize(byte[] bytes, int offset, int length) } } - set(attribute, type, Deserializer.deserializeValue(state, bytes, type, encoding)); + set(attribute, type, Deserializer.deserializeValue(state, bytes, type)); } if (bytesStoreSize != state.currentIndex() - offset) { throw new EventSystemException("Deserializing " + type + " field " + attribute + diff --git a/src/main/java/org/lwes/serializer/Deserializer.java b/src/main/java/org/lwes/serializer/Deserializer.java index 72d3d32..4f6e7be 100644 --- a/src/main/java/org/lwes/serializer/Deserializer.java +++ b/src/main/java/org/lwes/serializer/Deserializer.java @@ -20,7 +20,6 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.lwes.Event; import org.lwes.EventSystemException; import org.lwes.FieldType; import org.lwes.util.EncodedString; @@ -269,12 +268,11 @@ public static String deserializeIPADDRtoHexString(DeserializerState myState, } public static String[] deserializeStringArray(DeserializerState state, - byte[] bytes, - short encoding) { + byte[] bytes) { int length = deserializeUINT16(state, bytes); String[] rtn = new String[length]; for (int i = 0; i < length; i++) { - rtn[i] = deserializeSTRING(state, bytes, encoding); + rtn[i] = deserializeSTRING(state, bytes); } return rtn; } @@ -396,15 +394,8 @@ public static float[] deserializeFloatArray(DeserializerState state, * in the byte array bytes * @param bytes the bytes to deserialize * @return a String. - * @deprecated */ - @Deprecated public static String deserializeSTRING(DeserializerState myState, byte[] bytes) { - return deserializeSTRING(myState, bytes, Event.DEFAULT_ENCODING); - } - - public static String deserializeSTRING(DeserializerState myState, - byte[] bytes, short encoding) { String aString = null; int len = -1; try { @@ -417,8 +408,7 @@ public static String deserializeSTRING(DeserializerState myState, // log.trace("State: " + myState); // } - aString = EncodedString.bytesToString(bytes, myState.currentIndex(), len, - Event.ENCODING_STRINGS[encoding]); + aString = EncodedString.decode(bytes, myState.currentIndex(), len); myState.incr(len); } catch (ArrayIndexOutOfBoundsException aioobe) { @@ -444,11 +434,6 @@ public static String deserializeSTRING(DeserializerState myState, */ public static String deserializeEVENTWORD(DeserializerState myState, byte[] bytes) { - return deserializeEVENTWORD(myState, bytes, Event.DEFAULT_ENCODING); - } - - public static String deserializeEVENTWORD(DeserializerState myState, - byte[] bytes, short encoding) { String aString = null; int len = -1; try { @@ -461,8 +446,7 @@ public static String deserializeEVENTWORD(DeserializerState myState, // log.trace("State: " + myState); // } - aString = EncodedString.bytesToString(bytes, myState.currentIndex(), len, - Event.ENCODING_STRINGS[encoding]); + aString = EncodedString.decode(bytes, myState.currentIndex(), len); myState.incr(len); } catch (ArrayIndexOutOfBoundsException aioobe) { @@ -488,7 +472,7 @@ public static String deserializeEVENTWORD(DeserializerState myState, */ public static String deserializeATTRIBUTEWORD(DeserializerState myState, byte[] bytes) { - return deserializeEVENTWORD(myState, bytes, Event.DEFAULT_ENCODING); + return deserializeEVENTWORD(myState, bytes); } public static BitSet deserializeBitSet(DeserializerState myState, byte[] bytes) { @@ -531,15 +515,14 @@ public static int deserializeBitSetCount(DeserializerState myState, byte[] bytes } public static String[] deserializeNStringArray(DeserializerState state, - byte[] bytes, - short encoding) { + byte[] bytes) { int length = deserializeUINT16(state, bytes); BitSet bs = deserializeBitSet(state, bytes); String[] rtn = new String[length]; for (int i = 0; i < length; i++) { if (bs.get(i)) { - rtn[i] = deserializeSTRING(state, bytes, encoding); + rtn[i] = deserializeSTRING(state, bytes); } else { rtn[i] = null; @@ -712,8 +695,7 @@ public static Byte[] deserializeNByteArray(DeserializerState state, public static Object deserializeValue(DeserializerState state, byte[] bytes, - FieldType type, - short encoding) throws EventSystemException { + FieldType type) throws EventSystemException { switch (type) { case BOOLEAN: return Deserializer.deserializeBOOLEAN(state, bytes); @@ -736,11 +718,11 @@ public static Object deserializeValue(DeserializerState state, case DOUBLE: return Deserializer.deserializeDOUBLE(state, bytes); case STRING: - return Deserializer.deserializeSTRING(state, bytes, encoding); + return Deserializer.deserializeSTRING(state, bytes); case IPADDR: return Deserializer.deserializeIPADDR(state, bytes); case STRING_ARRAY: - return Deserializer.deserializeStringArray(state, bytes, encoding); + return Deserializer.deserializeStringArray(state, bytes); case INT16_ARRAY: return Deserializer.deserializeInt16Array(state, bytes); case INT32_ARRAY: @@ -780,7 +762,7 @@ public static Object deserializeValue(DeserializerState state, case NBOOLEAN_ARRAY: return Deserializer.deserializeNBooleanArray(state, bytes); case NSTRING_ARRAY: - return Deserializer.deserializeNStringArray(state, bytes, encoding); + return Deserializer.deserializeNStringArray(state, bytes); case NUINT64_ARRAY: return Deserializer.deserializeNUInt64Array(state, bytes); case NBYTE_ARRAY: diff --git a/src/main/java/org/lwes/serializer/Serializer.java b/src/main/java/org/lwes/serializer/Serializer.java index 61fea37..944780c 100644 --- a/src/main/java/org/lwes/serializer/Serializer.java +++ b/src/main/java/org/lwes/serializer/Serializer.java @@ -19,8 +19,9 @@ import java.util.BitSet; import java.util.List; -import org.lwes.Event; import org.lwes.FieldType; +import org.lwes.MemoryPool; +import org.lwes.MemoryPool.Buffer; import org.lwes.util.EncodedString; import org.lwes.util.IPAddress; import org.lwes.util.NumberCodec; @@ -131,26 +132,16 @@ public static int serializeUINT64(BigInteger anInt, byte[] bytes, int offset) { return (8); } - /** - * @deprecated - */ - @Deprecated public static int serializeSTRING(String aString, byte[] bytes, int offset) { - return serializeSTRING(aString, bytes, offset, Event.DEFAULT_ENCODING); - } - - public static int serializeSTRING(String aString, byte[] bytes, int offset, - short encoding) { - byte[] stringBytes = - EncodedString.getBytes(aString, Event.ENCODING_STRINGS[encoding]); - int length = stringBytes.length; + Buffer buffer = EncodedString.encode(aString); + byte[] stringBytes = buffer.getEncoderOutputBuffer().array(); + int length = buffer.getEncoderOutputBuffer().position(); if (length < 65535 && length >= 0) { offset += serializeUINT16(length, bytes, offset); System.arraycopy(stringBytes, 0, bytes, offset, length); - return (length + 2); } - return 0; - + MemoryPool.putBack(buffer); + return (length < 65535 && length >= 0) ? (length + 2) : 0; } /** @@ -161,20 +152,18 @@ public static int serializeSTRING(String aString, byte[] bytes, int offset, * @param value * @param bytes * @param offset - * @param encoding * @return the offset */ public static int serializeStringArray(String[] value, byte[] bytes, - int offset, - short encoding) { + int offset) { int numbytes = 0; int offsetStart = offset; numbytes = serializeUINT16(value.length, bytes, offset); offset += numbytes; for (String s : value) { - numbytes = serializeSTRING(s, bytes, offset, encoding); + numbytes = serializeSTRING(s, bytes, offset); offset += numbytes; } return (offset - offsetStart); @@ -331,28 +320,19 @@ public static int serializeFloatArray(float[] value, byte[] bytes, int offset) { } public static int serializeEVENTWORD(String aString, byte[] bytes, int offset) { - return serializeEVENTWORD(aString, bytes, offset, Event.DEFAULT_ENCODING); - } - - private static int serializeEVENTWORD(String aString, - byte[] bytes, - int offset, - short encoding) { - byte[] stringBytes = - EncodedString.getBytes(aString, Event.ENCODING_STRINGS[encoding]); - int length = stringBytes.length; + Buffer buffer = EncodedString.encode(aString); + int length = buffer.getEncoderOutputBuffer().position(); if (0 <= length && length <= 255) { offset += serializeUBYTE((short) length, bytes, offset); - System.arraycopy(stringBytes, 0, bytes, offset, length); - return (length + 1); + System.arraycopy(buffer.getEncoderOutputBuffer().array(), 0, bytes, offset, length); } - return 0; - + MemoryPool.putBack(buffer); + return (0 <= length && length <= 255) ? (length+1) : 0; } public static int serializeATTRIBUTEWORD(String aString, byte[] bytes, int offset) { - return serializeEVENTWORD(aString, bytes, offset, Event.DEFAULT_ENCODING); + return serializeEVENTWORD(aString, bytes, offset); } /** @@ -412,7 +392,6 @@ public static int serializeIPV4(InetAddress ip, byte[] bytes, int offset) { public static int serializeValue(FieldType type, Object data, - short encoding, byte[] bytes, final int offset) { switch (type) { @@ -433,7 +412,7 @@ public static int serializeValue(FieldType type, case INT64: return Serializer.serializeINT64((Long) data, bytes, offset); case STRING: - return Serializer.serializeSTRING(((String) data), bytes, offset, encoding); + return Serializer.serializeSTRING(((String) data), bytes, offset); case DOUBLE: return Serializer.serializeDOUBLE(((Double) data), bytes, offset); case FLOAT: @@ -441,8 +420,7 @@ public static int serializeValue(FieldType type, case IPADDR: return Serializer.serializeIPADDR(((IPAddress) data), bytes, offset); case STRING_ARRAY: - return Serializer.serializeStringArray - (((String[]) data), bytes, offset, encoding); + return Serializer.serializeStringArray (((String[]) data), bytes, offset); case NUINT32_ARRAY: return Serializer.serializeNUInt32Array((Long[]) data, bytes, offset); case NINT32_ARRAY: @@ -486,7 +464,7 @@ public static int serializeValue(FieldType type, case IP_ADDR_ARRAY: return Serializer.serializeIPADDRArray((IPAddress[]) data, bytes, offset); case NSTRING_ARRAY: - return Serializer.serializeNStringArray((String[]) data, bytes, offset, encoding); + return Serializer.serializeNStringArray((String[]) data, bytes, offset); } @@ -531,10 +509,9 @@ public static int serializeBitSet(BitSet bitSet, int arrayLength, byte[] data, i * @param data array to serialize * @param bytes byte array to write to * @param offset index in byte array to start at - * @param encoding encoding to use for strings * @return number of bytes written */ - public static int serializeNStringArray(String[] data, byte[] bytes, int offset, short encoding) { + public static int serializeNStringArray(String[] data, byte[] bytes, int offset) { int numbytes = 0; int offsetStart = offset; @@ -550,10 +527,13 @@ public static int serializeNStringArray(String[] data, byte[] bytes, int offset, int i = 0; for (String s : data) { if (s != null) { - byte[] stringBytes = EncodedString.getBytes(s, Event.ENCODING_STRINGS[encoding]); - byte[] tmpArr = new byte[stringBytes.length + 2]; - serializeUINT16(stringBytes.length, tmpArr, 0); - System.arraycopy(stringBytes, 0, tmpArr, 2, stringBytes.length); + Buffer buffer = EncodedString.encode(s); + byte[] stringBytes = buffer.getEncoderOutputBuffer().array(); + int length = buffer.getEncoderOutputBuffer().position(); + byte[] tmpArr = new byte[length + 2]; + serializeUINT16(length, tmpArr, 0); + System.arraycopy(stringBytes, 0, tmpArr, 2, length); + MemoryPool.putBack(buffer); tmp.add(i, tmpArr); bitSet.set(i); } diff --git a/src/main/java/org/lwes/util/CharacterEncoding.java b/src/main/java/org/lwes/util/CharacterEncoding.java deleted file mode 100644 index f00fdb9..0000000 --- a/src/main/java/org/lwes/util/CharacterEncoding.java +++ /dev/null @@ -1,178 +0,0 @@ -/*======================================================================* - * Copyright (c) 2008, Yahoo! Inc. All rights reserved. * - * * - * Licensed under the New BSD License (the "License"); you may not use * - * this file except in compliance with the License. Unless required * - * by applicable law or agreed to in writing, software distributed * - * under the License is distributed on an "AS IS" BASIS, WITHOUT * - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * - * See the License for the specific language governing permissions and * - * limitations under the License. See accompanying LICENSE file. * - *======================================================================*/ - -package org.lwes.util; - -import java.io.UnsupportedEncodingException; -import java.util.HashMap; - -/** - * This is a little class to abstract the character encoding strings that Java - * uses into classes which can be checked at compile time. - * - * @author Kevin Scaldeferri - */ -public abstract class CharacterEncoding { - - // static loading - private static final HashMap ENCODING_HASH = new HashMap(); - private static final String CANONICAL_ASCII_NAME = "ASCII"; - private static final String CANONICAL_ISO_8859_1_NAME = "ISO8859_1"; - private static final String CANONICAL_UTF_8_NAME = "UTF8"; - private static final String CANONICAL_SHIFT_JIS_NAME = "SJIS"; - private static final String CANONICAL_EUC_JP_NAME = "EUC_JP"; - private static final String CANONICAL_EUC_KR_NAME = "EUC_KR"; - private static final String[] ASCII_ALIASES = { CANONICAL_ASCII_NAME, - "US-ASCII", "ISO646-US" }; - private static final String[] ISO_8859_1_ALIASES = { - CANONICAL_ISO_8859_1_NAME, "ISO-8859-1", "ISO-LATIN-1", "8859_1" }; - private static final String[] UTF_8_ALIASES = { CANONICAL_UTF_8_NAME, - "UTF-8" }; - private static final String[] SHIFT_JIS_ALIASES = { - CANONICAL_SHIFT_JIS_NAME, "SHIFTJIS", "SHIFT-JIS", "SHIFT_JIS" }; - private static final String[] EUC_JP_ALIASES = { CANONICAL_EUC_JP_NAME, - "EUC-JP" }; - private static final String[] EUC_KR_ALIASES = { CANONICAL_EUC_KR_NAME, - "EUC-KR" }; - - /* - * Returns Java's canonical form of the encoding. - */ - public abstract String getEncodingString(); - - /* - * Returns the official IANA name for the encoding. Everything expect Java - * expects this form. - */ - public abstract String getIANAString(); - - public static final CharacterEncoding ISO_8859_1 = new CharacterEncoding() { - @Override - public String getEncodingString() { - return "ISO-8859-1"; - } - - @Override - public String getIANAString() { - return "ISO-8859-1"; - } - }; - - public static final CharacterEncoding UTF_8 = new CharacterEncoding() { - @Override - public String getEncodingString() { - return "UTF-8"; - } - - @Override - public String getIANAString() { - return "UTF-8"; - } - }; - - public static final CharacterEncoding ASCII = new CharacterEncoding() { - @Override - public String getEncodingString() { - return "ASCII"; - } - - @Override - public String getIANAString() { - return "US-ASCII"; - } - }; - - public static final CharacterEncoding SHIFT_JIS = new CharacterEncoding() { - @Override - public String getEncodingString() { - return "SJIS"; - } - - @Override - public String getIANAString() { - return "Shift_JIS"; - } - }; - - public static final CharacterEncoding EUC_JP = new CharacterEncoding() { - @Override - public String getEncodingString() { - return "EUC_JP"; - } - - @Override - public String getIANAString() { - return "EUC-JP"; - } - }; - - public static final CharacterEncoding EUC_KR = new CharacterEncoding() { - @Override - public String getEncodingString() { - return "EUC_KR"; - } - - @Override - public String getIANAString() { - return "EUC-KR"; - } - }; - - /** - * This is a highly limited implementation at the moment, so don't expect - * too much from it. - * @param enc the String representation of the encoding. - * @return CharacterEncoding - * @throws java.io.UnsupportedEncodingException if the encoding doesnt exist. - */ - public static CharacterEncoding getInstance(String enc) - throws UnsupportedEncodingException { - if (ENCODING_HASH.containsKey(enc.toUpperCase())) { - return ENCODING_HASH.get(enc.toUpperCase()); - } else { - throw new UnsupportedEncodingException(enc); - } - } - - @Override - public int hashCode() { - return getEncodingString().hashCode(); - } - - @Override - public boolean equals(Object o) { - return (o instanceof CharacterEncoding) - && getEncodingString().equals(((CharacterEncoding) o).getEncodingString()); - } - - static { - int i; - for (i = 0; i < ASCII_ALIASES.length; i++) { - ENCODING_HASH.put(ASCII_ALIASES[i], ASCII); - } - for (i = 0; i < ISO_8859_1_ALIASES.length; i++) { - ENCODING_HASH.put(ISO_8859_1_ALIASES[i], ISO_8859_1); - } - for (i = 0; i < UTF_8_ALIASES.length; i++) { - ENCODING_HASH.put(UTF_8_ALIASES[i], UTF_8); - } - for (i = 0; i < SHIFT_JIS_ALIASES.length; i++) { - ENCODING_HASH.put(SHIFT_JIS_ALIASES[i], SHIFT_JIS); - } - for (i = 0; i < EUC_JP_ALIASES.length; i++) { - ENCODING_HASH.put(EUC_JP_ALIASES[i], EUC_JP); - } - for (i = 0; i < EUC_KR_ALIASES.length; i++) { - ENCODING_HASH.put(EUC_KR_ALIASES[i], EUC_KR); - } - } -} diff --git a/src/main/java/org/lwes/util/EncodedString.java b/src/main/java/org/lwes/util/EncodedString.java index c2f42cc..b8d823b 100644 --- a/src/main/java/org/lwes/util/EncodedString.java +++ b/src/main/java/org/lwes/util/EncodedString.java @@ -12,7 +12,17 @@ package org.lwes.util; -import java.io.UnsupportedEncodingException; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.Charset; +import java.nio.charset.CharsetDecoder; +import java.nio.charset.CharsetEncoder; +import java.nio.charset.CoderResult; + +import org.lwes.MemoryPool; +import org.lwes.MemoryPool.Buffer; + +import static org.lwes.Event.UTF_8_NAME; /** * EncodedString is a wrapper class which wraps a String, but replaces all @@ -25,63 +35,126 @@ * @since 0.0.1 */ public class EncodedString { - private String myString; - private CharacterEncoding myEncoding; - - public static String bytesToString(byte[] bytes, CharacterEncoding enc) { - if (bytes == null) { - return null; - } - - try { - return new String(bytes, enc.getEncodingString()); - } catch (UnsupportedEncodingException e) { - throw new IllegalArgumentException("Unknown Encoding"); - } - } - - public static String bytesToString(byte[] bytes, int offset, int length, CharacterEncoding enc) { - if (bytes == null) { - return null; - } - - try { - return new String(bytes, offset, length, enc.getEncodingString()); - } catch (UnsupportedEncodingException e) { - throw new IllegalArgumentException("Unknown Encoding"); - } - } - - public static byte[] getBytes(String string, CharacterEncoding enc) { - if (string == null) { - return null; - } - - try { - return string.getBytes(enc.getEncodingString()); - } catch (UnsupportedEncodingException e) { - throw new IllegalArgumentException("Unknown Encoding"); - } - } - - public EncodedString(String string, CharacterEncoding enc) { - myString = string; - myEncoding = enc; - } - - public EncodedString(byte[] bytes, CharacterEncoding enc) { - myString = bytesToString(bytes, enc); - myEncoding = enc; - } - - public EncodedString(byte[] bytes, int offset, - int length, CharacterEncoding enc) - { - myString = bytesToString(bytes, offset, length, enc); - myEncoding = enc; - } - - public byte[] getBytes() { return getBytes(myString, myEncoding); } - @Override - public String toString() { return myString; } + + // encoder object, one per thread + static protected ThreadLocal encoder = + new ThreadLocal() { + @Override protected CharsetEncoder initialValue() { + return Charset.forName(UTF_8_NAME).newEncoder(); + } + }; + + // decoder object, one per thread + static protected ThreadLocal decoder = + new ThreadLocal() { + @Override protected CharsetDecoder initialValue() { + return Charset.forName(UTF_8_NAME).newDecoder(); + } + }; + + + /** + * return a {@code Buffer} object that contains the encoded version of the + * input string. The returned {@code Buffer} object must be put back in to + * memory pool when no longer needed. + * + * @param input - input string to encode + * @return a {@code Buffer} object that contains the encoded input + * + * @throws IllegalArgumentException if encoding fails because of overflow of + * the buffer, or any other encoding failures. + */ + public static Buffer encode(String input) { + // get a buffer from memory pool + Buffer buffer = MemoryPool.get(); + ByteBuffer outputBuffer = buffer.getEncoderOutputBuffer(); + CharBuffer inputBuffer = buffer.getEncoderInputBuffer(); + + // no need to encode if input is null + if(input == null) { + inputBuffer.limit(0); + return buffer; + } + + if(input.length() > inputBuffer.capacity()) { + MemoryPool.putBack(buffer); + throw new IllegalArgumentException("Failed to encode the input, input is too long, size: " + input.length()); + } + + inputBuffer.limit(input.length()); + + // copy input string + char[] tempChars = inputBuffer.array(); + input.getChars(0, input.length(), tempChars, 0); + // reset encoder and then encode + encoder.get().reset(); + CoderResult result = encoder.get().encode(inputBuffer, outputBuffer, true); + if(result != CoderResult.UNDERFLOW) { + // return the buffer back to the pool + MemoryPool.putBack(buffer); + throw new IllegalArgumentException("Failed to encode the input, code: " + result); + } + encoder.get().flush(outputBuffer); + return buffer; + } + + /** + * return the length of a string when encoded in utf-8 + * + * @param input - input string + * @return length of the encoded input string in utf-8, or 0 if input is null + * + * @throws IllegalArgumentException if encoding fails because of overflow of + * the buffer, or any other encoding failures. + */ + public static int getEncodedLength(String input) { + if(input == null) { + return 0; + } + // encode and put the buffer back in to the memory pool + Buffer buffer = encode(input); + int len = buffer.getEncoderOutputBuffer().position(); + MemoryPool.putBack(buffer); + return len; + } + + /** + * decodes an array of bytes to a string using utf-8 + * + * @param input - input array of bytes + * @param offset - offset in the array + * @param length - length + * @return a string representing decoded bytes, or empty string if the length + * of the input is 0 + */ + public static String decode(byte[] input, int offset, int length) { + // nothing to decode if empty bytes + if(input == null) { + return null; + } + if(length == 0) { + return ""; + } + + Buffer buffer = MemoryPool.get(); + CharBuffer outputBuffer = buffer.getDecoderOutputBuffer(); + ByteBuffer inputBuffer = ByteBuffer.wrap(input, offset, length); + + // reset decoder + decoder.get().reset(); + + CoderResult result = decoder.get().decode(inputBuffer, outputBuffer, true); + if(result != CoderResult.UNDERFLOW) { + // put the buffer back in to the pool + MemoryPool.putBack(buffer); + throw new IllegalArgumentException("Failed to decode the input, code: " + result); + } + decoder.get().flush(outputBuffer); + String output = new String(outputBuffer.array(), 0, outputBuffer.position()); + + MemoryPool.putBack(buffer); + + return output; + } + } diff --git a/src/main/java/org/lwes/util/NumberCodec.java b/src/main/java/org/lwes/util/NumberCodec.java index c8d4afa..7ed58a9 100644 --- a/src/main/java/org/lwes/util/NumberCodec.java +++ b/src/main/java/org/lwes/util/NumberCodec.java @@ -14,6 +14,8 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.lwes.MemoryPool; +import org.lwes.MemoryPool.Buffer; import java.math.BigInteger; @@ -562,8 +564,9 @@ public static byte[] hexStringToByteArray(String aString) { for (int k = 0; k < (length / 2); k++) { bytes[k] = (byte) 0; } - byte[] str_bytes = aString.getBytes(); - if (str_bytes.length != length) { + Buffer buffer = EncodedString.encode(aString); + byte[] str_bytes = buffer.getEncoderOutputBuffer().array(); + if (buffer.getEncoderOutputBuffer().position() != length) { log.error("ERROR: Mismatching lengths"); return null; } @@ -759,9 +762,11 @@ public static byte[] hexStringToByteArray(String aString) { default: log.error("ERROR: non-hex character"); + MemoryPool.putBack(buffer); return null; } } + MemoryPool.putBack(buffer); return bytes; } @@ -879,32 +884,32 @@ public static byte[] encodeLong(long pLong) { * @param pBytes the byte array from which the encoded form should be read * @return int decoded from bytes */ - public static int decodeInt (byte[] pBytes) - throws NumberFormatException - { - if (pBytes == null) - { - throw new NumberFormatException("null byte array passed"); - } - if (pBytes.length != NumberCodec.LONG_BYTES) - { - throw new NumberFormatException("expecting byte array length of: " + - NumberCodec.INT_BYTES + " got: " + pBytes.length); - } - return NumberCodec.decodeInt(pBytes, 0, pBytes.length); - } - - /** + public static int decodeInt (byte[] pBytes) + throws NumberFormatException + { + if (pBytes == null) + { + throw new NumberFormatException("null byte array passed"); + } + if (pBytes.length != NumberCodec.LONG_BYTES) + { + throw new NumberFormatException("expecting byte array length of: " + + NumberCodec.INT_BYTES + " got: " + pBytes.length); + } + return NumberCodec.decodeInt(pBytes, 0, pBytes.length); + } + + /** * Encode a int into a byte-array buffer. (convienience method) * * @param pInt the int to be encoded * @return encoded bytes of the int */ - public static byte[] encodeInt (int pInt) - throws NumberFormatException - { - byte[] bytes = new byte[NumberCodec.INT_BYTES]; - NumberCodec.encodeInt(pInt, bytes, 0, bytes.length); - return bytes; - } + public static byte[] encodeInt (int pInt) + throws NumberFormatException + { + byte[] bytes = new byte[NumberCodec.INT_BYTES]; + NumberCodec.encodeInt(pInt, bytes, 0, bytes.length); + return bytes; + } } diff --git a/src/test/java/org/lwes/ArrayEventPerformanceTest.java b/src/test/java/org/lwes/ArrayEventPerformanceTest.java index 29ada8d..b7d2234 100644 --- a/src/test/java/org/lwes/ArrayEventPerformanceTest.java +++ b/src/test/java/org/lwes/ArrayEventPerformanceTest.java @@ -19,7 +19,7 @@ public class ArrayEventPerformanceTest { private static final Log LOG = LogFactory.getLog(ArrayEventPerformanceTest.class); - private static final int NUM_EVENTS = 100, NUM_PASSES = 10000; + private static final int NUM_EVENTS = 500, NUM_PASSES = 10000; private static double CPU_SCALE; // used to reduce CPU-dependent effects private static final double TOLERANCE = 1.5; private ArrayEvent[] events; @@ -27,11 +27,11 @@ public class ArrayEventPerformanceTest { private static ThreadMXBean tmx; private long t0; private Map stats0; - + // Change these values as performance shifts. If ArrayEvent gets faster, lower // them. If we are forced to accept it getting slower, raise them. private static final double DIRECT_GET_CPU_TIME = 13500; - + @BeforeClass public static void beforeClass() { tmx = ManagementFactory.getThreadMXBean(); @@ -65,12 +65,12 @@ public void before() { t0 = tmx.getCurrentThreadCpuTime(); stats0 = ArrayEvent.getStatsSnapshot(); } - + @After public void after() { events = null; } - + @AfterClass public static void afterClass() { tmx = null; @@ -79,7 +79,7 @@ public static void afterClass() { @Test public void direct() throws Exception { t0 = tmx.getCurrentThreadCpuTime(); - + for (int p=0; p stats = getStatsChanges(stats0); assertEquals(0, stats.get(ArrayEventStats.CREATIONS).intValue()); @@ -95,7 +95,7 @@ public void direct() throws Exception { assertEquals(0, stats.get(ArrayEventStats.SWAPS).intValue()); assertEquals(3*(numFields-NUM_EVENTS)*NUM_PASSES, stats.get(ArrayEventStats.PARSES).intValue()); assertEquals(3*NUM_PASSES*NUM_EVENTS, stats.get(ArrayEventStats.FINDS).intValue()); - + final long dt = tmx.getCurrentThreadCpuTime() - t0; LOG.info(String.format( "get() unpacked %d fields from %d events averaging %1.1f fields %d times in %f seconds, or %f ns/event, or %f ns/field", diff --git a/src/test/java/org/lwes/ArrayEventTest.java b/src/test/java/org/lwes/ArrayEventTest.java index 9f848ba..043dae9 100644 --- a/src/test/java/org/lwes/ArrayEventTest.java +++ b/src/test/java/org/lwes/ArrayEventTest.java @@ -16,6 +16,8 @@ import org.junit.Test; import org.lwes.ArrayEvent.ArrayEventStats; +import static org.lwes.Event.UTF_8; + import junit.framework.Assert; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -24,24 +26,11 @@ public final class ArrayEventTest extends EventTest { - @Test(expected = IllegalArgumentException.class) - public void testInvalidEncoding() { - ArrayEvent evt = new ArrayEvent("Event"); - evt.setEncoding((short) 10); - } - - @Test - public void testValidEncoding() { - ArrayEvent evt = new ArrayEvent("Event"); - evt.setEncoding(Event.UTF_8); - Assert.assertEquals(Event.UTF_8, evt.getEncoding()); - } - final byte[] testBytes = new byte[]{4, 'T', 'e', 's', 't', 0, 1, 2, 'a', 'b', FieldType.INT16.token, -10, 12}; - + @Test public void testBasicFunctions() throws EventSystemException { - + final ArrayEvent e1 = new ArrayEvent(testBytes); assertTrue(Arrays.equals(testBytes, e1.serialize())); @@ -60,10 +49,6 @@ public void testBasicFunctions() throws EventSystemException { assertEquals("Different { \tab = -2548; }", e3.toOneLineString()); e3.set("cd", FieldType.STRING, "value"); assertEquals("Different { \tab = -2548; \tcd = value; \tenc = 1; }", e3.toOneLineString()); - e3.setEncoding(Event.ISO_8859_1); - assertEquals("Different { \tab = -2548; \tcd = value; \tenc = 0; }", e3.toOneLineString()); - e3.set("enc", FieldType.INT16, Event.DEFAULT_ENCODING); - assertEquals("Different { \tab = -2548; \tcd = value; \tenc = 1; }", e3.toOneLineString()); e3.set("ab", FieldType.INT16, (short) -1234); assertEquals("Different { \tab = -1234; \tcd = value; \tenc = 1; }", e3.toOneLineString()); @@ -94,7 +79,7 @@ public void testBasicFunctions() throws EventSystemException { System.gc(); } - + @Test public void testReadOnly() { ArrayEvent.resetStats(); @@ -112,15 +97,15 @@ public void testReadOnly() { assertEquals(e2, e4); assertEquals(testBytes.length, e4.getBytesSize()); assertEquals(Event.MAX_MESSAGE_SIZE, e4.getCapacity()); - + final int bigSize = testBytes.length * 3; byte[] big = new byte[bigSize]; - System.arraycopy(testBytes, 0, big, 0, testBytes.length); + System.arraycopy(testBytes, 0, big, 0, testBytes.length); final ArrayEvent e5 = new ArrayEvent(big, testBytes.length, false); // no copy assertEquals(e5, e1); assertEquals(testBytes.length, e5.getBytesSize()); assertEquals(big.length, e5.getCapacity()); - + assertEquals(3, ArrayEvent.getStats().get(ArrayEventStats.WRAPS).intValue()); assertEquals(5, ArrayEvent.getStats().get(ArrayEventStats.CREATIONS).intValue()); // System.out.print(e5.toStringDetailed()); @@ -207,7 +192,7 @@ public void testStringArray() { @Test(expected = EventSystemException.class) public void testInvalidEncodingType() throws EventSystemException { final ArrayEvent event = new ArrayEvent("Event"); - event.set("enc", FieldType.INT32, Event.DEFAULT_ENCODING); + event.set("enc", FieldType.INT32, UTF_8); } @Test diff --git a/src/test/java/org/lwes/BaseTypeTest.java b/src/test/java/org/lwes/BaseTypeTest.java index 2494659..6f81ae3 100644 --- a/src/test/java/org/lwes/BaseTypeTest.java +++ b/src/test/java/org/lwes/BaseTypeTest.java @@ -10,17 +10,18 @@ package org.lwes; import org.junit.Test; +import org.lwes.serializer.StringParser; import org.lwes.util.IPAddress; import static org.junit.Assert.assertEquals; public class BaseTypeTest { - @SuppressWarnings({ "deprecation", "unused" }) + @SuppressWarnings({ "deprecation" }) @Test(expected = IllegalStateException.class) public void inconsistentConstructor() { new BaseType(FieldType.INT16.name, FieldType.UINT16.token); } - + @Test public void parsing() { assertEquals(Byte.valueOf((byte) 100), new BaseType(FieldType.BYTE).parseFromString("100")); @@ -36,4 +37,45 @@ public void parsing() { assertEquals(Float.valueOf(100), new BaseType(FieldType.FLOAT).parseFromString("100")); assertEquals(Double.valueOf(100), new BaseType(FieldType.DOUBLE).parseFromString("100")); } + + @Test + public void sizes() { + // constant sizes + { + assertEquals(2, (new BaseType(FieldType.UINT16, StringParser.fromStringUINT16("100")) ).getByteSize() ); + assertEquals(2, (new BaseType(FieldType.INT16, StringParser.fromStringINT16("100")) ).getByteSize() ); + assertEquals(4, (new BaseType(FieldType.UINT32, StringParser.fromStringUINT32("10000")) ).getByteSize() ); + assertEquals(4, (new BaseType(FieldType.INT32, StringParser.fromStringINT32("10000")) ).getByteSize() ); + assertEquals(8, (new BaseType(FieldType.INT64, StringParser.fromStringINT64("1000000")) ).getByteSize() ); + } + + // string + { + assertEquals(21, (new BaseType(FieldType.STRING, (Object)"string with size 19") ).getByteSize() ); + assertEquals(10, (new BaseType(FieldType.STRING, (Object)new String("A" + "\u00ea" + "\u00f1" + "\u00fc" + "C")) ).getByteSize() ); + } + + // array + { + String[] strArray = new String[3]; + strArray[0] = "size-6"; + strArray[1] = "size--7"; + strArray[2] = "size---8"; + assertEquals(2 + (2+6) + (2+7) + (2+8), (new BaseType(FieldType.STRING_ARRAY, (Object)strArray) ).getByteSize() ); + boolean[] boolArray = new boolean[3]; + boolArray[0]= false; + boolArray[1]= false; + boolArray[2]= true; + assertEquals(2 + boolArray.length, (new BaseType(FieldType.BOOLEAN_ARRAY, (Object)boolArray) ).getByteSize() ); + } + // nullable array + { + String[] strArray = new String[3]; + strArray[0] = "size-6"; + strArray[1] = null; + strArray[2] = new String("A" + "\u00ea" + "\u00f1" + "\u00fc" + "C"); + assertEquals( (2+2+1) + (2+6) + (0) + (10), (new BaseType(FieldType.NSTRING_ARRAY, (Object)strArray) ).getByteSize() ); + + } + } } diff --git a/src/test/java/org/lwes/EventFactoryTest.java b/src/test/java/org/lwes/EventFactoryTest.java index c65ec17..6c90240 100644 --- a/src/test/java/org/lwes/EventFactoryTest.java +++ b/src/test/java/org/lwes/EventFactoryTest.java @@ -70,13 +70,7 @@ public void testEventFactoryCreateEvent() throws EventSystemException { evt.setInt32("field3", 42); assertEquals(42, (long) evt.getInt32("field3")); - evt = fact.createEvent("TestEvent", (short) 1); - assertNotNull(evt); - evt = fact.createEvent("TestEvent", true); assertNotNull(evt); - - evt = fact.createEvent("TestEvent", false, (short) 0); - assertNotNull(evt); } } diff --git a/src/test/java/org/lwes/MapEventTest.java b/src/test/java/org/lwes/MapEventTest.java index 9c3bfbe..897f604 100644 --- a/src/test/java/org/lwes/MapEventTest.java +++ b/src/test/java/org/lwes/MapEventTest.java @@ -54,6 +54,70 @@ public void testNullableArrays() { evt.set("Integer[]", FieldType.NUINT16_ARRAY, new Integer[] { 5, null, 10 }); Assert.assertEquals(5, evt.getIntegerObjArray("Integer[]")[0].shortValue()); Assert.assertNull(evt.getIntegerObjArray("Integer[]")[1]); + + } + + @Test + public void testSerializeDeserialize() { + Event evt = new MapEvent("Event_Name"); + + evt.set("str", FieldType.STRING, "testing"); + evt.set("str2", FieldType.STRING, new String("A" + "\u00ea" + "\u00f1" + "\u00fc" + "C")); + evt.set("boolean", FieldType.BOOLEAN, true); + evt.set("byte", FieldType.BYTE, Byte.parseByte("32")); + evt.set("double", FieldType.DOUBLE, 5.0); + evt.set("float", FieldType.FLOAT, 1.2f); + evt.set("int16", FieldType.INT16, (short) 10); + evt.set("uint16", FieldType.UINT16, 20); + evt.set("int32", FieldType.INT32, 30); + evt.set("long[]", FieldType.NINT64_ARRAY, new Long[] { 5000000000l, null, 8675309l }); + evt.set("Double[]", FieldType.NDOUBLE_ARRAY, new Double[] { 1.23, null, 1.26 }); + evt.set("Float[]", FieldType.NFLOAT_ARRAY, new Float[] { 1.11f, 1.12f, null }); + evt.set("Long[]", FieldType.NUINT32_ARRAY, new Long[] { 5000L, null, 12345L }); + evt.set("Integer[]", FieldType.NUINT16_ARRAY, new Integer[] { 5, null, 10 }); + evt.set("string[]", FieldType.STRING_ARRAY, new String[] {"value1", "value2"}); + evt.set("nstring[]", FieldType.NSTRING_ARRAY, new String[] {null, "value3"}); + + byte[] serialize = new byte[ 64 * 1024 ]; + int len = evt.serialize(serialize, 0); + Event evt2 = new MapEvent(); + evt2.deserialize(serialize, 0, len); + + // make sure 2 events are identical + Assert.assertEquals("Event_Name", evt2.getEventName()); + Assert.assertEquals("testing", evt2.getString("str")); + Assert.assertEquals(new String("A" + "\u00ea" + "\u00f1" + "\u00fc" + "C"), evt2.getString("str2")); + Assert.assertTrue(evt2.getBoolean("boolean")); + Assert.assertEquals(32, (byte) evt2.getByte("byte")); + Assert.assertEquals(5.0, evt2.getDouble("double")); + Assert.assertEquals(1.2f, evt2.getFloat("float")); + Assert.assertEquals(10, (short) evt2.getInt16("int16")); + Assert.assertEquals(20, (int) evt2.getUInt16("uint16")); + Assert.assertEquals(30, (int) evt.getInt32("int32")); + Assert.assertEquals(5000000000l, evt2.getLongObjArray("long[]")[0].longValue()); + Assert.assertNull(evt2.getLongObjArray("long[]")[1]); + Assert.assertEquals(8675309l, evt2.getLongObjArray("long[]")[2].longValue()); + Assert.assertEquals(1.23, evt2.getDoubleObjArray("Double[]")[0]); + Assert.assertNull(evt2.getDoubleObjArray("Double[]")[1]); + Assert.assertEquals(1.26, evt2.getDoubleObjArray("Double[]")[2]); + Assert.assertEquals(1.11f, evt2.getFloatObjArray("Float[]")[0]); + Assert.assertEquals(1.12f, evt2.getFloatObjArray("Float[]")[1]); + Assert.assertNull(evt2.getFloatObjArray("Float[]")[2]); + Assert.assertEquals(5000, evt2.getLongObjArray("Long[]")[0].longValue()); + Assert.assertNull( evt2.getLongObjArray("Long[]")[1]); + Assert.assertEquals(12345, evt2.getLongObjArray("Long[]")[2].longValue()); + Assert.assertEquals(5, evt2.getIntegerObjArray("Integer[]")[0].shortValue()); + Assert.assertNull(evt2.getIntegerObjArray("Integer[]")[1]); + Assert.assertEquals(10, evt2.getIntegerObjArray("Integer[]")[2].shortValue()); + Assert.assertEquals("value1", evt2.getStringArray("string[]")[0]); + Assert.assertEquals("value2", evt2.getStringArray("string[]")[1]); + Assert.assertNull(evt2.getStringArray("nstring[]")[0]); + Assert.assertEquals("value3", evt2.getStringArray("nstring[]")[1]); + + byte[] serialize2 = new byte[ 64 * 1024 ]; + int len2 = evt2.serialize(serialize2, 0); + Assert.assertEquals(len, len2); + } @Test diff --git a/src/test/java/org/lwes/RandomEventTest.java b/src/test/java/org/lwes/RandomEventTest.java index 1142521..154ac93 100644 --- a/src/test/java/org/lwes/RandomEventTest.java +++ b/src/test/java/org/lwes/RandomEventTest.java @@ -42,7 +42,7 @@ public void serialization() throws InstantiationException, IllegalAccessExceptio // Ensure that the random events are equal. for (int i = 1; i < E; ++i) { - assertEquals(randomEvents[0], randomEvents[i]); + assertEquals(randomEvents[0], randomEvents[i]); } // Ensure that serializing any event type and deserializing it as @@ -101,7 +101,7 @@ public void iterators() throws InstantiationException, IllegalAccessException { } } } - // Ensure that the different implementations + // Ensure that the different implementations for (int i = 1; i < contents.size(); ++i) { assertEquals(contents.get(0).keySet(), contents.get(1).keySet()); } diff --git a/src/test/java/org/lwes/serializer/SerializerTest.java b/src/test/java/org/lwes/serializer/SerializerTest.java index 32980a5..793d71e 100644 --- a/src/test/java/org/lwes/serializer/SerializerTest.java +++ b/src/test/java/org/lwes/serializer/SerializerTest.java @@ -109,7 +109,7 @@ public void testSerializeNBigIntegerArray() { new BigInteger("9812398123") }; byte[] bytes = new byte[64]; - int num = Serializer.serializeValue(FieldType.NUINT64_ARRAY, array, (short) 0, bytes, 0); + int num = Serializer.serializeValue(FieldType.NUINT64_ARRAY, array, bytes, 0); // length + bitSet_len + bitSet + values // 2 + 2 + 1 + (8*4) @@ -153,10 +153,10 @@ public void testSerializeValueNArray() { }; byte[] bytes = new byte[64]; - int num = Serializer.serializeValue(FieldType.NFLOAT_ARRAY, array, (short) 1, bytes, 0); + int num = Serializer.serializeValue(FieldType.NFLOAT_ARRAY, array, bytes, 0); Assert.assertEquals(17, num); DeserializerState state = new DeserializerState(); - Float[] rtn = (Float[]) Deserializer.deserializeValue(state, bytes, FieldType.NFLOAT_ARRAY, (short) 1); + Float[] rtn = (Float[]) Deserializer.deserializeValue(state, bytes, FieldType.NFLOAT_ARRAY); Assert.assertNotNull(rtn); Assert.assertEquals(1.2f, rtn[0]); Assert.assertNull(rtn[1]); @@ -265,14 +265,12 @@ public void testSerializeNStringArray() { byte[] bytes = new byte[30]; int offset = 0; - short encoding = 1; int numbytes = Serializer.serializeNStringArray(array, bytes, - offset, - encoding); + offset); assertEquals("Number of bytes serialized incorrect", 23, numbytes); DeserializerState state = new DeserializerState(); - String[] a = Deserializer.deserializeNStringArray(state, bytes, encoding); + String[] a = Deserializer.deserializeNStringArray(state, bytes); assertNotNull(a); assertEquals("wrong number of elements", 4, a.length); int index = 0; @@ -287,14 +285,12 @@ public void testSerializeStringArray() { byte[] bytes = new byte[30]; int offset = 0; - short encoding = 1; int numbytes = Serializer.serializeStringArray(array, bytes, - offset, - encoding); + offset); assertEquals("Number of bytes serialized incorrect", 25, numbytes); DeserializerState state = new DeserializerState(); - String[] a = Deserializer.deserializeStringArray(state, bytes, encoding); + String[] a = Deserializer.deserializeStringArray(state, bytes); assertNotNull(a); assertEquals("wrong number of elements", 4, a.length); int index = 0; diff --git a/src/test/java/org/lwes/util/CharacterEncodingTest.java b/src/test/java/org/lwes/util/CharacterEncodingTest.java deleted file mode 100644 index 5d51738..0000000 --- a/src/test/java/org/lwes/util/CharacterEncodingTest.java +++ /dev/null @@ -1,68 +0,0 @@ -/*======================================================================* - * Copyright (c) 2010, Frank Maritato All rights reserved. * - * * - * Licensed under the New BSD License (the "License"); you may not use * - * this file except in compliance with the License. Unless required * - * by applicable law or agreed to in writing, software distributed * - * under the License is distributed on an "AS IS" BASIS, WITHOUT * - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * - * See the License for the specific language governing permissions and * - * limitations under the License. See accompanying LICENSE file. * - *======================================================================*/ - -package org.lwes.util; -/** - * @author fmaritato - */ - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.fail; -import org.junit.Test; - -import java.io.UnsupportedEncodingException; - -public class CharacterEncodingTest { - - @Test - @SuppressWarnings("null") - public void testCharEncodeInstance() { - CharacterEncoding utf8 = null; - try { - utf8 = CharacterEncoding.getInstance("UTF-8"); - } - catch (UnsupportedEncodingException e) { - fail(e.getMessage()); - } - assertNotNull(utf8); - - CharacterEncoding iso88591 = null; - try { - iso88591 = CharacterEncoding.getInstance("ISO-8859-1"); - } - catch (UnsupportedEncodingException e) { - fail(e.getMessage()); - } - assertNotNull(iso88591); - - assertFalse("utf8 = iso-8859-1", utf8.equals(iso88591)); - - } - - @Test - public void testCharEncodeStatics() { - CharacterEncoding utf8 = CharacterEncoding.UTF_8; - assertEquals(utf8.getEncodingString(), "UTF-8"); - CharacterEncoding ascii = CharacterEncoding.ASCII; - assertEquals(ascii.getEncodingString(), "ASCII"); - CharacterEncoding jp = CharacterEncoding.EUC_JP; - assertEquals(jp.getEncodingString(), "EUC_JP"); - CharacterEncoding kr = CharacterEncoding.EUC_KR; - assertEquals(kr.getEncodingString(), "EUC_KR"); - CharacterEncoding iso8859 = CharacterEncoding.ISO_8859_1; - assertEquals(iso8859.getEncodingString(), "ISO-8859-1"); - CharacterEncoding jis = CharacterEncoding.SHIFT_JIS; - assertEquals(jis.getEncodingString(), "SJIS"); - } -} diff --git a/src/test/java/org/lwes/util/EncodedStringTest.java b/src/test/java/org/lwes/util/EncodedStringTest.java index b66c38d..21623c1 100644 --- a/src/test/java/org/lwes/util/EncodedStringTest.java +++ b/src/test/java/org/lwes/util/EncodedStringTest.java @@ -15,45 +15,111 @@ * @author fmaritato */ -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; +import static org.junit.Assert.*; + +import java.io.UnsupportedEncodingException; +import java.util.Random; import org.junit.Test; +import org.lwes.MemoryPool; +import org.lwes.MemoryPool.Buffer; + +import static org.lwes.Event.MAX_MESSAGE_SIZE; public class EncodedStringTest { @Test - public void testByteConstructor() { - byte[] bytes = new byte[] {116,101,115,116,105,110,103}; - EncodedString str = new EncodedString(bytes, CharacterEncoding.UTF_8); - assertEquals("Byte Constructor failed.", "testing", str.toString()); - } + public void testEncode() { + Buffer buffer = EncodedString.encode("testing"); + assertTrue(buffer.getEncoderOutputBuffer().position() == "testing".length() ); + assertTrue( Util.compareByteArrays(buffer.getEncoderOutputBuffer().array(), 7, "testing".getBytes(), 7) ); + MemoryPool.putBack(buffer); + + buffer = EncodedString.encode(null); + assertEquals(buffer.getEncoderOutputBuffer().position(), 0); + MemoryPool.putBack(buffer); + + buffer = EncodedString.encode(""); + assertEquals(buffer.getEncoderOutputBuffer().position(), 0); + MemoryPool.putBack(buffer); + + // 64k of chars that are encoded to 1 byte, there should be no exception + StringBuilder sb = new StringBuilder(); + for(int i=0; i Date: Fri, 29 Jan 2016 03:02:37 +0000 Subject: [PATCH 2/2] added new files --- src/main/java/org/lwes/MemoryPool.java | 97 +++++++++++++++ src/main/java/org/lwes/util/Util.java | 34 ++++++ src/test/java/org/lwes/MemoryPoolTest.java | 135 +++++++++++++++++++++ src/test/java/org/lwes/util/UtilTest.java | 44 +++++++ 4 files changed, 310 insertions(+) create mode 100644 src/main/java/org/lwes/MemoryPool.java create mode 100644 src/main/java/org/lwes/util/Util.java create mode 100644 src/test/java/org/lwes/MemoryPoolTest.java create mode 100644 src/test/java/org/lwes/util/UtilTest.java diff --git a/src/main/java/org/lwes/MemoryPool.java b/src/main/java/org/lwes/MemoryPool.java new file mode 100644 index 0000000..aee9f8c --- /dev/null +++ b/src/main/java/org/lwes/MemoryPool.java @@ -0,0 +1,97 @@ +package org.lwes; + +import static org.lwes.Event.MAX_MESSAGE_SIZE; + +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.util.concurrent.ConcurrentLinkedQueue; + +/** + * this object maintains a memory pool for + * + */ +public class MemoryPool { + + // Buffer pool is a queue of + static protected ConcurrentLinkedQueue bufferPool = new ConcurrentLinkedQueue(); + + /** + * Buffer object to keep a ByteBuffer and CharBuffer object + */ + public static class Buffer { + private boolean inPool = false; // used to make sure one buffer is not + // put in to the pool more than once + private ByteBuffer byteBuffer; + private CharBuffer charBuffer; + + /** + * @return the {@code CharBuffer} object of the buffer used as the input + * for encoder + */ + public CharBuffer getEncoderInputBuffer() { + return charBuffer; + } + /** + * @return the {@code ByteBuffer} object of the Buffer used as the output + * for encoder + */ + public ByteBuffer getEncoderOutputBuffer() { + return byteBuffer; + } + + /** + * @return the {@code CharBuffer} object of the buffer used as the output + * for decoder + */ + public CharBuffer getDecoderOutputBuffer() { + return charBuffer; + } + } + + /** + * returns a {@code Buffer} object, it is either a newly allocated object + * (if the Buffer pool is empty) or is coming from the pool. + * + * @return a {@code Buffer} object + */ + static public Buffer get() { + // get a Buffer object from the pool + Buffer buffer = bufferPool.poll(); + if(buffer == null) { + // queue was empty (may not be empty when we get here, but it is ok), + // create a buffer and return it + buffer = new Buffer(); + buffer.byteBuffer = ByteBuffer.allocate( MAX_MESSAGE_SIZE ); + buffer.charBuffer = CharBuffer.allocate( MAX_MESSAGE_SIZE ); + } + // reset buffer (for when it is a used one coming from queue) + buffer.byteBuffer.clear(); + buffer.charBuffer.clear(); + buffer.inPool = false; + return buffer; + } + + /** + * returns a {@code ByteBuffer} object back in to the pool of ByteBuffers if it + * is not already in the pool + * + * @param buffer - buffer object to be put back in the pool + */ + static public void putBack(Buffer buffer) { + if(buffer != null) { + if(buffer.inPool) { + throw new IllegalArgumentException("Trying to put a buffer that is already in the pool back in to the pool!"); + } + buffer.inPool = true; + bufferPool.add(buffer); + } + } + + /** + * @return size of the pool + */ + static public int size() { + return bufferPool.size(); + } + +} diff --git a/src/main/java/org/lwes/util/Util.java b/src/main/java/org/lwes/util/Util.java new file mode 100644 index 0000000..63c95b8 --- /dev/null +++ b/src/main/java/org/lwes/util/Util.java @@ -0,0 +1,34 @@ +package org.lwes.util; + +public class Util { + + /** + * compares two arrays of bytes + * + * @param b1 - first array + * @param len1 - length of the first array + * @param b2 - second array + * @param len2 - length of the second array + * @return true if two arrays have identical contents, false otherwise. + */ + public static boolean compareByteArrays(byte[] b1, int len1, byte[] b2, int len2) { + if (len1 != len2) { + return false; + } + if (b1 == b2) { + return true; + } + if (b1==null || b2==null) { + return false; + } + + for (int i=0; i(); + } + + /** + * test all operations in one thread only + */ + @Test + public void singleThreadTest() { + + final int bufferCount = rnd.nextInt(100) + 100; + Buffer[] buffers = new Buffer[bufferCount]; + + // first test only getting buffers + for(int i = 0; i allBuffers = + new ConcurrentLinkedQueue(); + + // create threads that get buffers from pool for the first time + for(int i=0; i