From f1efacdb70923a2e877c96a2131aeca8f6da99e0 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Mon, 3 Jun 2024 23:01:11 +0800 Subject: [PATCH 01/18] remove xlang-unsupported object serialization --- .../src/main/java/org/apache/fury/Fury.java | 42 +------------------ 1 file changed, 2 insertions(+), 40 deletions(-) diff --git a/java/fury-core/src/main/java/org/apache/fury/Fury.java b/java/fury-core/src/main/java/org/apache/fury/Fury.java index ccef2aaa3d..fb601551ae 100644 --- a/java/fury-core/src/main/java/org/apache/fury/Fury.java +++ b/java/fury-core/src/main/java/org/apache/fury/Fury.java @@ -250,7 +250,7 @@ public MemoryBuffer serialize(MemoryBuffer buffer, Object obj, BufferCallback ca write(buffer, obj); } else { buffer.writeByte((byte) Language.JAVA.ordinal()); - xserializeInternal(buffer, obj); + xwriteRef(buffer, obj); } return buffer; } catch (StackOverflowError t) { @@ -321,22 +321,6 @@ private void write(MemoryBuffer buffer, Object obj) { } } - private void xserializeInternal(MemoryBuffer buffer, Object obj) { - int startOffset = buffer.writerIndex(); - buffer.writeInt32(-1); // preserve 4-byte for nativeObjects start offsets. - buffer.writeInt32(-1); // preserve 4-byte for nativeObjects size - xwriteRef(buffer, obj); - buffer.putInt32(startOffset, buffer.writerIndex()); - buffer.putInt32(startOffset + 4, nativeObjects.size()); - refResolver.resetWrite(); - // fury write opaque object classname which cause later write of classname only write an id. - classResolver.resetWrite(); - metaStringResolver.resetWrite(); - for (Object nativeObject : nativeObjects) { - writeRef(buffer, nativeObject); - } - } - /** Serialize a nullable referencable object to buffer. */ public void writeRef(MemoryBuffer buffer, Object obj) { if (!refResolver.writeRefOrNull(buffer, obj)) { @@ -752,7 +736,7 @@ public Object deserialize(MemoryBuffer buffer, Iterable outOfBandB } Object obj; if (isTargetXLang) { - obj = xdeserializeInternal(buffer); + obj = xreadRef(buffer); } else { if (config.isMetaShareEnabled()) { classResolver.readClassDefs(buffer); @@ -794,28 +778,6 @@ public Object deserialize(FuryReadableChannel channel, Iterable ou return deserialize(buf, outOfBandBuffers); } - private Object xdeserializeInternal(MemoryBuffer buffer) { - Object obj; - int nativeObjectsStartOffset = buffer.readInt32(); - int nativeObjectsSize = buffer.readInt32(); - int endReaderIndex = nativeObjectsStartOffset; - if (peerLanguage == Language.JAVA) { - int readerIndex = buffer.readerIndex(); - buffer.readerIndex(nativeObjectsStartOffset); - for (int i = 0; i < nativeObjectsSize; i++) { - nativeObjects.add(readRef(buffer)); - } - endReaderIndex = buffer.readerIndex(); - buffer.readerIndex(readerIndex); - refResolver.resetRead(); - classResolver.resetRead(); - metaStringResolver.resetRead(); - } - obj = xreadRef(buffer); - buffer.readerIndex(endReaderIndex); - return obj; - } - /** Deserialize nullable referencable object from buffer. */ public Object readRef(MemoryBuffer buffer) { RefResolver refResolver = this.refResolver; From ee381aff30c3a3ee03f4195ab0aaef7036c4e137 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 16 Jun 2024 22:39:53 +0800 Subject: [PATCH 02/18] new xlang spec in java --- .../specification/xlang_serialization_spec.md | 5 + .../main/java/org/apache/fury/BaseFury.java | 22 ++- .../src/main/java/org/apache/fury/Fury.java | 165 +++++------------- .../org/apache/fury/resolver/ClassInfo.java | 25 ++- .../apache/fury/resolver/ClassResolver.java | 110 +++++------- .../apache/fury/resolver/XtypeResolver.java | 115 ++++++++++++ .../fury/serializer/ArraySerializers.java | 17 +- .../fury/serializer/BufferSerializers.java | 3 +- .../fury/serializer/EnumSerializer.java | 12 +- .../fury/serializer/PrimitiveSerializers.java | 53 +----- .../apache/fury/serializer/Serializer.java | 18 +- .../apache/fury/serializer/Serializers.java | 19 +- .../fury/serializer/StringSerializer.java | 6 - .../fury/serializer/StructSerializer.java | 34 ++-- .../fury/serializer/TimeSerializers.java | 19 -- .../AbstractCollectionSerializer.java | 7 +- .../collection/AbstractMapSerializer.java | 50 ++++-- .../collection/CollectionSerializers.java | 41 ----- .../collection/FuryArrayAsListSerializer.java | 6 - .../GuavaCollectionSerializers.java | 21 --- .../serializer/collection/MapSerializers.java | 36 ---- .../java/org/apache/fury/type/ScalaTypes.java | 6 + .../main/java/org/apache/fury/type/Types.java | 38 ++++ .../org/apache/fury/CrossLanguageTest.java | 19 +- .../format/vectorized/ArrowSerializers.java | 3 +- .../vectorized/ArrowTableSerializer.java | 3 +- 26 files changed, 367 insertions(+), 486 deletions(-) create mode 100644 java/fury-core/src/main/java/org/apache/fury/resolver/XtypeResolver.java create mode 100644 java/fury-core/src/main/java/org/apache/fury/type/Types.java diff --git a/docs/specification/xlang_serialization_spec.md b/docs/specification/xlang_serialization_spec.md index 5882d00991..1ed0c9bbf9 100644 --- a/docs/specification/xlang_serialization_spec.md +++ b/docs/specification/xlang_serialization_spec.md @@ -67,6 +67,11 @@ Note: - Unsigned int/long are not added here, since not every language support those types. +### Polymorphisms + +For polymorphism, if one non-final class is registered, and only one subclass is registered, then we can take all +elements in List/Map have same type, thus reduce runtime check cost. + ### Type disambiguation Due to differences between type systems of languages, those types can't be mapped one-to-one between languages. When diff --git a/java/fury-core/src/main/java/org/apache/fury/BaseFury.java b/java/fury-core/src/main/java/org/apache/fury/BaseFury.java index 1ee1aec82a..fedddda57a 100644 --- a/java/fury-core/src/main/java/org/apache/fury/BaseFury.java +++ b/java/fury-core/src/main/java/org/apache/fury/BaseFury.java @@ -33,14 +33,18 @@ public interface BaseFury { /** - * register class. + * Register class and allocate an auto-grown ID for this class. Note that the registration order + * is important. If registration order is inconsistent, the allocated ID will be different, and + * the deserialization will failed. * * @param cls class to register. */ void register(Class cls); /** - * Register class. + * Register class and allocate an auto-grown ID for this class. Note that the registration order + * is important. If registration order is inconsistent, the allocated ID will be different, and + * the deserialization will failed. * * @param cls class to register. * @param createSerializer whether to create serializer, if true and codegen enabled, this will @@ -62,7 +66,9 @@ public interface BaseFury { void register(Class cls, Short id, boolean createSerializer); /** - * Register a Serializer. + * Register a Serializer for a class, and allocate an auto-grown ID for this class if it's not + * registered yet. Note that the registration order is important. If registration order is + * inconsistent, the allocated ID will be different, and the deserialization will failed. * * @param type class needed to be serialized/deserialized. * @param serializerClass serializer class can be created with {@link Serializers#newSerializer}. @@ -70,10 +76,18 @@ public interface BaseFury { */ void registerSerializer(Class type, Class serializerClass); + /** + * Register a Serializer for a class, and allocate an auto-grown ID for this class if it's not + * registered yet. Note that the registration order is important. If registration order is + * inconsistent, the allocated ID will be different, and the deserialization will failed. + */ void registerSerializer(Class type, Serializer serializer); /** - * Register a Serializer created by serializerCreator when fury created. + * Register a Serializer created by serializerCreator when fury created. And allocate an + * auto-grown ID for this class if it's not registered yet. Note that the registration order is + * important. If registration order is inconsistent, the allocated ID will be different, and the + * deserialization will failed. * * @param type class needed to be serialized/deserialized. * @param serializerCreator serializer creator with param {@link Fury} diff --git a/java/fury-core/src/main/java/org/apache/fury/Fury.java b/java/fury-core/src/main/java/org/apache/fury/Fury.java index fb601551ae..ac67eae6f9 100644 --- a/java/fury-core/src/main/java/org/apache/fury/Fury.java +++ b/java/fury-core/src/main/java/org/apache/fury/Fury.java @@ -23,9 +23,7 @@ import java.io.IOException; import java.io.OutputStream; import java.nio.ByteOrder; -import java.util.ArrayList; import java.util.Iterator; -import java.util.List; import java.util.function.Consumer; import java.util.function.Function; import javax.annotation.concurrent.NotThreadSafe; @@ -49,16 +47,15 @@ import org.apache.fury.resolver.NoRefResolver; import org.apache.fury.resolver.RefResolver; import org.apache.fury.resolver.SerializationContext; +import org.apache.fury.resolver.XtypeResolver; import org.apache.fury.serializer.ArraySerializers; import org.apache.fury.serializer.BufferCallback; import org.apache.fury.serializer.BufferObject; -import org.apache.fury.serializer.OpaqueObjects; import org.apache.fury.serializer.PrimitiveSerializers.LongSerializer; import org.apache.fury.serializer.Serializer; import org.apache.fury.serializer.SerializerFactory; import org.apache.fury.serializer.StringSerializer; import org.apache.fury.type.Generics; -import org.apache.fury.type.Type; import org.apache.fury.util.ExceptionUtils; import org.apache.fury.util.Preconditions; import org.apache.fury.util.StringUtils; @@ -84,8 +81,7 @@ public final class Fury implements BaseFury { public static final byte NOT_NULL_VALUE_FLAG = -1; // this flag indicates that the object is a referencable and first write. public static final byte REF_VALUE_FLAG = 0; - public static final byte NOT_SUPPORT_CROSS_LANGUAGE = 0; - public static final short FURY_TYPE_TAG_ID = Type.FURY_TYPE_TAG.getId(); + public static final byte NOT_SUPPORT_XLANG = 0; private static final byte isNilFlag = 1; private static final byte isLittleEndianFlag = 1 << 1; private static final byte isCrossLanguageFlag = 1 << 2; @@ -99,12 +95,12 @@ public final class Fury implements BaseFury { private final boolean refTracking; private final RefResolver refResolver; private final ClassResolver classResolver; + private final XtypeResolver xtypeResolver; private final MetaStringResolver metaStringResolver; private final SerializationContext serializationContext; private final ClassLoader classLoader; private final JITContext jitContext; private MemoryBuffer buffer; - private final List nativeObjects; private final StringSerializer stringSerializer; private final Language language; private final boolean compressInt; @@ -133,9 +129,9 @@ public Fury(FuryBuilder builder, ClassLoader classLoader) { metaStringResolver = new MetaStringResolver(); classResolver = new ClassResolver(this); classResolver.initialize(); + xtypeResolver = new XtypeResolver(classResolver); serializationContext = new SerializationContext(config); this.classLoader = classLoader; - nativeObjects = new ArrayList<>(); generics = new Generics(this); stringSerializer = new StringSerializer(this); LOG.info("Created new fury {}", this); @@ -143,27 +139,38 @@ public Fury(FuryBuilder builder, ClassLoader classLoader) { @Override public void register(Class cls) { - classResolver.register(cls); + if (language == Language.JAVA) { + classResolver.register(cls); + } else { + xtypeResolver.register(cls); + } } @Override public void register(Class cls, boolean createSerializer) { - classResolver.register(cls, createSerializer); + if (language == Language.JAVA) { + classResolver.register(cls, createSerializer); + } else { + xtypeResolver.register(cls); + } } @Override public void register(Class cls, Short id) { - classResolver.register(cls, id); + if (language == Language.JAVA) { + classResolver.register(cls, id); + } else { + xtypeResolver.register(cls, id); + } } @Override public void register(Class cls, Short id, boolean createSerializer) { - classResolver.register(cls, id, createSerializer); - } - - /** register class with given type tag which will be used for cross-language serialization. */ - public void register(Class cls, String typeTag) { - classResolver.register(cls, typeTag); + if (language == Language.JAVA) { + classResolver.register(cls, id, createSerializer); + } else { + xtypeResolver.register(cls, id); + } } @Override @@ -307,7 +314,7 @@ private void write(MemoryBuffer buffer, Object obj) { int startOffset = buffer.writerIndex(); boolean shareMeta = config.isMetaShareEnabled(); if (shareMeta) { - buffer.writeInt32(-1); // preserve 4-byte for nativeObjects start offsets. + buffer.writeInt32(-1); // preserve 4-byte for meta start offsets. } // reduce caller stack if (!refResolver.writeRefOrNull(buffer, obj)) { @@ -428,72 +435,33 @@ public void writeNonRef(MemoryBuffer buffer, Object obj, ClassInfo classInfo) { depth--; } - public void writeNonRef(MemoryBuffer buffer, T obj, Serializer serializer) { - depth++; - serializer.write(buffer, obj); - depth--; - } - public void xwriteRef(MemoryBuffer buffer, Object obj) { if (!refResolver.writeRefOrNull(buffer, obj)) { - xwriteNonRef(buffer, obj, null); + ClassInfo classInfo = classResolver.getOrUpdateClassInfo(obj.getClass()); + buffer.writeVarUint32Small7(classInfo.getXtypeId()); + depth++; + classInfo.getSerializer().xwrite(buffer, obj); + depth--; } } public void xwriteRef(MemoryBuffer buffer, T obj, Serializer serializer) { if (serializer.needToWriteRef()) { if (!refResolver.writeRefOrNull(buffer, obj)) { - xwriteNonRef(buffer, obj, serializer); + depth++; + serializer.xwrite(buffer, obj); + depth--; } } else { if (obj == null) { buffer.writeByte(Fury.NULL_FLAG); } else { buffer.writeByte(Fury.NOT_NULL_VALUE_FLAG); - xwriteNonRef(buffer, obj, serializer); - } - } - } - - public void xwriteRefByNullableSerializer( - MemoryBuffer buffer, T obj, Serializer serializer) { - if (serializer == null) { - xwriteRef(buffer, obj); - } else { - xwriteRef(buffer, obj, serializer); - } - } - - public void xwriteNonRef(MemoryBuffer buffer, T obj, Serializer serializer) { - depth++; - @SuppressWarnings("unchecked") - Class cls = (Class) obj.getClass(); - if (serializer == null) { - serializer = classResolver.getSerializer(cls); - } - short typeId = serializer.getXtypeId(); - buffer.writeInt16(typeId); - if (typeId != NOT_SUPPORT_CROSS_LANGUAGE) { - if (typeId == FURY_TYPE_TAG_ID) { - classResolver.xwriteTypeTag(buffer, cls); - } - if (typeId < NOT_SUPPORT_CROSS_LANGUAGE) { - classResolver.xwriteClass(buffer, cls); + depth++; + serializer.xwrite(buffer, obj); + depth--; } - serializer.xwrite(buffer, obj); - } else { - // Write classname so it can be used for debugging which object doesn't support - // cross-language. - // TODO add a config to disable this to reduce space cost. - classResolver.xwriteClass(buffer, cls); - // serializer may increase reference id multi times internally, thus peer cross-language later - // fields/objects deserialization will use wrong reference id since we skip opaque objects - // deserialization. - // So we stash native objects and serialize all those object at the last. - buffer.writeVarUint32(nativeObjects.size()); - nativeObjects.add(obj); } - depth--; } /** Write not null data to buffer. */ @@ -892,7 +860,7 @@ public Object xreadRef(MemoryBuffer buffer) { RefResolver refResolver = this.refResolver; int nextReadRefId = refResolver.tryPreserveRefId(buffer); if (nextReadRefId >= NOT_NULL_VALUE_FLAG) { - Object o = xreadNonRef(buffer, null); + Object o = xreadNonRef(buffer, xtypeResolver.readClassInfo(buffer)); refResolver.setReadObject(nextReadRefId, o); return o; } else { @@ -921,53 +889,19 @@ public Object xreadRef(MemoryBuffer buffer, Serializer serializer) { } } - public Object xreadRefByNullableSerializer(MemoryBuffer buffer, Serializer serializer) { - if (serializer == null) { - return xreadRef(buffer); - } else { - return xreadRef(buffer, serializer); - } + public Object xreadNonRef(MemoryBuffer buffer, Serializer serializer) { + depth++; + Object o = serializer.xread(buffer); + depth--; + return o; } - public Object xreadNonRef(MemoryBuffer buffer, Serializer serializer) { + public Object xreadNonRef(MemoryBuffer buffer, ClassInfo classInfo) { depth++; - short typeId = buffer.readInt16(); - ClassResolver classResolver = this.classResolver; - if (typeId != NOT_SUPPORT_CROSS_LANGUAGE) { - Class cls = null; - if (typeId == FURY_TYPE_TAG_ID) { - cls = classResolver.readClassByTypeTag(buffer); - } - if (typeId < NOT_SUPPORT_CROSS_LANGUAGE) { - if (peerLanguage != Language.JAVA) { - classResolver.xreadClassName(buffer); - cls = classResolver.getClassByTypeId((short) -typeId); - } else { - cls = classResolver.xreadClass(buffer); - } - } else { - if (typeId != FURY_TYPE_TAG_ID) { - cls = classResolver.getClassByTypeId(typeId); - } - } - Preconditions.checkNotNull(cls); - if (serializer == null) { - serializer = classResolver.getSerializer(cls); - } - // TODO check serializer consistent with `classResolver.getSerializer(cls)` when serializer - // not null; - Object o = serializer.xread(buffer); - depth--; - return o; - } else { - String className = classResolver.xreadClassName(buffer); - int ordinal = buffer.readVarUint32(); - if (peerLanguage != Language.JAVA) { - return OpaqueObjects.of(peerLanguage, className, ordinal); - } else { - return nativeObjects.get(ordinal); - } - } + assert classInfo != null; + Object o = classInfo.getSerializer().xread(buffer); + depth--; + return o; } @Override @@ -989,7 +923,7 @@ public void serializeJavaObject(MemoryBuffer buffer, Object obj) { } if (config.isMetaShareEnabled()) { int startOffset = buffer.writerIndex(); - buffer.writeInt32(-1); // preserve 4-byte for nativeObjects start offsets. + buffer.writeInt32(-1); // preserve 4-byte for meta start offsets. if (!refResolver.writeRefOrNull(buffer, obj)) { ClassInfo classInfo = classResolver.getOrUpdateClassInfo(obj.getClass()); writeData(buffer, classInfo, obj); @@ -1215,7 +1149,6 @@ public void reset() { classResolver.reset(); metaStringResolver.reset(); serializationContext.reset(); - nativeObjects.clear(); peerOutOfBandEnabled = false; bufferCallback = null; depth = 0; @@ -1226,7 +1159,6 @@ public void resetWrite() { classResolver.resetWrite(); metaStringResolver.resetWrite(); serializationContext.resetWrite(); - nativeObjects.clear(); bufferCallback = null; depth = 0; } @@ -1236,7 +1168,6 @@ public void resetRead() { classResolver.resetRead(); metaStringResolver.resetRead(); serializationContext.resetRead(); - nativeObjects.clear(); peerOutOfBandEnabled = false; depth = 0; } diff --git a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassInfo.java b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassInfo.java index 562343c7de..868847e1d3 100644 --- a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassInfo.java +++ b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassInfo.java @@ -20,7 +20,6 @@ package org.apache.fury.resolver; import static org.apache.fury.meta.Encoders.GENERIC_ENCODER; -import static org.apache.fury.meta.Encoders.PACKAGE_ENCODER; import org.apache.fury.collection.Tuple2; import org.apache.fury.config.Language; @@ -42,7 +41,7 @@ public class ClassInfo { final MetaStringBytes packageNameBytes; final MetaStringBytes classNameBytes; final boolean isDynamicGeneratedClass; - final MetaStringBytes typeTagBytes; + short xtypeId; Serializer serializer; // use primitive to avoid boxing // class id must be less than Integer.MAX_VALUE/2 since we use bit 0 as class id flag. @@ -54,15 +53,15 @@ public class ClassInfo { MetaStringBytes packageNameBytes, MetaStringBytes classNameBytes, boolean isDynamicGeneratedClass, - MetaStringBytes typeTagBytes, Serializer serializer, - short classId) { + short classId, + short xtypeId) { this.cls = cls; this.fullClassNameBytes = fullClassNameBytes; this.packageNameBytes = packageNameBytes; this.classNameBytes = classNameBytes; this.isDynamicGeneratedClass = isDynamicGeneratedClass; - this.typeTagBytes = typeTagBytes; + this.xtypeId = xtypeId; this.serializer = serializer; this.classId = classId; if (cls != null && classId == ClassResolver.NO_CLASS_ID) { @@ -73,9 +72,9 @@ public class ClassInfo { ClassInfo( ClassResolver classResolver, Class cls, - String tag, Serializer serializer, - short classId) { + short classId, + short xtypeId) { this.cls = cls; this.serializer = serializer; MetaStringResolver metaStringResolver = classResolver.getMetaStringResolver(); @@ -103,13 +102,7 @@ public class ClassInfo { this.packageNameBytes = null; this.classNameBytes = null; } - if (tag != null) { - this.typeTagBytes = - metaStringResolver.getOrCreateMetaStringBytes( - PACKAGE_ENCODER.encode(tag, Encoding.UTF_8)); - } else { - this.typeTagBytes = null; - } + this.xtypeId = xtypeId; this.classId = classId; if (cls != null) { boolean isLambda = Functions.isLambda(cls); @@ -134,6 +127,10 @@ public short getClassId() { return classId; } + public short getXtypeId() { + return xtypeId; + } + public MetaStringBytes getPackageNameBytes() { return packageNameBytes; } diff --git a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java index e8147a498f..9395966516 100644 --- a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java +++ b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java @@ -19,6 +19,7 @@ package org.apache.fury.resolver; +import static org.apache.fury.Fury.NOT_SUPPORT_XLANG; import static org.apache.fury.meta.ClassDef.SIZE_TWO_BYTES_FLAG; import static org.apache.fury.meta.Encoders.PACKAGE_DECODER; import static org.apache.fury.meta.Encoders.PACKAGE_ENCODER; @@ -146,6 +147,7 @@ import org.apache.fury.type.GenericType; import org.apache.fury.type.ScalaTypes; import org.apache.fury.type.TypeUtils; +import org.apache.fury.type.Types; import org.apache.fury.util.GraalvmSupport; import org.apache.fury.util.Preconditions; import org.apache.fury.util.function.Functions; @@ -201,12 +203,10 @@ public class ClassResolver { public static final short CLASS_CLASS_ID = (short) (PRIMITIVE_DOUBLE_ARRAY_CLASS_ID + 6); public static final short EMPTY_OBJECT_ID = (short) (PRIMITIVE_DOUBLE_ARRAY_CLASS_ID + 7); // use a lower load factor to minimize hash collision - private static final float loadFactor = 0.25f; private static final float furyMapLoadFactor = 0.25f; private static final int estimatedNumRegistered = 150; - private static final String META_SHARE_FIELDS_INFO_KEY = "shareFieldsInfo"; private static final ClassInfo NIL_CLASS_INFO = - new ClassInfo(null, null, null, null, false, null, null, ClassResolver.NO_CLASS_ID); + new ClassInfo(null, null, null, null, false, null, NO_CLASS_ID, NOT_SUPPORT_XLANG); private final Fury fury; private ClassInfo[] registeredId2ClassInfo = new ClassInfo[] {}; @@ -215,13 +215,9 @@ public class ClassResolver { private final IdentityMap, ClassInfo> classInfoMap = new IdentityMap<>(estimatedNumRegistered, furyMapLoadFactor); private ClassInfo classInfoCache; - private final ObjectMap> classNameBytes2Class = - new ObjectMap<>(16, furyMapLoadFactor); // Every deserialization for unregistered class will query it, performance is important. private final ObjectMap compositeClassNameBytes2ClassInfo = new ObjectMap<>(16, furyMapLoadFactor); - private final HashMap> typeIdToClassXLangMap = new HashMap<>(8, loadFactor); - private final HashMap> typeTagToClassXLangMap = new HashMap<>(8, loadFactor); private final MetaStringResolver metaStringResolver; private final boolean metaContextShareEnabled; private final Map, ClassDef> classDefMap = new HashMap<>(); @@ -424,8 +420,7 @@ public void register(Class cls, String typeTag) { + "Fury#register(Class) or Fury.register(Class, Short)"); } register(cls); - Preconditions.checkArgument(!typeTagToClassXLangMap.containsKey(typeTag)); - addSerializer(cls, new StructSerializer<>(fury, cls, typeTag)); + addSerializer(cls, new StructSerializer<>(fury, cls)); } /** @@ -464,7 +459,7 @@ public void register(Class cls, int classId) { if (classInfo != null) { classInfo.classId = id; } else { - classInfo = new ClassInfo(this, cls, null, null, id); + classInfo = new ClassInfo(this, cls, null, id, NOT_SUPPORT_XLANG); // make `extRegistry.registeredClassIdMap` and `classInfoMap` share same classInfo // instances. classInfoMap.put(cls, classInfo); @@ -615,6 +610,15 @@ public void registerSerializer(Class type, Serializer serializer) { && fury.getLanguage() == Language.JAVA) { register(type); } + if (fury.getConfig().getLanguage() != Language.JAVA) { + if (isSet(type)) { + getClassInfo(type).xtypeId = Types.SET; + } else if (isCollection(type)) { + getClassInfo(type).xtypeId = Types.LIST; + } else if (isMap(type)) { + getClassInfo(type).xtypeId = Types.MAP; + } + } addSerializer(type, serializer); } @@ -709,18 +713,7 @@ public void clearSerializer(Class cls) { /** Ass serializer for specified class. */ private void addSerializer(Class type, Serializer serializer) { Preconditions.checkNotNull(serializer); - String typeTag = null; short typeId = serializer.getXtypeId(); - if (typeId != Fury.NOT_SUPPORT_CROSS_LANGUAGE) { - if (typeId > Fury.NOT_SUPPORT_CROSS_LANGUAGE) { - typeIdToClassXLangMap.put(typeId, type); - } - if (typeId == Fury.FURY_TYPE_TAG_ID) { - typeTag = serializer.getCrossLanguageTypeTag(); - typeTagToClassXLangMap.put(typeTag, type); - } - } - // 1. Try to get ClassInfo from `registeredId2ClassInfo` and // `classInfoMap` or create a new `ClassInfo`. ClassInfo classInfo; @@ -738,8 +731,8 @@ private void addSerializer(Class type, Serializer serializer) { classInfo = classInfoMap.get(type); } - if (classInfo == null || typeTag != null || classId != classInfo.classId) { - classInfo = new ClassInfo(this, type, typeTag, null, classId); + if (classInfo == null || classId != classInfo.classId) { + classInfo = new ClassInfo(this, type, null, classId, typeId); classInfoMap.put(type, classInfo); if (registered) { registeredId2ClassInfo[classId] = classInfo; @@ -938,6 +931,21 @@ public boolean isCollection(Class cls) { } } + public boolean isSet(Class cls) { + if (Set.class.isAssignableFrom(cls)) { + return true; + } + if (fury.getConfig().isScalaOptimizationEnabled()) { + // Scala map is scala iterable too. + if (ScalaTypes.getScalaMapType().isAssignableFrom(cls)) { + return false; + } + return ScalaTypes.getScalaSetType().isAssignableFrom(cls); + } else { + return false; + } + } + public boolean isMap(Class cls) { return Map.class.isAssignableFrom(cls) || (fury.getConfig().isScalaOptimizationEnabled() @@ -1324,7 +1332,9 @@ private Class readClassWithMetaShare(MemoryBuffer buffer) { classInfo = getClassInfo(cls, false); if (classInfo == null) { Short classId = extRegistry.registeredClassIdMap.get(cls); - classInfo = new ClassInfo(this, cls, null, null, classId == null ? NO_CLASS_ID : classId); + classInfo = + new ClassInfo( + this, cls, null, classId == null ? NO_CLASS_ID : classId, NOT_SUPPORT_XLANG); classInfoMap.put(cls, classInfo); } readClassInfos.set(id, classInfo); @@ -1374,7 +1384,7 @@ private ClassInfo getMetaSharedClassInfo(ClassDef classDef, Class clz) { Class cls = clz; Short classId = extRegistry.registeredClassIdMap.get(cls); ClassInfo classInfo = - new ClassInfo(this, cls, null, null, classId == null ? NO_CLASS_ID : classId); + new ClassInfo(this, cls, null, classId == null ? NO_CLASS_ID : classId, NOT_SUPPORT_XLANG); if (NonexistentClass.class.isAssignableFrom(TypeUtils.getComponentIfArray(cls))) { if (cls == NonexistentMetaShared.class) { classInfo.serializer = new NonexistentClassSerializer(fury, classDef); @@ -1515,7 +1525,9 @@ public void writeClassInternal(MemoryBuffer buffer, Class cls) { Short classId = extRegistry.registeredClassIdMap.get(cls); // Don't create serializer in case the object for class is non-serializable, // Or class is abstract or interface. - classInfo = new ClassInfo(this, cls, null, null, classId == null ? NO_CLASS_ID : classId); + classInfo = + new ClassInfo( + this, cls, null, classId == null ? NO_CLASS_ID : classId, NOT_SUPPORT_XLANG); classInfoMap.put(cls, classInfo); } short classId = classInfo.classId; @@ -1675,8 +1687,8 @@ private ClassInfo populateBytesToClassInfo( simpleClassNameBytes, false, null, - null, - NO_CLASS_ID); + NO_CLASS_ID, + NOT_SUPPORT_XLANG); if (NonexistentClass.class.isAssignableFrom(TypeUtils.getComponentIfArray(cls))) { classInfo.serializer = NonexistentClassSerializers.getSerializer(fury, classSpec.entireClassName, cls); @@ -1691,39 +1703,10 @@ private ClassInfo populateBytesToClassInfo( return classInfo; } - public void xwriteClass(MemoryBuffer buffer, Class cls) { - metaStringResolver.writeMetaStringBytes(buffer, getOrUpdateClassInfo(cls).fullClassNameBytes); - } - - public void xwriteTypeTag(MemoryBuffer buffer, Class cls) { - metaStringResolver.writeMetaStringBytes(buffer, getOrUpdateClassInfo(cls).typeTagBytes); - } - - public Class xreadClass(MemoryBuffer buffer) { - MetaStringBytes byteString = metaStringResolver.readMetaStringBytes(buffer); - Class cls = classNameBytes2Class.get(byteString); - if (cls == null) { - Preconditions.checkNotNull(byteString); - String className = byteString.decode(Encoders.GENERIC_DECODER); - cls = loadClass(className); - classNameBytes2Class.put(byteString, cls); - } - currentReadClass = cls; - return cls; - } - - public String xreadClassName(MemoryBuffer buffer) { - return metaStringResolver.readMetaString(buffer); - } - public Class getCurrentReadClass() { return currentReadClass; } - private Class loadClass(String className) { - return loadClass(className, false, 0); - } - private Class loadClass(ClassSpec classSpec) { return loadClass(classSpec.entireClassName, classSpec.isEnum, classSpec.dimension); } @@ -1759,15 +1742,6 @@ public void resetRead() {} public void resetWrite() {} - public Class getClassByTypeId(short typeId) { - return typeIdToClassXLangMap.get(typeId); - } - - public Class readClassByTypeTag(MemoryBuffer buffer) { - String tag = metaStringResolver.readMetaString(buffer); - return typeTagToClassXLangMap.get(tag); - } - private static class ClassNameBytes { private final long packageHash; private final long classNameHash; @@ -1821,12 +1795,12 @@ public GenericType getObjectGenericType() { } public ClassInfo newClassInfo(Class cls, Serializer serializer, short classId) { - return new ClassInfo(this, cls, null, serializer, classId); + return new ClassInfo(this, cls, serializer, classId, NOT_SUPPORT_XLANG); } // Invoked by fury JIT. public ClassInfo nilClassInfo() { - return new ClassInfo(this, null, null, null, NO_CLASS_ID); + return new ClassInfo(this, null, null, NO_CLASS_ID, NOT_SUPPORT_XLANG); } public ClassInfoHolder nilClassInfoHolder() { diff --git a/java/fury-core/src/main/java/org/apache/fury/resolver/XtypeResolver.java b/java/fury-core/src/main/java/org/apache/fury/resolver/XtypeResolver.java new file mode 100644 index 0000000000..bdd7aad8ab --- /dev/null +++ b/java/fury-core/src/main/java/org/apache/fury/resolver/XtypeResolver.java @@ -0,0 +1,115 @@ +package org.apache.fury.resolver; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.sql.Timestamp; +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.fury.collection.LongMap; +import org.apache.fury.memory.MemoryBuffer; +import org.apache.fury.memory.Platform; +import org.apache.fury.serializer.EnumSerializer; +import org.apache.fury.serializer.StructSerializer; +import org.apache.fury.type.Types; +import org.apache.fury.util.Preconditions; + +public class XtypeResolver { + private static final float loadFactor = 0.25f; + + private final ClassResolver classResolver; + private int xtypeIdGenerator = 64; + + private final LongMap xtypeIdToClassMap = new LongMap<>(8, loadFactor); + private final Set registeredTypeIds = new HashSet<>(); + + public XtypeResolver(ClassResolver classResolver) { + this.classResolver = classResolver; + registerDefaultTypes(); + } + + public ClassInfo readClassInfo(MemoryBuffer buffer) { + int typeId = buffer.readVarUint32Small7(); + return xtypeIdToClassMap.get(typeId); + } + + public void register(Class type) { + while (registeredTypeIds.contains(xtypeIdGenerator)) { + xtypeIdGenerator++; + } + register(type, xtypeIdGenerator++); + } + + public void register(Class type, int xtypeId) { + // We can relax this limit in the future, but currently we keep it in small range for future + // extension. + Preconditions.checkArgument(xtypeId < Short.MAX_VALUE, "Too big type id %s", xtypeId); + ClassInfo classInfo = xtypeIdToClassMap.get(xtypeId); + if (classInfo != null) { + throw new IllegalArgumentException( + String.format("Type %s has been registered with id %s", type, classInfo.xtypeId)); + } + if (type.isEnum()) { + classResolver.registerSerializer( + type, new EnumSerializer<>(classResolver.getFury(), type.asSubclass(Enum.class))); + } else { + classResolver.registerSerializer(type, new StructSerializer<>(classResolver.getFury(), type)); + } + registeredTypeIds.add(xtypeId); + classInfo = classResolver.getClassInfo(type); + xtypeIdToClassMap.put(xtypeId, classInfo); + internalRegister(type, xtypeId); + } + + private void registerDefaultTypes(int xtypeId, Class defaultType, Class... otherTypes) { + internalRegister(defaultType, xtypeId); + xtypeIdToClassMap.put(xtypeId, classResolver.getClassInfo(defaultType)); + for (Class otherType : otherTypes) { + internalRegister(otherType, xtypeId); + } + } + + private void internalRegister(Class type, int xtypeId) { + ClassInfo classInfo = classResolver.getClassInfo(type); + Preconditions.checkArgument( + classInfo.xtypeId == 0, "Type %s has be registered with id %s", type, classInfo.xtypeId); + classInfo.xtypeId = (short) xtypeId; + } + + private void registerDefaultTypes() { + registerDefaultTypes(Types.BOOL, Boolean.class, boolean.class, AtomicBoolean.class); + registerDefaultTypes(Types.INT8, Byte.class, byte.class); + registerDefaultTypes(Types.INT16, Short.class, short.class); + registerDefaultTypes(Types.INT32, Integer.class, int.class, AtomicInteger.class); + registerDefaultTypes(Types.INT64, Long.class, long.class, AtomicLong.class); + registerDefaultTypes(Types.FLOAT32, Float.class, float.class); + registerDefaultTypes(Types.FLOAT64, Double.class, double.class); + registerDefaultTypes(Types.STRING, String.class); + registerDefaultTypes(Types.DURATION, Duration.class); + registerDefaultTypes( + Types.TIMESTAMP, Instant.class, Date.class, Timestamp.class, LocalDateTime.class); + registerDefaultTypes(Types.DECIMAL, BigDecimal.class, BigInteger.class); + registerDefaultTypes( + Types.BINARY, + byte[].class, + Platform.HEAP_BYTE_BUFFER_CLASS, + Platform.DIRECT_BYTE_BUFFER_CLASS); + registerDefaultTypes(Types.BOOL_ARRAY, boolean[].class); + registerDefaultTypes(Types.INT16_ARRAY, short[].class); + registerDefaultTypes(Types.INT32_ARRAY, int[].class); + registerDefaultTypes(Types.INT64_ARRAY, long[].class); + registerDefaultTypes(Types.FLOAT32_ARRAY, float[].class); + registerDefaultTypes(Types.FLOAT64_ARRAY, double[].class); + registerDefaultTypes(Types.LIST, ArrayList.class, Object[].class); + registerDefaultTypes(Types.SET, HashSet.class); + registerDefaultTypes(Types.MAP, HashMap.class); + } +} diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/ArraySerializers.java b/java/fury-core/src/main/java/org/apache/fury/serializer/ArraySerializers.java index 06f3f88b8d..46a771630f 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/ArraySerializers.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/ArraySerializers.java @@ -72,11 +72,6 @@ public ObjectArraySerializer(Fury fury, Class cls) { classInfoHolder = fury.getClassResolver().nilClassInfoHolder(); } - @Override - public short getXtypeId() { - return (short) -Type.LIST.getId(); - } - @Override public void write(MemoryBuffer buffer, T[] arr) { int len = arr.length; @@ -350,11 +345,6 @@ public char[] read(MemoryBuffer buffer) { } } - @Override - public short getXtypeId() { - return Fury.NOT_SUPPORT_CROSS_LANGUAGE; - } - @Override public void xwrite(MemoryBuffer buffer, char[] value) { throw new UnsupportedOperationException(); @@ -559,11 +549,6 @@ public StringArraySerializer(Fury fury) { list = new FuryArrayAsListSerializer.ArrayAsList(0); } - @Override - public short getXtypeId() { - return (short) -Type.FURY_STRING_ARRAY.getId(); - } - @Override public void write(MemoryBuffer buffer, String[] value) { int len = value.length; @@ -693,7 +678,7 @@ public static PrimitiveArrayBufferObject byteArrayBufferObject(byte[] array) { new int[] {Platform.BOOLEAN_ARRAY_OFFSET, 1, Type.FURY_PRIMITIVE_BOOL_ARRAY.getId()}); primitiveInfo.put(byte.class, new int[] {Platform.BYTE_ARRAY_OFFSET, 1, Type.BINARY.getId()}); primitiveInfo.put( - char.class, new int[] {Platform.CHAR_ARRAY_OFFSET, 2, Fury.NOT_SUPPORT_CROSS_LANGUAGE}); + char.class, new int[] {Platform.CHAR_ARRAY_OFFSET, 2, Fury.NOT_SUPPORT_XLANG}); primitiveInfo.put( short.class, new int[] {Platform.SHORT_ARRAY_OFFSET, 2, Type.FURY_PRIMITIVE_SHORT_ARRAY.getId()}); diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/BufferSerializers.java b/java/fury-core/src/main/java/org/apache/fury/serializer/BufferSerializers.java index c7ea6ee711..7a84c29b1c 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/BufferSerializers.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/BufferSerializers.java @@ -23,7 +23,6 @@ import java.nio.ByteOrder; import org.apache.fury.Fury; import org.apache.fury.memory.MemoryBuffer; -import org.apache.fury.type.Type; /** Serializers for buffer related classes. */ public class BufferSerializers { @@ -36,7 +35,7 @@ public static final class ByteBufferSerializer extends Serializers.CrossLanguageCompatibleSerializer { public ByteBufferSerializer(Fury fury, Class cls) { - super(fury, cls, Type.FURY_BUFFER.getId()); + super(fury, cls); } @Override diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/EnumSerializer.java b/java/fury-core/src/main/java/org/apache/fury/serializer/EnumSerializer.java index f03db52fa4..deee769dee 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/EnumSerializer.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/EnumSerializer.java @@ -25,17 +25,17 @@ import org.apache.fury.util.Preconditions; @SuppressWarnings("rawtypes") -public final class EnumSerializer extends Serializer { - private final Enum[] enumConstants; +public final class EnumSerializer extends Serializer { + private final T[] enumConstants; - public EnumSerializer(Fury fury, Class cls) { + public EnumSerializer(Fury fury, Class cls) { super(fury, cls, false); if (cls.isEnum()) { enumConstants = cls.getEnumConstants(); } else { Preconditions.checkArgument(Enum.class.isAssignableFrom(cls) && cls != Enum.class); @SuppressWarnings("unchecked") - Class enclosingClass = (Class) cls.getEnclosingClass(); + Class enclosingClass = (Class) cls.getEnclosingClass(); Preconditions.checkNotNull(enclosingClass); Preconditions.checkArgument(enclosingClass.isEnum()); enumConstants = enclosingClass.getEnumConstants(); @@ -48,7 +48,7 @@ public void write(MemoryBuffer buffer, Enum value) { } @Override - public Enum read(MemoryBuffer buffer) { + public T read(MemoryBuffer buffer) { int value = buffer.readVarUint32Small7(); if (value >= enumConstants.length) { return handleNonexistentEnumValue(value); @@ -56,7 +56,7 @@ public Enum read(MemoryBuffer buffer) { return enumConstants[value]; } - private Enum handleNonexistentEnumValue(int value) { + private T handleNonexistentEnumValue(int value) { if (fury.getConfig().deserializeNonexistentEnumValueAsNull()) { return null; } else { diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/PrimitiveSerializers.java b/java/fury-core/src/main/java/org/apache/fury/serializer/PrimitiveSerializers.java index e27cda8d70..dc547662ca 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/PrimitiveSerializers.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/PrimitiveSerializers.java @@ -27,7 +27,6 @@ import org.apache.fury.config.LongEncoding; import org.apache.fury.memory.MemoryBuffer; import org.apache.fury.memory.Platform; -import org.apache.fury.type.Type; import org.apache.fury.util.Preconditions; /** Serializers for java primitive types. */ @@ -36,11 +35,7 @@ public class PrimitiveSerializers { public static final class BooleanSerializer extends Serializers.CrossLanguageCompatibleSerializer { public BooleanSerializer(Fury fury, Class cls) { - super( - fury, - (Class) cls, - Type.BOOL.getId(), - !(cls.isPrimitive() || fury.isBasicTypesRefIgnored())); + super(fury, (Class) cls, !(cls.isPrimitive() || fury.isBasicTypesRefIgnored())); } @Override @@ -57,11 +52,7 @@ public Boolean read(MemoryBuffer buffer) { public static final class ByteSerializer extends Serializers.CrossLanguageCompatibleSerializer { public ByteSerializer(Fury fury, Class cls) { - super( - fury, - (Class) cls, - Type.INT8.getId(), - !(cls.isPrimitive() || fury.isBasicTypesRefIgnored())); + super(fury, (Class) cls, !(cls.isPrimitive() || fury.isBasicTypesRefIgnored())); } @Override @@ -80,11 +71,6 @@ public Uint8Serializer(Fury fury) { super(fury, Integer.class); } - @Override - public short getXtypeId() { - return Type.UINT8.getId(); - } - @Override public void xwrite(MemoryBuffer buffer, Integer value) { Preconditions.checkArgument(value >= 0 && value <= 255); @@ -103,11 +89,6 @@ public Uint16Serializer(Fury fury) { super(fury, Integer.class); } - @Override - public short getXtypeId() { - return Type.UINT16.getId(); - } - @Override public void xwrite(MemoryBuffer buffer, Integer value) { Preconditions.checkArgument(value >= 0 && value <= 65535); @@ -140,11 +121,7 @@ public Character read(MemoryBuffer buffer) { public static final class ShortSerializer extends Serializers.CrossLanguageCompatibleSerializer { public ShortSerializer(Fury fury, Class cls) { - super( - fury, - (Class) cls, - Type.INT16.getId(), - !(cls.isPrimitive() || fury.isBasicTypesRefIgnored())); + super(fury, (Class) cls, !(cls.isPrimitive() || fury.isBasicTypesRefIgnored())); } @Override @@ -163,11 +140,7 @@ public static final class IntSerializer private final boolean compressNumber; public IntSerializer(Fury fury, Class cls) { - super( - fury, - (Class) cls, - Type.INT32.getId(), - !(cls.isPrimitive() || fury.isBasicTypesRefIgnored())); + super(fury, (Class) cls, !(cls.isPrimitive() || fury.isBasicTypesRefIgnored())); compressNumber = fury.compressInt(); } @@ -206,11 +179,7 @@ public static final class LongSerializer private final LongEncoding longEncoding; public LongSerializer(Fury fury, Class cls) { - super( - fury, - (Class) cls, - Type.INT64.getId(), - !(cls.isPrimitive() || fury.isBasicTypesRefIgnored())); + super(fury, (Class) cls, !(cls.isPrimitive() || fury.isBasicTypesRefIgnored())); longEncoding = fury.longEncoding(); } @@ -290,11 +259,7 @@ public Long xread(MemoryBuffer buffer) { public static final class FloatSerializer extends Serializers.CrossLanguageCompatibleSerializer { public FloatSerializer(Fury fury, Class cls) { - super( - fury, - (Class) cls, - Type.FLOAT.getId(), - !(cls.isPrimitive() || fury.isBasicTypesRefIgnored())); + super(fury, (Class) cls, !(cls.isPrimitive() || fury.isBasicTypesRefIgnored())); } @Override @@ -311,11 +276,7 @@ public Float read(MemoryBuffer buffer) { public static final class DoubleSerializer extends Serializers.CrossLanguageCompatibleSerializer { public DoubleSerializer(Fury fury, Class cls) { - super( - fury, - (Class) cls, - Type.DOUBLE.getId(), - !(cls.isPrimitive() || fury.isBasicTypesRefIgnored())); + super(fury, (Class) cls, !(cls.isPrimitive() || fury.isBasicTypesRefIgnored())); } @Override diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/Serializer.java b/java/fury-core/src/main/java/org/apache/fury/serializer/Serializer.java index c8afaaeff8..92a0ccacf7 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/Serializer.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/Serializer.java @@ -48,20 +48,14 @@ public T read(MemoryBuffer buffer) { } /** - * Returns {@link Fury#NOT_SUPPORT_CROSS_LANGUAGE} if the serializer doesn't support - * cross-language serialization. Return a number in range (0, 32767) if the serializer support - * cross-language serialization and native serialization data is the same with cross-language - * serialization. Return a negative short in range [-32768, 0) if the serializer support - * cross-language serialization and native serialization data is not the same with cross-language - * serialization. + * Returns {@link Fury#NOT_SUPPORT_XLANG} if the serializer doesn't support cross-language + * serialization. Return a number in range (0, 32767) if the serializer support cross-language + * serialization and native serialization data is the same with cross-language serialization. + * Return a negative short in range [-32768, 0) if the serializer support cross-language + * serialization and native serialization data is not the same with cross-language serialization. */ public short getXtypeId() { - return Fury.NOT_SUPPORT_CROSS_LANGUAGE; - } - - /** Returns a type tag used for setup type mapping between languages. */ - public String getCrossLanguageTypeTag() { - throw new UnsupportedOperationException(); + return Fury.NOT_SUPPORT_XLANG; } public void xwrite(MemoryBuffer buffer, T value) { diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/Serializers.java b/java/fury-core/src/main/java/org/apache/fury/serializer/Serializers.java index 63d0099c5c..9e1b55ca29 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/Serializers.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/Serializers.java @@ -48,7 +48,6 @@ import org.apache.fury.memory.Platform; import org.apache.fury.reflect.ReflectionUtils; import org.apache.fury.resolver.ClassResolver; -import org.apache.fury.type.Type; import org.apache.fury.util.ExceptionUtils; import org.apache.fury.util.GraalvmSupport; import org.apache.fury.util.StringUtils; @@ -178,22 +177,13 @@ public static Object readPrimitiveValue(Fury fury, MemoryBuffer buffer, short cl } public abstract static class CrossLanguageCompatibleSerializer extends Serializer { - private final short typeId; - public CrossLanguageCompatibleSerializer(Fury fury, Class cls, short typeId) { + public CrossLanguageCompatibleSerializer(Fury fury, Class cls) { super(fury, cls); - this.typeId = typeId; } - public CrossLanguageCompatibleSerializer( - Fury fury, Class cls, short typeId, boolean needToWriteRef) { + public CrossLanguageCompatibleSerializer(Fury fury, Class cls, boolean needToWriteRef) { super(fury, cls, needToWriteRef); - this.typeId = typeId; - } - - @Override - public short getXtypeId() { - return typeId; } @Override @@ -236,11 +226,6 @@ public void xwrite(MemoryBuffer buffer, T value) { stringSerializer.writeUTF8String(buffer, value.toString()); } - @Override - public short getXtypeId() { - return (short) -Type.STRING.getId(); - } - @Override public void write(MemoryBuffer buffer, T value) { if (GET_CODER != null) { diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/StringSerializer.java b/java/fury-core/src/main/java/org/apache/fury/serializer/StringSerializer.java index 0c77bca401..8313faa1de 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/StringSerializer.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/StringSerializer.java @@ -40,7 +40,6 @@ import org.apache.fury.memory.MemoryBuffer; import org.apache.fury.memory.Platform; import org.apache.fury.reflect.ReflectionUtils; -import org.apache.fury.type.Type; import org.apache.fury.util.MathUtils; import org.apache.fury.util.Preconditions; import org.apache.fury.util.StringUtils; @@ -113,11 +112,6 @@ public StringSerializer(Fury fury) { compressString = fury.compressString(); } - @Override - public short getXtypeId() { - return Type.STRING.getId(); - } - @Override public void write(MemoryBuffer buffer, String value) { writeJavaString(buffer, value); diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/StructSerializer.java b/java/fury-core/src/main/java/org/apache/fury/serializer/StructSerializer.java index e37d203b1c..f89d35bdab 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/StructSerializer.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/StructSerializer.java @@ -36,11 +36,11 @@ import org.apache.fury.memory.Platform; import org.apache.fury.reflect.FieldAccessor; import org.apache.fury.reflect.TypeRef; +import org.apache.fury.resolver.ClassInfo; import org.apache.fury.type.Descriptor; import org.apache.fury.type.GenericType; import org.apache.fury.type.Generics; import org.apache.fury.type.Type; -import org.apache.fury.type.TypeUtils; import org.apache.fury.util.ExceptionUtils; import org.apache.fury.util.Preconditions; @@ -49,10 +49,9 @@ * *

TODO(chaokunyang) support generics optimization for {@code SomeClass}. */ -@SuppressWarnings({"unchecked", "rawtypes", "UnstableApiUsage"}) +@SuppressWarnings({"unchecked", "rawtypes"}) public class StructSerializer extends Serializer { private static final Logger LOG = LoggerFactory.getLogger(StructSerializer.class); - private final String typeTag; private final Constructor constructor; private final FieldAccessor[] fieldAccessors; private GenericType[] fieldGenerics; @@ -60,9 +59,8 @@ public class StructSerializer extends Serializer { private final IdentityHashMap genericTypesCache; private int typeHash; - public StructSerializer(Fury fury, Class cls, String typeTag) { + public StructSerializer(Fury fury, Class cls) { super(fury, cls); - this.typeTag = typeTag; if (fury.getLanguage() == Language.JAVA) { LOG.warn("Type of class {} shouldn't be serialized using cross-language serializer", cls); } @@ -103,16 +101,6 @@ public T read(MemoryBuffer buffer) { return xread(buffer); } - @Override - public short getXtypeId() { - return Fury.FURY_TYPE_TAG_ID; - } - - @Override - public String getCrossLanguageTypeTag() { - return typeTag; - } - @Override public void xwrite(MemoryBuffer buffer, T value) { // TODO(chaokunyang) support fields back and forward compatible. @@ -186,7 +174,12 @@ public T xread(MemoryBuffer buffer) { if (hasGenerics) { generics.pushGenericType(fieldGeneric); } - Object fieldValue = fury.xreadRefByNullableSerializer(buffer, serializer); + Object fieldValue; + if (serializer == null) { + fieldValue = fury.xreadRef(buffer); + } else { + fieldValue = fury.xreadRef(buffer, serializer); + } fieldAccessor.set(obj, fieldValue); if (hasGenerics) { generics.pushGenericType(fieldGeneric); @@ -225,15 +218,12 @@ int computeFieldHash(int hash, GenericType fieldGeneric) { id = Type.MAP.getId(); } else { try { - Serializer serializer = fury.getClassResolver().getSerializer(fieldGeneric.getCls()); - short xtypeId = serializer.getXtypeId(); - if (xtypeId == Fury.NOT_SUPPORT_CROSS_LANGUAGE) { + ClassInfo classInfo = fury.getClassResolver().getClassInfo(fieldGeneric.getCls()); + short xtypeId = classInfo.getXtypeId(); + if (xtypeId == Fury.NOT_SUPPORT_XLANG) { return hash; } id = Math.abs(xtypeId); - if (id == Type.FURY_TYPE_TAG.getId()) { - id = TypeUtils.computeStringHash(serializer.getCrossLanguageTypeTag()); - } } catch (Exception e) { return hash; } diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/TimeSerializers.java b/java/fury-core/src/main/java/org/apache/fury/serializer/TimeSerializers.java index e45652966d..2a5de4be4d 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/TimeSerializers.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/TimeSerializers.java @@ -41,7 +41,6 @@ import java.util.TimeZone; import org.apache.fury.Fury; import org.apache.fury.memory.MemoryBuffer; -import org.apache.fury.type.Type; import org.apache.fury.util.DateTimeUtils; /** Serializers for all time related types. */ @@ -126,17 +125,14 @@ protected Time newInstance(long time) { } public static final class TimestampSerializer extends TimeSerializer { - private final short typeId; public TimestampSerializer(Fury fury) { // conflict with instant super(fury, Timestamp.class); - typeId = (short) -Type.TIMESTAMP.getId(); } public TimestampSerializer(Fury fury, boolean needToWriteRef) { super(fury, Timestamp.class, needToWriteRef); - typeId = (short) -Type.TIMESTAMP.getId(); } @Override @@ -149,11 +145,6 @@ public Timestamp xread(MemoryBuffer buffer) { return DateTimeUtils.toJavaTimestamp(buffer.readInt64()); } - @Override - public short getXtypeId() { - return typeId; - } - @Override public void write(MemoryBuffer buffer, Timestamp value) { long time = value.getTime() - (value.getNanos() / 1_000_000); @@ -178,11 +169,6 @@ public LocalDateSerializer(Fury fury, boolean needToWriteRef) { super(fury, LocalDate.class, needToWriteRef); } - @Override - public short getXtypeId() { - return Type.DATE32.getId(); - } - @Override public void xwrite(MemoryBuffer buffer, LocalDate value) { // TODO use java encoding to support larger range. @@ -227,11 +213,6 @@ public InstantSerializer(Fury fury, boolean needToWriteRef) { super(fury, Instant.class, needToWriteRef); } - @Override - public short getXtypeId() { - return Type.TIMESTAMP.getId(); - } - @Override public void xwrite(MemoryBuffer buffer, Instant value) { // FIXME JDK17 may have higher precision than millisecond diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractCollectionSerializer.java b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractCollectionSerializer.java index 156694896f..b5aae9e737 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractCollectionSerializer.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractCollectionSerializer.java @@ -694,7 +694,12 @@ public void xreadElements( if (elemGenericType.isMonomorphic()) { Serializer elemSerializer = elemGenericType.getSerializer(fury.getClassResolver()); for (int i = 0; i < numElements; i++) { - Object elem = fury.xreadRefByNullableSerializer(buffer, elemSerializer); + Object elem; + if (elemSerializer == null) { + elem = fury.xreadRef(buffer); + } else { + elem = fury.xreadRef(buffer, elemSerializer); + } collection.add(elem); } } else { diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java index 9e979442da..2d9472f7ab 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java @@ -347,38 +347,47 @@ public static void xwriteElements(Fury fury, MemoryBuffer buffer, Map value) { if (!keyGenericType.hasGenericParameters() && !valueGenericType.hasGenericParameters()) { for (Object object : value.entrySet()) { Map.Entry entry = (Map.Entry) object; - fury.xwriteRefByNullableSerializer(buffer, entry.getKey(), keySerializer); - fury.xwriteRefByNullableSerializer(buffer, entry.getValue(), valueSerializer); + xwriteRefByNullableSerializer(fury, buffer, entry.getKey(), keySerializer); + xwriteRefByNullableSerializer(fury, buffer, entry.getValue(), valueSerializer); } } else if (valueGenericType.hasGenericParameters()) { for (Object object : value.entrySet()) { Map.Entry entry = (Map.Entry) object; - fury.xwriteRefByNullableSerializer(buffer, entry.getKey(), keySerializer); + xwriteRefByNullableSerializer(fury, buffer, entry.getKey(), keySerializer); generics.pushGenericType(valueGenericType); - fury.xwriteRefByNullableSerializer(buffer, entry.getValue(), valueSerializer); + xwriteRefByNullableSerializer(fury, buffer, entry.getValue(), valueSerializer); generics.popGenericType(); } } else if (keyGenericType.hasGenericParameters()) { for (Object object : value.entrySet()) { Map.Entry entry = (Map.Entry) object; generics.pushGenericType(keyGenericType); - fury.xwriteRefByNullableSerializer(buffer, entry.getKey(), keySerializer); + xwriteRefByNullableSerializer(fury, buffer, entry.getKey(), keySerializer); generics.popGenericType(); - fury.xwriteRefByNullableSerializer(buffer, entry.getValue(), valueSerializer); + xwriteRefByNullableSerializer(fury, buffer, entry.getValue(), valueSerializer); } } else { for (Object object : value.entrySet()) { Map.Entry entry = (Map.Entry) object; generics.pushGenericType(keyGenericType); - fury.xwriteRefByNullableSerializer(buffer, entry.getKey(), keySerializer); + xwriteRefByNullableSerializer(fury, buffer, entry.getKey(), keySerializer); generics.pushGenericType(valueGenericType); - fury.xwriteRefByNullableSerializer(buffer, entry.getValue(), valueSerializer); + xwriteRefByNullableSerializer(fury, buffer, entry.getValue(), valueSerializer); } } generics.popGenericType(); } } + public static void xwriteRefByNullableSerializer( + Fury fury, MemoryBuffer buffer, T obj, Serializer serializer) { + if (serializer == null) { + fury.xwriteRef(buffer, obj); + } else { + fury.xwriteRef(buffer, obj, serializer); + } + } + private Tuple2 getKVGenericType(GenericType genericType) { Tuple2 genericTypes = partialGenericKVTypeMap.get(genericType); if (genericTypes == null) { @@ -589,33 +598,33 @@ public static void xreadElements(Fury fury, MemoryBuffer buffer, Map map, int si Serializer valueSerializer = valueGenericType.getSerializer(fury.getClassResolver()); if (!keyGenericType.hasGenericParameters() && !valueGenericType.hasGenericParameters()) { for (int i = 0; i < size; i++) { - Object key = fury.xreadRefByNullableSerializer(buffer, keySerializer); - Object value = fury.xreadRefByNullableSerializer(buffer, valueSerializer); + Object key = xreadRefByNullableSerializer(fury, buffer, keySerializer); + Object value = xreadRefByNullableSerializer(fury, buffer, valueSerializer); map.put(key, value); } } else if (valueGenericType.hasGenericParameters()) { for (int i = 0; i < size; i++) { - Object key = fury.xreadRefByNullableSerializer(buffer, keySerializer); + Object key = xreadRefByNullableSerializer(fury, buffer, keySerializer); generics.pushGenericType(valueGenericType); - Object value = fury.xreadRefByNullableSerializer(buffer, valueSerializer); + Object value = xreadRefByNullableSerializer(fury, buffer, valueSerializer); generics.popGenericType(); map.put(key, value); } } else if (keyGenericType.hasGenericParameters()) { for (int i = 0; i < size; i++) { generics.pushGenericType(keyGenericType); - Object key = fury.xreadRefByNullableSerializer(buffer, keySerializer); + Object key = xreadRefByNullableSerializer(fury, buffer, keySerializer); generics.popGenericType(); - Object value = fury.xreadRefByNullableSerializer(buffer, valueSerializer); + Object value = xreadRefByNullableSerializer(fury, buffer, valueSerializer); map.put(key, value); } } else { for (int i = 0; i < size; i++) { // FIXME(chaokunyang) nested generics may be get by mistake. generics.pushGenericType(keyGenericType); - Object key = fury.xreadRefByNullableSerializer(buffer, keySerializer); + Object key = xreadRefByNullableSerializer(fury, buffer, keySerializer); generics.pushGenericType(valueGenericType); - Object value = fury.xreadRefByNullableSerializer(buffer, valueSerializer); + Object value = xreadRefByNullableSerializer(fury, buffer, valueSerializer); map.put(key, value); } } @@ -623,6 +632,15 @@ public static void xreadElements(Fury fury, MemoryBuffer buffer, Map map, int si } } + public static Object xreadRefByNullableSerializer( + Fury fury, MemoryBuffer buffer, Serializer serializer) { + if (serializer == null) { + return fury.xreadRef(buffer); + } else { + return fury.xreadRef(buffer, serializer); + } + } + /** * Hook for java serialization codegen, read/write key/value by entrySet. * diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/CollectionSerializers.java b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/CollectionSerializers.java index 3332b0c0bf..2dc7aabccd 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/CollectionSerializers.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/CollectionSerializers.java @@ -54,7 +54,6 @@ import org.apache.fury.serializer.ReplaceResolveSerializer; import org.apache.fury.serializer.Serializer; import org.apache.fury.serializer.Serializers; -import org.apache.fury.type.Type; import org.apache.fury.util.Preconditions; /** @@ -69,11 +68,6 @@ public ArrayListSerializer(Fury fury) { super(fury, ArrayList.class, true); } - @Override - public short getXtypeId() { - return Type.LIST.getId(); - } - @Override public ArrayList newCollection(MemoryBuffer buffer) { int numElements = buffer.readVarUint32Small7(); @@ -101,11 +95,6 @@ public ArraysAsListSerializer(Fury fury, Class> cls) { super(fury, cls, false); } - @Override - public short getXtypeId() { - return (short) -Type.LIST.getId(); - } - @Override public void write(MemoryBuffer buffer, List value) { try { @@ -146,11 +135,6 @@ public HashSetSerializer(Fury fury) { super(fury, HashSet.class, true); } - @Override - public short getXtypeId() { - return Type.FURY_SET.getId(); - } - @Override public HashSet newCollection(MemoryBuffer buffer) { int numElements = buffer.readVarUint32Small7(); @@ -166,11 +150,6 @@ public LinkedHashSetSerializer(Fury fury) { super(fury, LinkedHashSet.class, true); } - @Override - public short getXtypeId() { - return Type.FURY_SET.getId(); - } - @Override public LinkedHashSet newCollection(MemoryBuffer buffer) { int numElements = buffer.readVarUint32Small7(); @@ -237,11 +216,6 @@ public EmptyListSerializer(Fury fury, Class> cls) { @Override public void write(MemoryBuffer buffer, List value) {} - @Override - public short getXtypeId() { - return (short) -Type.LIST.getId(); - } - @Override public void xwrite(MemoryBuffer buffer, List value) { // write length @@ -290,11 +264,6 @@ public EmptySetSerializer(Fury fury, Class> cls) { @Override public void write(MemoryBuffer buffer, Set value) {} - @Override - public short getXtypeId() { - return (short) -Type.FURY_SET.getId(); - } - @Override public void xwrite(MemoryBuffer buffer, Set value) { // write length @@ -340,11 +309,6 @@ public void write(MemoryBuffer buffer, List value) { fury.writeRef(buffer, value.get(0)); } - @Override - public short getXtypeId() { - return (short) -Type.LIST.getId(); - } - @Override public void xwrite(MemoryBuffer buffer, List value) { buffer.writeVarUint32Small7(1); @@ -374,11 +338,6 @@ public void write(MemoryBuffer buffer, Set value) { fury.writeRef(buffer, value.iterator().next()); } - @Override - public short getXtypeId() { - return (short) -Type.FURY_SET.getId(); - } - @Override public void xwrite(MemoryBuffer buffer, Set value) { buffer.writeVarUint32Small7(1); diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/FuryArrayAsListSerializer.java b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/FuryArrayAsListSerializer.java index 057acb7bcc..09268c4e15 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/FuryArrayAsListSerializer.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/FuryArrayAsListSerializer.java @@ -26,7 +26,6 @@ import org.apache.fury.Fury; import org.apache.fury.annotation.Internal; import org.apache.fury.memory.MemoryBuffer; -import org.apache.fury.type.Type; /** Serializer for {@link ArrayAsList}. Helper for serialization of other classes. */ @Internal @@ -37,11 +36,6 @@ public FuryArrayAsListSerializer(Fury fury) { super(fury, ArrayAsList.class, true); } - @Override - public short getXtypeId() { - return (short) -Type.LIST.getId(); - } - public Collection newCollection(MemoryBuffer buffer) { int numElements = buffer.readVarUint32Small7(); setNumElements(numElements); diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/GuavaCollectionSerializers.java b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/GuavaCollectionSerializers.java index 44aa025de5..7d5791ad68 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/GuavaCollectionSerializers.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/GuavaCollectionSerializers.java @@ -38,7 +38,6 @@ import org.apache.fury.Fury; import org.apache.fury.memory.MemoryBuffer; import org.apache.fury.memory.Platform; -import org.apache.fury.type.Type; import org.apache.fury.util.unsafe._JDKAccess; /** Serializers for common guava types. */ @@ -83,11 +82,6 @@ public T onCollectionRead(Collection collection) { return (T) list; } - @Override - public short getXtypeId() { - return (short) -Type.LIST.getId(); - } - public T xnewInstance(Collection collection) { return (T) ImmutableList.copyOf(collection); } @@ -133,11 +127,6 @@ public T onCollectionRead(Collection collection) { return (T) function.apply(elements); } - @Override - public short getXtypeId() { - return (short) -Type.LIST.getId(); - } - @Override protected T xnewInstance(Collection collection) { return (T) ImmutableList.copyOf(collection); @@ -164,11 +153,6 @@ public T onCollectionRead(Collection collection) { return (T) ImmutableSet.copyOf(elements); } - @Override - public short getXtypeId() { - return (short) -Type.FURY_SET.getId(); - } - @Override protected T xnewInstance(Collection collection) { return (T) ImmutableSet.copyOf(collection); @@ -234,11 +218,6 @@ public T onMapRead(Map map) { return (T) builder.build(); } - @Override - public short getXtypeId() { - return (short) -Type.MAP.getId(); - } - @Override public T xread(MemoryBuffer buffer) { int size = buffer.readVarUint32Small7(); diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapSerializers.java b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapSerializers.java index 94c142b307..7fc9e1a50c 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapSerializers.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapSerializers.java @@ -41,7 +41,6 @@ import org.apache.fury.serializer.Serializer; import org.apache.fury.serializer.Serializers; import org.apache.fury.serializer.StringSerializer; -import org.apache.fury.type.Type; import org.apache.fury.util.Preconditions; /** @@ -56,11 +55,6 @@ public HashMapSerializer(Fury fury) { super(fury, HashMap.class, true); } - @Override - public short getXtypeId() { - return Type.MAP.getId(); - } - @Override public HashMap newMap(MemoryBuffer buffer) { int numElements = buffer.readVarUint32Small7(); @@ -76,11 +70,6 @@ public LinkedHashMapSerializer(Fury fury) { super(fury, LinkedHashMap.class, true); } - @Override - public short getXtypeId() { - return Type.MAP.getId(); - } - @Override public LinkedHashMap newMap(MemoryBuffer buffer) { int numElements = buffer.readVarUint32Small7(); @@ -96,11 +85,6 @@ public LazyMapSerializer(Fury fury) { super(fury, LazyMap.class, true); } - @Override - public short getXtypeId() { - return Type.MAP.getId(); - } - @Override public LazyMap newMap(MemoryBuffer buffer) { int numElements = buffer.readVarUint32Small7(); @@ -156,11 +140,6 @@ public EmptyMapSerializer(Fury fury, Class> cls) { @Override public void write(MemoryBuffer buffer, Map value) {} - @Override - public short getXtypeId() { - return (short) -Type.MAP.getId(); - } - @Override public void xwrite(MemoryBuffer buffer, Map value) { // write length @@ -206,11 +185,6 @@ public void write(MemoryBuffer buffer, Map value) { fury.writeRef(buffer, entry.getValue()); } - @Override - public short getXtypeId() { - return (short) -Type.MAP.getId(); - } - @Override public void xwrite(MemoryBuffer buffer, Map value) { buffer.writeVarUint32Small7(1); @@ -249,11 +223,6 @@ public ConcurrentHashMap newMap(MemoryBuffer buffer) { fury.getRefResolver().reference(map); return map; } - - @Override - public short getXtypeId() { - return Fury.NOT_SUPPORT_CROSS_LANGUAGE; - } } public static final class ConcurrentSkipListMapSerializer @@ -272,11 +241,6 @@ public ConcurrentSkipListMap newMap(MemoryBuffer buffer) { fury.getRefResolver().reference(map); return map; } - - @Override - public short getXtypeId() { - return Fury.NOT_SUPPORT_CROSS_LANGUAGE; - } } public static class EnumMapSerializer extends MapSerializer { diff --git a/java/fury-core/src/main/java/org/apache/fury/type/ScalaTypes.java b/java/fury-core/src/main/java/org/apache/fury/type/ScalaTypes.java index 485776a828..8aa1b09637 100644 --- a/java/fury-core/src/main/java/org/apache/fury/type/ScalaTypes.java +++ b/java/fury-core/src/main/java/org/apache/fury/type/ScalaTypes.java @@ -30,6 +30,7 @@ public class ScalaTypes { private static final Class SCALA_MAP_TYPE; private static final Class SCALA_SEQ_TYPE; + private static final Class SCALA_SET_TYPE; private static final Class SCALA_ITERABLE_TYPE; private static final Class SCALA_ITERATOR_TYPE; private static final java.lang.reflect.Type SCALA_ITERATOR_RETURN_TYPE; @@ -41,6 +42,7 @@ public class ScalaTypes { SCALA_ITERATOR_TYPE = ReflectionUtils.loadClass("scala.collection.Iterator"); SCALA_MAP_TYPE = ReflectionUtils.loadClass("scala.collection.Map"); SCALA_SEQ_TYPE = ReflectionUtils.loadClass("scala.collection.Seq"); + SCALA_SET_TYPE = ReflectionUtils.loadClass("scala.collection.Set"); SCALA_ITERATOR_RETURN_TYPE = SCALA_ITERABLE_TYPE.getMethod("iterator").getGenericReturnType(); SCALA_NEXT_RETURN_TYPE = SCALA_ITERATOR_TYPE.getMethod("next").getGenericReturnType(); } catch (NoSuchMethodException e) { @@ -56,6 +58,10 @@ public static Class getScalaSeqType() { return SCALA_SEQ_TYPE; } + public static Class getScalaSetType() { + return SCALA_SET_TYPE; + } + public static Class getScalaIterableType() { return SCALA_ITERABLE_TYPE; } diff --git a/java/fury-core/src/main/java/org/apache/fury/type/Types.java b/java/fury-core/src/main/java/org/apache/fury/type/Types.java new file mode 100644 index 0000000000..09c70df952 --- /dev/null +++ b/java/fury-core/src/main/java/org/apache/fury/type/Types.java @@ -0,0 +1,38 @@ +package org.apache.fury.type; + +public class Types { + public static final int NOT_SUPPORT_XLANG = 0; + public static final int BOOL = 1; + public static final int INT8 = 2; + public static final int INT16 = 3; + public static final int INT32 = 4; + public static final int VAR_INT32 = 5; + public static final int INT64 = 6; + public static final int VAR_INT64 = 7; + public static final int SLI_INT64 = 8; + public static final int FLOAT16 = 9; + public static final int FLOAT32 = 10; + public static final int FLOAT64 = 11; + public static final int STRING = 12; + public static final int ENUM = 13; + public static final int LIST = 14; + public static final int SET = 15; + public static final int MAP = 16; + public static final int DURATION = 17; + public static final int TIMESTAMP = 18; + public static final int DECIMAL = 19; + public static final int BINARY = 20; + public static final int ARRAY = 21; + public static final int BOOL_ARRAY = 22; + public static final int INT8_ARRAY = 23; + public static final int INT16_ARRAY = 24; + public static final int INT32_ARRAY = 25; + public static final int INT64_ARRAY = 26; + public static final int FLOAT16_ARRAY = 27; + public static final int FLOAT32_ARRAY = 28; + public static final int FLOAT64_ARRAY = 29; + public static final int TENSOR = 30; + public static final int SPARSE_TENSOR = 31; + public static final int ARROW_RECORD_BATCH = 32; + public static final int ARROW_TABLE = 33; +} diff --git a/java/fury-core/src/test/java/org/apache/fury/CrossLanguageTest.java b/java/fury-core/src/test/java/org/apache/fury/CrossLanguageTest.java index 8a85c678df..dd09ac801c 100644 --- a/java/fury-core/src/test/java/org/apache/fury/CrossLanguageTest.java +++ b/java/fury-core/src/test/java/org/apache/fury/CrossLanguageTest.java @@ -468,7 +468,7 @@ public void testSerializeSimpleStruct() throws Exception { .withRefTracking(true) .requireClassRegistration(false) .build(); - fury.register(ComplexObject2.class, "test.ComplexObject2"); + fury.register(ComplexObject2.class); ComplexObject2 obj2 = new ComplexObject2(); obj2.f1 = true; obj2.f2 = new HashMap<>(ImmutableMap.of((byte) -1, 2)); @@ -482,8 +482,8 @@ public void testSerializeComplexStruct() throws Exception { .withRefTracking(true) .requireClassRegistration(false) .build(); - fury.register(ComplexObject1.class, "test.ComplexObject1"); - fury.register(ComplexObject2.class, "test.ComplexObject2"); + fury.register(ComplexObject1.class); + fury.register(ComplexObject2.class); ComplexObject2 obj2 = new ComplexObject2(); obj2.f1 = true; obj2.f2 = ImmutableMap.of((byte) -1, 2); @@ -527,7 +527,7 @@ public void testSerializeOpaqueObjectSimple() { .withRefTracking(true) .requireClassRegistration(false) .build(); - fury.register(ComplexObject2.class, "test.ComplexObject2"); + fury.register(ComplexObject2.class); ComplexObject2 obj = new ComplexObject2(); obj.f1 = Foo.create(); byte[] serialized = fury.serialize(obj); @@ -542,7 +542,7 @@ public void testSerializeOpaqueObject() throws Exception { .withRefTracking(true) .requireClassRegistration(false) .build(); - fury.register(ComplexObject1.class, "test.ComplexObject1"); + fury.register(ComplexObject1.class); // don't register ComplexObject2/Foo to make them serialize as opaque blobs. ComplexObject1 obj = new ComplexObject1(); obj.f1 = new ComplexObject2(); @@ -588,11 +588,6 @@ public ComplexObject1 read(MemoryBuffer buffer) { return xread(buffer); } - @Override - public short getXtypeId() { - return Fury.FURY_TYPE_TAG_ID; - } - @Override public String getCrossLanguageTypeTag() { return "test.ComplexObject1"; @@ -727,8 +722,8 @@ static class ArrayField { @Test public void testStructArrayField() { Fury fury = Fury.builder().withLanguage(Language.XLANG).requireClassRegistration(true).build(); - fury.register(ArrayStruct.class, "example.bar"); - fury.register(ArrayField.class, "example.foo"); + fury.register(ArrayStruct.class); + fury.register(ArrayField.class); ArrayField a = new ArrayField(); a.a = "123"; diff --git a/java/fury-format/src/main/java/org/apache/fury/format/vectorized/ArrowSerializers.java b/java/fury-format/src/main/java/org/apache/fury/format/vectorized/ArrowSerializers.java index efc117a695..14e44386e9 100644 --- a/java/fury-format/src/main/java/org/apache/fury/format/vectorized/ArrowSerializers.java +++ b/java/fury-format/src/main/java/org/apache/fury/format/vectorized/ArrowSerializers.java @@ -39,7 +39,6 @@ import org.apache.fury.memory.Platform; import org.apache.fury.serializer.BufferObject; import org.apache.fury.serializer.Serializers.CrossLanguageCompatibleSerializer; -import org.apache.fury.type.Type; /** Serializers for apache arrow. */ public class ArrowSerializers { @@ -57,7 +56,7 @@ public VectorSchemaRootSerializer(Fury fury) { } public VectorSchemaRootSerializer(Fury fury, BufferAllocator allocator) { - super(fury, VectorSchemaRoot.class, Type.FURY_ARROW_RECORD_BATCH.getId()); + super(fury, VectorSchemaRoot.class); this.allocator = allocator; } diff --git a/java/fury-format/src/main/java/org/apache/fury/format/vectorized/ArrowTableSerializer.java b/java/fury-format/src/main/java/org/apache/fury/format/vectorized/ArrowTableSerializer.java index 451e5e07cd..25bda0e1c8 100644 --- a/java/fury-format/src/main/java/org/apache/fury/format/vectorized/ArrowTableSerializer.java +++ b/java/fury-format/src/main/java/org/apache/fury/format/vectorized/ArrowTableSerializer.java @@ -32,7 +32,6 @@ import org.apache.fury.memory.MemoryBuffer; import org.apache.fury.memory.Platform; import org.apache.fury.serializer.Serializers; -import org.apache.fury.type.Type; /** Serializers for {@link ArrowTable}. */ public class ArrowTableSerializer @@ -46,7 +45,7 @@ public ArrowTableSerializer(Fury fury) { } public ArrowTableSerializer(Fury fury, BufferAllocator allocator) { - super(fury, ArrowTable.class, Type.FURY_ARROW_TABLE.getId()); + super(fury, ArrowTable.class); this.allocator = allocator; } From 07e24c17d7f922ae987d67c933dc2354f2bbfd68 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 16 Jun 2024 22:42:13 +0800 Subject: [PATCH 03/18] fix compile error --- .../main/java/org/apache/fury/serializer/ArraySerializers.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/ArraySerializers.java b/java/fury-core/src/main/java/org/apache/fury/serializer/ArraySerializers.java index 46a771630f..09965aeede 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/ArraySerializers.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/ArraySerializers.java @@ -223,7 +223,7 @@ public abstract static class PrimitiveArraySerializer protected final int elemSize; public PrimitiveArraySerializer(Fury fury, Class cls) { - super(fury, cls, (short) primitiveInfo.get(TypeUtils.getArrayComponentInfo(cls).f0)[2]); + super(fury, cls); Class innerType = TypeUtils.getArrayComponentInfo(cls).f0; this.offset = primitiveInfo.get(innerType)[0]; this.elemSize = primitiveInfo.get(innerType)[1]; From 4a59adc3e1d39b787e58b4e3c2ac8f0e3a2728b5 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 13 Jul 2024 18:42:50 +0800 Subject: [PATCH 04/18] move arrow type id to format module --- .../fury/serializer/ArraySerializers.java | 16 +- .../fury/serializer/StructSerializer.java | 6 +- .../fury-core/native-image.properties | 2 - .../apache/fury/format/type/ArrowType.java} | 8 +- .../apache/fury/format/type/DataTypes.java | 186 +++++++++--------- 5 files changed, 107 insertions(+), 111 deletions(-) rename java/{fury-core/src/main/java/org/apache/fury/type/Type.java => fury-format/src/main/java/org/apache/fury/format/type/ArrowType.java} (97%) diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/ArraySerializers.java b/java/fury-core/src/main/java/org/apache/fury/serializer/ArraySerializers.java index 09965aeede..a59774f059 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/ArraySerializers.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/ArraySerializers.java @@ -31,8 +31,8 @@ import org.apache.fury.resolver.RefResolver; import org.apache.fury.serializer.collection.CollectionFlags; import org.apache.fury.serializer.collection.FuryArrayAsListSerializer; -import org.apache.fury.type.Type; import org.apache.fury.type.TypeUtils; +import org.apache.fury.type.Types; import org.apache.fury.util.Preconditions; /** Serializers for array types. */ @@ -675,24 +675,24 @@ public static PrimitiveArrayBufferObject byteArrayBufferObject(byte[] array) { static { primitiveInfo.put( boolean.class, - new int[] {Platform.BOOLEAN_ARRAY_OFFSET, 1, Type.FURY_PRIMITIVE_BOOL_ARRAY.getId()}); - primitiveInfo.put(byte.class, new int[] {Platform.BYTE_ARRAY_OFFSET, 1, Type.BINARY.getId()}); + new int[] {Platform.BOOLEAN_ARRAY_OFFSET, 1, Types.BOOL_ARRAY}); + primitiveInfo.put(byte.class, new int[] {Platform.BYTE_ARRAY_OFFSET, 1, Types.BINARY}); primitiveInfo.put( char.class, new int[] {Platform.CHAR_ARRAY_OFFSET, 2, Fury.NOT_SUPPORT_XLANG}); primitiveInfo.put( short.class, - new int[] {Platform.SHORT_ARRAY_OFFSET, 2, Type.FURY_PRIMITIVE_SHORT_ARRAY.getId()}); + new int[] {Platform.SHORT_ARRAY_OFFSET, 2, Types.INT16_ARRAY}); primitiveInfo.put( - int.class, new int[] {Platform.INT_ARRAY_OFFSET, 4, Type.FURY_PRIMITIVE_INT_ARRAY.getId()}); + int.class, new int[] {Platform.INT_ARRAY_OFFSET, 4, Types.INT32_ARRAY}); primitiveInfo.put( long.class, - new int[] {Platform.LONG_ARRAY_OFFSET, 8, Type.FURY_PRIMITIVE_LONG_ARRAY.getId()}); + new int[] {Platform.LONG_ARRAY_OFFSET, 8, Types.INT64_ARRAY}); primitiveInfo.put( float.class, - new int[] {Platform.FLOAT_ARRAY_OFFSET, 4, Type.FURY_PRIMITIVE_FLOAT_ARRAY.getId()}); + new int[] {Platform.FLOAT_ARRAY_OFFSET, 4, Types.FLOAT32_ARRAY}); primitiveInfo.put( double.class, - new int[] {Platform.DOUBLE_ARRAY_OFFSET, 8, Type.FURY_PRIMITIVE_DOUBLE_ARRAY.getId()}); + new int[] {Platform.DOUBLE_ARRAY_OFFSET, 8, Types.FLOAT64_ARRAY}); } public abstract static class AbstractedNonexistentArrayClassSerializer extends Serializer { diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/StructSerializer.java b/java/fury-core/src/main/java/org/apache/fury/serializer/StructSerializer.java index f89d35bdab..9a1d4a9eff 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/StructSerializer.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/StructSerializer.java @@ -40,7 +40,7 @@ import org.apache.fury.type.Descriptor; import org.apache.fury.type.GenericType; import org.apache.fury.type.Generics; -import org.apache.fury.type.Type; +import org.apache.fury.type.Types; import org.apache.fury.util.ExceptionUtils; import org.apache.fury.util.Preconditions; @@ -212,10 +212,10 @@ int computeFieldHash(int hash, GenericType fieldGeneric) { int id; if (fieldGeneric.getTypeRef().isSubtypeOf(List.class)) { // TODO(chaokunyang) add list element type into schema hash - id = Type.LIST.getId(); + id = Types.LIST; } else if (fieldGeneric.getTypeRef().isSubtypeOf(Map.class)) { // TODO(chaokunyang) add map key&value type into schema hash - id = Type.MAP.getId(); + id = Types.MAP; } else { try { ClassInfo classInfo = fury.getClassResolver().getClassInfo(fieldGeneric.getCls()); diff --git a/java/fury-core/src/main/resources/META-INF/native-image/org.apache.fury/fury-core/native-image.properties b/java/fury-core/src/main/resources/META-INF/native-image/org.apache.fury/fury-core/native-image.properties index 2af741d83d..8ea74ce4d3 100644 --- a/java/fury-core/src/main/resources/META-INF/native-image/org.apache.fury/fury-core/native-image.properties +++ b/java/fury-core/src/main/resources/META-INF/native-image/org.apache.fury/fury-core/native-image.properties @@ -32,10 +32,8 @@ Args=--initialize-at-build-time=org.apache.fury.memory.MemoryBuffer,\ org.apache.fury.memory.Platform,\ org.apache.fury.util.unsafe._Lookup,\ org.apache.fury.util.unsafe._JDKAccess,\ - org.apache.fury.type.Type,\ org.apache.fury.type.TypeUtils,\ org.apache.fury.Fury,\ - org.apache.fury.type.Type,\ org.apache.fury.serializer.JavaSerializer,\ org.apache.fury.reflect.ReflectionUtils,\ org.apache.fury.builder.ObjectCodecBuilder,\ diff --git a/java/fury-core/src/main/java/org/apache/fury/type/Type.java b/java/fury-format/src/main/java/org/apache/fury/format/type/ArrowType.java similarity index 97% rename from java/fury-core/src/main/java/org/apache/fury/type/Type.java rename to java/fury-format/src/main/java/org/apache/fury/format/type/ArrowType.java index 24e17579e7..cc41567277 100644 --- a/java/fury-core/src/main/java/org/apache/fury/type/Type.java +++ b/java/fury-format/src/main/java/org/apache/fury/format/type/ArrowType.java @@ -17,12 +17,12 @@ * under the License. */ -package org.apache.fury.type; +package org.apache.fury.format.type; import org.apache.fury.util.Preconditions; /** Keep in sync with Type::type in arrow/type_fwd.h */ -public enum Type { +public enum ArrowType { /// A NULL type having no physical storage NA, // NA = 0 @@ -164,12 +164,12 @@ public enum Type { private short id; - Type() { + ArrowType() { Preconditions.checkArgument(ordinal() < Short.MAX_VALUE); this.id = (short) ordinal(); } - Type(int id) { + ArrowType(int id) { Preconditions.checkArgument(id < Short.MAX_VALUE && id >= 0); this.id = (short) id; } diff --git a/java/fury-format/src/main/java/org/apache/fury/format/type/DataTypes.java b/java/fury-format/src/main/java/org/apache/fury/format/type/DataTypes.java index 383be40ce6..7ee8b8583f 100644 --- a/java/fury-format/src/main/java/org/apache/fury/format/type/DataTypes.java +++ b/java/fury-format/src/main/java/org/apache/fury/format/type/DataTypes.java @@ -36,62 +36,60 @@ import org.apache.arrow.vector.types.DateUnit; import org.apache.arrow.vector.types.FloatingPointPrecision; import org.apache.arrow.vector.types.TimeUnit; -import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; import org.apache.fury.exception.FuryException; import org.apache.fury.io.MemoryBufferOutputStream; import org.apache.fury.memory.MemoryBuffer; -import org.apache.fury.type.Type; import org.apache.fury.util.DecimalUtils; import org.apache.fury.util.Preconditions; /** Arrow data type utils. */ public class DataTypes { - public static Field PRIMITIVE_BOOLEAN_ARRAY_FIELD = primitiveArrayField(ArrowType.Bool.INSTANCE); + public static Field PRIMITIVE_BOOLEAN_ARRAY_FIELD = primitiveArrayField(org.apache.arrow.vector.types.pojo.ArrowType.Bool.INSTANCE); public static Field PRIMITIVE_BYTE_ARRAY_FIELD = primitiveArrayField(intType(8)); public static Field PRIMITIVE_SHORT_ARRAY_FIELD = primitiveArrayField(intType(16)); public static Field PRIMITIVE_INT_ARRAY_FIELD = primitiveArrayField(intType(32)); public static Field PRIMITIVE_LONG_ARRAY_FIELD = primitiveArrayField(intType(64)); public static Field PRIMITIVE_FLOAT_ARRAY_FIELD = - primitiveArrayField(new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)); + primitiveArrayField(new org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)); public static Field PRIMITIVE_DOUBLE_ARRAY_FIELD = - primitiveArrayField(new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)); + primitiveArrayField(new org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)); // Array item field default name public static final String ARRAY_ITEM_NAME = "item"; - private static final ArrowType.ArrowTypeVisitor typeWidthVisitor = + private static final org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeVisitor typeWidthVisitor = new DefaultTypeVisitor() { @Override - public Integer visit(ArrowType.Struct type) { + public Integer visit(org.apache.arrow.vector.types.pojo.ArrowType.Struct type) { return -1; } @Override - public Integer visit(ArrowType.List type) { + public Integer visit(org.apache.arrow.vector.types.pojo.ArrowType.List type) { return -1; } @Override - public Integer visit(ArrowType.Map type) { + public Integer visit(org.apache.arrow.vector.types.pojo.ArrowType.Map type) { return -1; } @Override - public Integer visit(ArrowType.Bool type) { + public Integer visit(org.apache.arrow.vector.types.pojo.ArrowType.Bool type) { return 1; } @Override - public Integer visit(ArrowType.Int type) { + public Integer visit(org.apache.arrow.vector.types.pojo.ArrowType.Int type) { return type.getBitWidth() / 8; } @Override - public Integer visit(ArrowType.FloatingPoint type) { + public Integer visit(org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint type) { switch (type.getPrecision()) { case SINGLE: return 4; @@ -103,52 +101,52 @@ public Integer visit(ArrowType.FloatingPoint type) { } @Override - public Integer visit(ArrowType.Date type) { + public Integer visit(org.apache.arrow.vector.types.pojo.ArrowType.Date type) { return 4; } @Override - public Integer visit(ArrowType.Timestamp type) { + public Integer visit(org.apache.arrow.vector.types.pojo.ArrowType.Timestamp type) { return 8; } @Override - public Integer visit(ArrowType.Binary type) { + public Integer visit(org.apache.arrow.vector.types.pojo.ArrowType.Binary type) { return -1; } @Override - public Integer visit(ArrowType.Decimal type) { + public Integer visit(org.apache.arrow.vector.types.pojo.ArrowType.Decimal type) { return -1; } @Override - public Integer visit(ArrowType.Utf8 type) { + public Integer visit(org.apache.arrow.vector.types.pojo.ArrowType.Utf8 type) { return -1; } }; - private static final ArrowType.ArrowTypeVisitor typeIdVisitor = - new DefaultTypeVisitor() { + private static final org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeVisitor typeIdVisitor = + new DefaultTypeVisitor() { @Override - public Type visit(ArrowType.Bool type) { - return Type.BOOL; + public ArrowType visit(org.apache.arrow.vector.types.pojo.ArrowType.Bool type) { + return ArrowType.BOOL; } @Override - public Type visit(ArrowType.Int type) { + public ArrowType visit(org.apache.arrow.vector.types.pojo.ArrowType.Int type) { if (type.getIsSigned()) { int byteWidth = type.getBitWidth() / 8; switch (byteWidth) { case 1: - return Type.INT8; + return ArrowType.INT8; case 2: - return Type.INT16; + return ArrowType.INT16; case 4: - return Type.INT32; + return ArrowType.INT32; case 8: - return Type.INT64; + return ArrowType.INT64; default: return unsupported(type); } @@ -157,138 +155,138 @@ public Type visit(ArrowType.Int type) { } @Override - public Type visit(ArrowType.FloatingPoint type) { + public ArrowType visit(org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint type) { switch (type.getPrecision()) { case SINGLE: - return Type.FLOAT; + return ArrowType.FLOAT; case DOUBLE: - return Type.DOUBLE; + return ArrowType.DOUBLE; default: return unsupported(type); } } @Override - public Type visit(ArrowType.Date type) { + public ArrowType visit(org.apache.arrow.vector.types.pojo.ArrowType.Date type) { switch (type.getUnit()) { case DAY: - return Type.DATE32; + return ArrowType.DATE32; case MILLISECOND: - return Type.DATE64; + return ArrowType.DATE64; default: return unsupported(type); } } @Override - public Type visit(ArrowType.Timestamp type) { - return Type.TIMESTAMP; + public ArrowType visit(org.apache.arrow.vector.types.pojo.ArrowType.Timestamp type) { + return ArrowType.TIMESTAMP; } @Override - public Type visit(ArrowType.Binary type) { - return Type.BINARY; + public ArrowType visit(org.apache.arrow.vector.types.pojo.ArrowType.Binary type) { + return ArrowType.BINARY; } @Override - public Type visit(ArrowType.Decimal type) { - return Type.DECIMAL; + public ArrowType visit(org.apache.arrow.vector.types.pojo.ArrowType.Decimal type) { + return ArrowType.DECIMAL; } @Override - public Type visit(ArrowType.Utf8 type) { - return Type.STRING; + public ArrowType visit(org.apache.arrow.vector.types.pojo.ArrowType.Utf8 type) { + return ArrowType.STRING; } @Override - public Type visit(ArrowType.Struct type) { - return Type.STRUCT; + public ArrowType visit(org.apache.arrow.vector.types.pojo.ArrowType.Struct type) { + return ArrowType.STRUCT; } @Override - public Type visit(ArrowType.List type) { - return Type.LIST; + public ArrowType visit(org.apache.arrow.vector.types.pojo.ArrowType.List type) { + return ArrowType.LIST; } @Override - public Type visit(ArrowType.Map type) { - return Type.MAP; + public ArrowType visit(org.apache.arrow.vector.types.pojo.ArrowType.Map type) { + return ArrowType.MAP; } }; - public static int getTypeWidth(ArrowType type) { + public static int getTypeWidth(org.apache.arrow.vector.types.pojo.ArrowType type) { return type.accept(typeWidthVisitor); } - public static Type getTypeId(ArrowType type) { + public static ArrowType getTypeId(org.apache.arrow.vector.types.pojo.ArrowType type) { return type.accept(typeIdVisitor); } - public static short getTypeIdValue(ArrowType type) { + public static short getTypeIdValue(org.apache.arrow.vector.types.pojo.ArrowType type) { return type.accept(typeIdVisitor).getId(); } - public static ArrowType.Bool bool() { - return ArrowType.Bool.INSTANCE; + public static org.apache.arrow.vector.types.pojo.ArrowType.Bool bool() { + return org.apache.arrow.vector.types.pojo.ArrowType.Bool.INSTANCE; } - public static ArrowType.Int intType(int bitWidth) { - return new ArrowType.Int(bitWidth, true); + public static org.apache.arrow.vector.types.pojo.ArrowType.Int intType(int bitWidth) { + return new org.apache.arrow.vector.types.pojo.ArrowType.Int(bitWidth, true); } - public static ArrowType.Int int8() { + public static org.apache.arrow.vector.types.pojo.ArrowType.Int int8() { return intType(8); } - public static ArrowType.Int int16() { + public static org.apache.arrow.vector.types.pojo.ArrowType.Int int16() { return intType(16); } - public static ArrowType.Int int32() { + public static org.apache.arrow.vector.types.pojo.ArrowType.Int int32() { return intType(32); } - public static ArrowType.Int int64() { + public static org.apache.arrow.vector.types.pojo.ArrowType.Int int64() { return intType(64); } - public static ArrowType.FloatingPoint float32() { - return new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE); + public static org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint float32() { + return new org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE); } - public static ArrowType.FloatingPoint float64() { - return new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE); + public static org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint float64() { + return new org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE); } - public static ArrowType.Date date32() { - return new ArrowType.Date(DateUnit.DAY); + public static org.apache.arrow.vector.types.pojo.ArrowType.Date date32() { + return new org.apache.arrow.vector.types.pojo.ArrowType.Date(DateUnit.DAY); } - public static ArrowType.Date date64() { - return new ArrowType.Date(DateUnit.MILLISECOND); + public static org.apache.arrow.vector.types.pojo.ArrowType.Date date64() { + return new org.apache.arrow.vector.types.pojo.ArrowType.Date(DateUnit.MILLISECOND); } - public static ArrowType.Timestamp timestamp() { - return new ArrowType.Timestamp(TimeUnit.MICROSECOND, null); + public static org.apache.arrow.vector.types.pojo.ArrowType.Timestamp timestamp() { + return new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(TimeUnit.MICROSECOND, null); } - public static ArrowType.Binary binary() { - return ArrowType.Binary.INSTANCE; + public static org.apache.arrow.vector.types.pojo.ArrowType.Binary binary() { + return org.apache.arrow.vector.types.pojo.ArrowType.Binary.INSTANCE; } - public static ArrowType.Utf8 utf8() { - return ArrowType.Utf8.INSTANCE; + public static org.apache.arrow.vector.types.pojo.ArrowType.Utf8 utf8() { + return org.apache.arrow.vector.types.pojo.ArrowType.Utf8.INSTANCE; } - public static ArrowType.Decimal decimal() { + public static org.apache.arrow.vector.types.pojo.ArrowType.Decimal decimal() { return decimal(DecimalUtils.MAX_PRECISION, DecimalUtils.MAX_SCALE); } - public static ArrowType.Decimal decimal(int precision, int scale) { - return new ArrowType.Decimal(precision, scale); + public static org.apache.arrow.vector.types.pojo.ArrowType.Decimal decimal(int precision, int scale) { + return new org.apache.arrow.vector.types.pojo.ArrowType.Decimal(precision, scale); } - public static ArrowType.Decimal bigintDecimal() { + public static org.apache.arrow.vector.types.pojo.ArrowType.Decimal bigintDecimal() { return decimal(DecimalUtils.MAX_PRECISION, 0); } @@ -297,7 +295,7 @@ public static Field field(String name, FieldType fieldType) { return field(name, fieldType, Collections.emptyList()); } - public static Field field(String name, boolean nullable, ArrowType type, Field... children) { + public static Field field(String name, boolean nullable, org.apache.arrow.vector.types.pojo.ArrowType type, Field... children) { return field(name, new FieldType(nullable, type, null), children); } @@ -305,11 +303,11 @@ public static Field field(String name, FieldType fieldType, Field... children) { return field(name, fieldType, Arrays.asList(children)); } - public static Field field(String name, boolean nullable, ArrowType type, List children) { + public static Field field(String name, boolean nullable, org.apache.arrow.vector.types.pojo.ArrowType type, List children) { return field(name, new FieldType(nullable, type, null), children); } - public static Field field(String name, ArrowType type, Field... children) { + public static Field field(String name, org.apache.arrow.vector.types.pojo.ArrowType type, Field... children) { return field(name, true, type, children); } @@ -317,34 +315,34 @@ public static Field field(String name, FieldType fieldType, List children return new ExtField(name, fieldType, children); } - public static Field notNullField(String name, ArrowType type, Field... children) { + public static Field notNullField(String name, org.apache.arrow.vector.types.pojo.ArrowType type, Field... children) { return field(name, false, type, children); } - public static FieldType notNullFieldType(ArrowType type) { + public static FieldType notNullFieldType(org.apache.arrow.vector.types.pojo.ArrowType type) { return new FieldType(false, type, null); } /* ========================= array field utils ========================= */ - public static Field primitiveArrayField(ArrowType type) { + public static Field primitiveArrayField(org.apache.arrow.vector.types.pojo.ArrowType type) { return primitiveArrayField("", type); } - public static Field primitiveArrayField(String name, ArrowType type) { + public static Field primitiveArrayField(String name, org.apache.arrow.vector.types.pojo.ArrowType type) { return field( name, - FieldType.nullable(ArrowType.List.INSTANCE), + FieldType.nullable(org.apache.arrow.vector.types.pojo.ArrowType.List.INSTANCE), Collections.singletonList(field(ARRAY_ITEM_NAME, false, type))); } - public static Field arrayField(ArrowType type) { + public static Field arrayField(org.apache.arrow.vector.types.pojo.ArrowType type) { return arrayField("", type); } - public static Field arrayField(String name, ArrowType type) { + public static Field arrayField(String name, org.apache.arrow.vector.types.pojo.ArrowType type) { return field( name, - FieldType.nullable(ArrowType.List.INSTANCE), + FieldType.nullable(org.apache.arrow.vector.types.pojo.ArrowType.List.INSTANCE), Collections.singletonList(field(ARRAY_ITEM_NAME, true, type))); } @@ -355,7 +353,7 @@ public static Field arrayField(FieldType valueType) { public static Field arrayField(String name, FieldType valueType) { return field( name, - FieldType.nullable(ArrowType.List.INSTANCE), + FieldType.nullable(org.apache.arrow.vector.types.pojo.ArrowType.List.INSTANCE), Collections.singletonList(field(ARRAY_ITEM_NAME, valueType))); } @@ -365,7 +363,7 @@ public static Field arrayField(Field valueField) { public static Field arrayField(String name, Field valueField) { return field( - name, FieldType.nullable(ArrowType.List.INSTANCE), Collections.singletonList(valueField)); + name, FieldType.nullable(org.apache.arrow.vector.types.pojo.ArrowType.List.INSTANCE), Collections.singletonList(valueField)); } public static Field arrayElementField(Field field) { @@ -373,11 +371,11 @@ public static Field arrayElementField(Field field) { } /* ========================= map field utils start ========================= */ - public static Field mapField(ArrowType keyType, ArrowType itemType) { + public static Field mapField(org.apache.arrow.vector.types.pojo.ArrowType keyType, org.apache.arrow.vector.types.pojo.ArrowType itemType) { return mapField("", keyType, itemType); } - public static Field mapField(String name, ArrowType keyType, ArrowType itemType) { + public static Field mapField(String name, org.apache.arrow.vector.types.pojo.ArrowType keyType, org.apache.arrow.vector.types.pojo.ArrowType itemType) { return mapField( name, field(MapVector.KEY_NAME, false, keyType), @@ -392,7 +390,7 @@ public static Field mapField(String name, Field keyField, Field itemField) { Preconditions.checkArgument(!keyField.isNullable(), "Map's keys must be non-nullable"); // Map's key-item pairs must be non-nullable structs Field valueField = structField(false, keyField, itemField); - return field(name, true, new ArrowType.Map(false), valueField); + return field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Map(false), valueField); } public static Field keyFieldForMap(Field mapField) { @@ -450,11 +448,11 @@ public static Field structField(boolean nullable, Field... fields) { } public static Field structField(String name, boolean nullable, Field... fields) { - return field(name, nullable, ArrowType.Struct.INSTANCE, fields); + return field(name, nullable, org.apache.arrow.vector.types.pojo.ArrowType.Struct.INSTANCE, fields); } public static Field structField(String name, boolean nullable, List fields) { - return field(name, nullable, ArrowType.Struct.INSTANCE, fields); + return field(name, nullable, org.apache.arrow.vector.types.pojo.ArrowType.Struct.INSTANCE, fields); } /* ========================= struct field utils end ========================= */ @@ -500,7 +498,7 @@ public static long computeSchemaHash(Schema schema) { } private static long computeHash(long hash, Field field) { - Type typeID = getTypeId(field.getType()); + ArrowType typeID = getTypeId(field.getType()); while (true) { try { hash = Math.addExact(Math.multiplyExact(hash, 31), (long) typeID.getId()); From 42a6f582fe2cc3be4d59218af16334d108299ec9 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 13 Jul 2024 19:11:12 +0800 Subject: [PATCH 05/18] refine type register API --- .../apache/fury/AbstractThreadSafeFury.java | 4 ++-- .../main/java/org/apache/fury/BaseFury.java | 4 ++-- .../src/main/java/org/apache/fury/Fury.java | 16 +++++++++++++-- .../apache/fury/resolver/ClassResolver.java | 2 +- .../apache/fury/resolver/XtypeResolver.java | 6 ++++++ .../org/apache/fury/CrossLanguageTest.java | 20 ++++++++----------- .../format/vectorized/ArrowSerializers.java | 3 +++ 7 files changed, 36 insertions(+), 19 deletions(-) diff --git a/java/fury-core/src/main/java/org/apache/fury/AbstractThreadSafeFury.java b/java/fury-core/src/main/java/org/apache/fury/AbstractThreadSafeFury.java index 752a4dc22c..abe663c23d 100644 --- a/java/fury-core/src/main/java/org/apache/fury/AbstractThreadSafeFury.java +++ b/java/fury-core/src/main/java/org/apache/fury/AbstractThreadSafeFury.java @@ -36,12 +36,12 @@ public void register(Class cls, boolean createSerializer) { } @Override - public void register(Class cls, Short id) { + public void register(Class cls, int id) { processCallback(fury -> fury.register(cls, id)); } @Override - public void register(Class cls, Short id, boolean createSerializer) { + public void register(Class cls, int id, boolean createSerializer) { processCallback(fury -> fury.register(cls, id, createSerializer)); } diff --git a/java/fury-core/src/main/java/org/apache/fury/BaseFury.java b/java/fury-core/src/main/java/org/apache/fury/BaseFury.java index fedddda57a..83bf4bf250 100644 --- a/java/fury-core/src/main/java/org/apache/fury/BaseFury.java +++ b/java/fury-core/src/main/java/org/apache/fury/BaseFury.java @@ -53,7 +53,7 @@ public interface BaseFury { void register(Class cls, boolean createSerializer); /** register class with given id. */ - void register(Class cls, Short id); + void register(Class cls, int id); /** * Register class with specified id. @@ -63,7 +63,7 @@ public interface BaseFury { * @param createSerializer whether to create serializer, if true and codegen enabled, this will * generate the serializer code too. */ - void register(Class cls, Short id, boolean createSerializer); + void register(Class cls, int id, boolean createSerializer); /** * Register a Serializer for a class, and allocate an auto-grown ID for this class if it's not diff --git a/java/fury-core/src/main/java/org/apache/fury/Fury.java b/java/fury-core/src/main/java/org/apache/fury/Fury.java index ac67eae6f9..808c0754cb 100644 --- a/java/fury-core/src/main/java/org/apache/fury/Fury.java +++ b/java/fury-core/src/main/java/org/apache/fury/Fury.java @@ -156,7 +156,7 @@ public void register(Class cls, boolean createSerializer) { } @Override - public void register(Class cls, Short id) { + public void register(Class cls, int id) { if (language == Language.JAVA) { classResolver.register(cls, id); } else { @@ -165,7 +165,7 @@ public void register(Class cls, Short id) { } @Override - public void register(Class cls, Short id, boolean createSerializer) { + public void register(Class cls, int id, boolean createSerializer) { if (language == Language.JAVA) { classResolver.register(cls, id, createSerializer); } else { @@ -173,6 +173,18 @@ public void register(Class cls, Short id, boolean createSerializer) { } } + /** register class with given type tag which will be used for cross-language serialization. */ + public void register(Class cls, String typeName) { + Preconditions.checkArgument(language != Language.JAVA); + register(cls, "", typeName); + } + + /** register class with given type tag which will be used for cross-language serialization. */ + public void register(Class cls, String namespace, String typeName) { + Preconditions.checkArgument(language != Language.JAVA); + xtypeResolver.register(cls, namespace, typeName); + } + @Override public void registerSerializer(Class type, Class serializerClass) { classResolver.registerSerializer(type, serializerClass); diff --git a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java index 9395966516..2969e4ba36 100644 --- a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java +++ b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java @@ -470,7 +470,7 @@ public void register(Class cls, int classId) { extRegistry.classIdGenerator++; } - public void register(Class cls, Short id, boolean createSerializer) { + public void register(Class cls, int id, boolean createSerializer) { register(cls, id); if (createSerializer) { getSerializer(cls); diff --git a/java/fury-core/src/main/java/org/apache/fury/resolver/XtypeResolver.java b/java/fury-core/src/main/java/org/apache/fury/resolver/XtypeResolver.java index bdd7aad8ab..119e8a491d 100644 --- a/java/fury-core/src/main/java/org/apache/fury/resolver/XtypeResolver.java +++ b/java/fury-core/src/main/java/org/apache/fury/resolver/XtypeResolver.java @@ -69,6 +69,12 @@ public void register(Class type, int xtypeId) { internalRegister(type, xtypeId); } + public void register(Class cls, String namespace, String typeName) { + Preconditions.checkArgument(!typeName.contains("."), + "Typename %s should not contains ., please put it into namespace", typeName); + + } + private void registerDefaultTypes(int xtypeId, Class defaultType, Class... otherTypes) { internalRegister(defaultType, xtypeId); xtypeIdToClassMap.put(xtypeId, classResolver.getClassInfo(defaultType)); diff --git a/java/fury-core/src/test/java/org/apache/fury/CrossLanguageTest.java b/java/fury-core/src/test/java/org/apache/fury/CrossLanguageTest.java index dd09ac801c..55c8ee39ba 100644 --- a/java/fury-core/src/test/java/org/apache/fury/CrossLanguageTest.java +++ b/java/fury-core/src/test/java/org/apache/fury/CrossLanguageTest.java @@ -468,7 +468,7 @@ public void testSerializeSimpleStruct() throws Exception { .withRefTracking(true) .requireClassRegistration(false) .build(); - fury.register(ComplexObject2.class); + fury.register(ComplexObject2.class, "test.ComplexObject2"); ComplexObject2 obj2 = new ComplexObject2(); obj2.f1 = true; obj2.f2 = new HashMap<>(ImmutableMap.of((byte) -1, 2)); @@ -482,8 +482,8 @@ public void testSerializeComplexStruct() throws Exception { .withRefTracking(true) .requireClassRegistration(false) .build(); - fury.register(ComplexObject1.class); - fury.register(ComplexObject2.class); + fury.register(ComplexObject1.class, "test.ComplexObject1"); + fury.register(ComplexObject2.class, "test.ComplexObject2"); ComplexObject2 obj2 = new ComplexObject2(); obj2.f1 = true; obj2.f2 = ImmutableMap.of((byte) -1, 2); @@ -527,7 +527,7 @@ public void testSerializeOpaqueObjectSimple() { .withRefTracking(true) .requireClassRegistration(false) .build(); - fury.register(ComplexObject2.class); + fury.register(ComplexObject2.class, "test.ComplexObject2"); ComplexObject2 obj = new ComplexObject2(); obj.f1 = Foo.create(); byte[] serialized = fury.serialize(obj); @@ -542,7 +542,7 @@ public void testSerializeOpaqueObject() throws Exception { .withRefTracking(true) .requireClassRegistration(false) .build(); - fury.register(ComplexObject1.class); + fury.register(ComplexObject1.class, "test.ComplexObject1"); // don't register ComplexObject2/Foo to make them serialize as opaque blobs. ComplexObject1 obj = new ComplexObject1(); obj.f1 = new ComplexObject2(); @@ -588,11 +588,6 @@ public ComplexObject1 read(MemoryBuffer buffer) { return xread(buffer); } - @Override - public String getCrossLanguageTypeTag() { - return "test.ComplexObject1"; - } - @Override public void xwrite(MemoryBuffer buffer, ComplexObject1 value) { fury.xwriteRef(buffer, value.f1); @@ -620,6 +615,7 @@ public void testRegisterSerializer() throws Exception { .withRefTracking(true) .requireClassRegistration(false) .build(); + fury.register(ComplexObject1.class, "test.ComplexObject1"); fury.registerSerializer(ComplexObject1.class, ComplexObject1Serializer.class); ComplexObject1 obj = new ComplexObject1(); obj.f1 = true; @@ -722,8 +718,8 @@ static class ArrayField { @Test public void testStructArrayField() { Fury fury = Fury.builder().withLanguage(Language.XLANG).requireClassRegistration(true).build(); - fury.register(ArrayStruct.class); - fury.register(ArrayField.class); + fury.register(ArrayStruct.class, "example.bar"); + fury.register(ArrayField.class, "example.foo"); ArrayField a = new ArrayField(); a.a = "123"; diff --git a/java/fury-format/src/main/java/org/apache/fury/format/vectorized/ArrowSerializers.java b/java/fury-format/src/main/java/org/apache/fury/format/vectorized/ArrowSerializers.java index 14e44386e9..c6f3c83f9d 100644 --- a/java/fury-format/src/main/java/org/apache/fury/format/vectorized/ArrowSerializers.java +++ b/java/fury-format/src/main/java/org/apache/fury/format/vectorized/ArrowSerializers.java @@ -39,6 +39,7 @@ import org.apache.fury.memory.Platform; import org.apache.fury.serializer.BufferObject; import org.apache.fury.serializer.Serializers.CrossLanguageCompatibleSerializer; +import org.apache.fury.type.Types; /** Serializers for apache arrow. */ public class ArrowSerializers { @@ -165,6 +166,8 @@ public MemoryBuffer toBuffer() { } public static void registerSerializers(Fury fury) { + fury.register(ArrowTable.class, Types.ARROW_TABLE); + fury.register(VectorSchemaRoot.class, Types.ARROW_RECORD_BATCH); fury.registerSerializer(ArrowTable.class, new ArrowTableSerializer(fury)); fury.registerSerializer(VectorSchemaRoot.class, new VectorSchemaRootSerializer(fury)); } From a3d4d9dc21fa852508932ee1d910be2b1441b2b0 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 13 Jul 2024 19:16:50 +0800 Subject: [PATCH 06/18] refine type register API --- .../org/apache/fury/AbstractThreadSafeFury.java | 16 ++++++++++++---- .../src/main/java/org/apache/fury/BaseFury.java | 15 ++++++++++++--- .../src/main/java/org/apache/fury/Fury.java | 14 ++++++-------- 3 files changed, 30 insertions(+), 15 deletions(-) diff --git a/java/fury-core/src/main/java/org/apache/fury/AbstractThreadSafeFury.java b/java/fury-core/src/main/java/org/apache/fury/AbstractThreadSafeFury.java index abe663c23d..6656a0477d 100644 --- a/java/fury-core/src/main/java/org/apache/fury/AbstractThreadSafeFury.java +++ b/java/fury-core/src/main/java/org/apache/fury/AbstractThreadSafeFury.java @@ -21,8 +21,11 @@ import java.util.function.Consumer; import java.util.function.Function; + +import org.apache.fury.config.Language; import org.apache.fury.serializer.Serializer; import org.apache.fury.serializer.SerializerFactory; +import org.apache.fury.util.Preconditions; public abstract class AbstractThreadSafeFury implements ThreadSafeFury { @Override @@ -30,19 +33,24 @@ public void register(Class clz) { processCallback(fury -> fury.register(clz)); } + @Override + public void register(Class cls, int id) { + processCallback(fury -> fury.register(cls, id)); + } + @Override public void register(Class cls, boolean createSerializer) { processCallback(fury -> fury.register(cls, createSerializer)); } @Override - public void register(Class cls, int id) { - processCallback(fury -> fury.register(cls, id)); + public void register(Class cls, String typeName) { + processCallback(fury -> fury.register(cls, typeName)); } @Override - public void register(Class cls, int id, boolean createSerializer) { - processCallback(fury -> fury.register(cls, id, createSerializer)); + public void register(Class cls, String namespace, String typeName) { + processCallback(fury -> fury.register(cls, namespace, typeName)); } @Override diff --git a/java/fury-core/src/main/java/org/apache/fury/BaseFury.java b/java/fury-core/src/main/java/org/apache/fury/BaseFury.java index 83bf4bf250..c4e4972bd7 100644 --- a/java/fury-core/src/main/java/org/apache/fury/BaseFury.java +++ b/java/fury-core/src/main/java/org/apache/fury/BaseFury.java @@ -21,6 +21,8 @@ import java.io.OutputStream; import java.util.function.Function; + +import org.apache.fury.config.Language; import org.apache.fury.io.FuryInputStream; import org.apache.fury.io.FuryReadableChannel; import org.apache.fury.memory.MemoryBuffer; @@ -28,6 +30,7 @@ import org.apache.fury.serializer.Serializer; import org.apache.fury.serializer.SerializerFactory; import org.apache.fury.serializer.Serializers; +import org.apache.fury.util.Preconditions; /** All Fury’s basic interface, including Fury’s basic methods. */ public interface BaseFury { @@ -41,6 +44,9 @@ public interface BaseFury { */ void register(Class cls); + /** register class with given id. */ + void register(Class cls, int id); + /** * Register class and allocate an auto-grown ID for this class. Note that the registration order * is important. If registration order is inconsistent, the allocated ID will be different, and @@ -52,9 +58,6 @@ public interface BaseFury { */ void register(Class cls, boolean createSerializer); - /** register class with given id. */ - void register(Class cls, int id); - /** * Register class with specified id. * @@ -65,6 +68,12 @@ public interface BaseFury { */ void register(Class cls, int id, boolean createSerializer); + /** register class with given type name which will be used for cross-language serialization. */ + void register(Class cls, String typeName); + + /** register class with given type namespace and name which will be used for cross-language serialization. */ + void register(Class cls, String namespace, String typeName); + /** * Register a Serializer for a class, and allocate an auto-grown ID for this class if it's not * registered yet. Note that the registration order is important. If registration order is diff --git a/java/fury-core/src/main/java/org/apache/fury/Fury.java b/java/fury-core/src/main/java/org/apache/fury/Fury.java index 808c0754cb..316487342f 100644 --- a/java/fury-core/src/main/java/org/apache/fury/Fury.java +++ b/java/fury-core/src/main/java/org/apache/fury/Fury.java @@ -147,20 +147,20 @@ public void register(Class cls) { } @Override - public void register(Class cls, boolean createSerializer) { + public void register(Class cls, int id) { if (language == Language.JAVA) { - classResolver.register(cls, createSerializer); + classResolver.register(cls, id); } else { - xtypeResolver.register(cls); + xtypeResolver.register(cls, id); } } @Override - public void register(Class cls, int id) { + public void register(Class cls, boolean createSerializer) { if (language == Language.JAVA) { - classResolver.register(cls, id); + classResolver.register(cls, createSerializer); } else { - xtypeResolver.register(cls, id); + xtypeResolver.register(cls); } } @@ -173,13 +173,11 @@ public void register(Class cls, int id, boolean createSerializer) { } } - /** register class with given type tag which will be used for cross-language serialization. */ public void register(Class cls, String typeName) { Preconditions.checkArgument(language != Language.JAVA); register(cls, "", typeName); } - /** register class with given type tag which will be used for cross-language serialization. */ public void register(Class cls, String namespace, String typeName) { Preconditions.checkArgument(language != Language.JAVA); xtypeResolver.register(cls, namespace, typeName); From bf98a76a66fc249957bf211a5a214e3bf5627b46 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 14 Jul 2024 14:36:40 +0800 Subject: [PATCH 07/18] add comments to MetaStringResolver --- .../java/org/apache/fury/resolver/MetaStringResolver.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/java/fury-core/src/main/java/org/apache/fury/resolver/MetaStringResolver.java b/java/fury-core/src/main/java/org/apache/fury/resolver/MetaStringResolver.java index 30ede25229..bd07d24c84 100644 --- a/java/fury-core/src/main/java/org/apache/fury/resolver/MetaStringResolver.java +++ b/java/fury-core/src/main/java/org/apache/fury/resolver/MetaStringResolver.java @@ -73,10 +73,12 @@ public void writeMetaStringBytesWithFlag(MemoryBuffer buffer, MetaStringBytes by dynamicWrittenMetaString = growWrite(id); } dynamicWrittenMetaString[id] = byteString; - buffer.writeVarUint32Small7(byteString.bytes.length << 2 | 0b1); + // last bit `1` indicates class is written by name instead of registered id. + buffer.writeVarUint32Small7(byteString.bytes.length << 2 | 0b01); buffer.writeInt64(byteString.hashCode); buffer.writeBytes(byteString.bytes); } else { + // last bit `1` indicates class is written by name instead of registered id. buffer.writeVarUint32Small7(((id + 1) << 2) | 0b11); } } From 8dba32eda92a53e5b5d361cd06bb1d723648f1d3 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 14 Jul 2024 15:42:03 +0800 Subject: [PATCH 08/18] refactor type system --- .../apache/fury/AbstractThreadSafeFury.java | 3 - .../main/java/org/apache/fury/BaseFury.java | 8 +- .../fury/serializer/ArraySerializers.java | 62 ++-- .../fury/serializer/OptionalSerializers.java | 10 +- .../fury/serializer/PrimitiveSerializers.java | 34 +- .../apache/fury/serializer/Serializers.java | 29 +- .../fury/serializer/TimeSerializers.java | 36 +- .../collection/CollectionSerializers.java | 40 ++- .../GuavaCollectionSerializers.java | 36 +- .../ImmutableCollectionSerializers.java | 16 +- .../serializer/collection/MapSerializers.java | 19 +- .../collection/SynchronizedSerializers.java | 4 +- .../collection/UnmodifiableSerializers.java | 4 +- .../main/java/org/apache/fury/type/Types.java | 47 +-- .../apache/fury/format/type/DataTypes.java | 324 ++++++++++-------- 15 files changed, 359 insertions(+), 313 deletions(-) diff --git a/java/fury-core/src/main/java/org/apache/fury/AbstractThreadSafeFury.java b/java/fury-core/src/main/java/org/apache/fury/AbstractThreadSafeFury.java index 6656a0477d..42c53a7487 100644 --- a/java/fury-core/src/main/java/org/apache/fury/AbstractThreadSafeFury.java +++ b/java/fury-core/src/main/java/org/apache/fury/AbstractThreadSafeFury.java @@ -21,11 +21,8 @@ import java.util.function.Consumer; import java.util.function.Function; - -import org.apache.fury.config.Language; import org.apache.fury.serializer.Serializer; import org.apache.fury.serializer.SerializerFactory; -import org.apache.fury.util.Preconditions; public abstract class AbstractThreadSafeFury implements ThreadSafeFury { @Override diff --git a/java/fury-core/src/main/java/org/apache/fury/BaseFury.java b/java/fury-core/src/main/java/org/apache/fury/BaseFury.java index c4e4972bd7..ef0469b377 100644 --- a/java/fury-core/src/main/java/org/apache/fury/BaseFury.java +++ b/java/fury-core/src/main/java/org/apache/fury/BaseFury.java @@ -21,8 +21,6 @@ import java.io.OutputStream; import java.util.function.Function; - -import org.apache.fury.config.Language; import org.apache.fury.io.FuryInputStream; import org.apache.fury.io.FuryReadableChannel; import org.apache.fury.memory.MemoryBuffer; @@ -30,7 +28,6 @@ import org.apache.fury.serializer.Serializer; import org.apache.fury.serializer.SerializerFactory; import org.apache.fury.serializer.Serializers; -import org.apache.fury.util.Preconditions; /** All Fury’s basic interface, including Fury’s basic methods. */ public interface BaseFury { @@ -71,7 +68,10 @@ public interface BaseFury { /** register class with given type name which will be used for cross-language serialization. */ void register(Class cls, String typeName); - /** register class with given type namespace and name which will be used for cross-language serialization. */ + /** + * register class with given type namespace and name which will be used for cross-language + * serialization. + */ void register(Class cls, String namespace, String typeName); /** diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/ArraySerializers.java b/java/fury-core/src/main/java/org/apache/fury/serializer/ArraySerializers.java index a59774f059..b7b8ceace6 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/ArraySerializers.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/ArraySerializers.java @@ -631,26 +631,29 @@ public String[] xread(MemoryBuffer buffer) { } public static void registerDefaultSerializers(Fury fury) { - fury.registerSerializer(Object[].class, new ObjectArraySerializer<>(fury, Object[].class)); - fury.registerSerializer(Class[].class, new ObjectArraySerializer<>(fury, Class[].class)); - fury.registerSerializer(byte[].class, new ByteArraySerializer(fury)); - fury.registerSerializer(Byte[].class, new ObjectArraySerializer<>(fury, Byte[].class)); - fury.registerSerializer(char[].class, new CharArraySerializer(fury)); - fury.registerSerializer( + ClassResolver resolver = fury.getClassResolver(); + resolver.registerSerializer(Object[].class, new ObjectArraySerializer<>(fury, Object[].class)); + resolver.registerSerializer(Class[].class, new ObjectArraySerializer<>(fury, Class[].class)); + resolver.registerSerializer(byte[].class, new ByteArraySerializer(fury)); + resolver.registerSerializer(Byte[].class, new ObjectArraySerializer<>(fury, Byte[].class)); + resolver.registerSerializer(char[].class, new CharArraySerializer(fury)); + resolver.registerSerializer( Character[].class, new ObjectArraySerializer<>(fury, Character[].class)); - fury.registerSerializer(short[].class, new ShortArraySerializer(fury)); - fury.registerSerializer(Short[].class, new ObjectArraySerializer<>(fury, Short[].class)); - fury.registerSerializer(int[].class, new IntArraySerializer(fury)); - fury.registerSerializer(Integer[].class, new ObjectArraySerializer<>(fury, Integer[].class)); - fury.registerSerializer(long[].class, new LongArraySerializer(fury)); - fury.registerSerializer(Long[].class, new ObjectArraySerializer<>(fury, Long[].class)); - fury.registerSerializer(float[].class, new FloatArraySerializer(fury)); - fury.registerSerializer(Float[].class, new ObjectArraySerializer<>(fury, Float[].class)); - fury.registerSerializer(double[].class, new DoubleArraySerializer(fury)); - fury.registerSerializer(Double[].class, new ObjectArraySerializer<>(fury, Double[].class)); - fury.registerSerializer(boolean[].class, new BooleanArraySerializer(fury)); - fury.registerSerializer(Boolean[].class, new ObjectArraySerializer<>(fury, Boolean[].class)); - fury.registerSerializer(String[].class, new StringArraySerializer(fury)); + resolver.registerSerializer(short[].class, new ShortArraySerializer(fury)); + resolver.registerSerializer(Short[].class, new ObjectArraySerializer<>(fury, Short[].class)); + resolver.registerSerializer(int[].class, new IntArraySerializer(fury)); + resolver.registerSerializer( + Integer[].class, new ObjectArraySerializer<>(fury, Integer[].class)); + resolver.registerSerializer(long[].class, new LongArraySerializer(fury)); + resolver.registerSerializer(Long[].class, new ObjectArraySerializer<>(fury, Long[].class)); + resolver.registerSerializer(float[].class, new FloatArraySerializer(fury)); + resolver.registerSerializer(Float[].class, new ObjectArraySerializer<>(fury, Float[].class)); + resolver.registerSerializer(double[].class, new DoubleArraySerializer(fury)); + resolver.registerSerializer(Double[].class, new ObjectArraySerializer<>(fury, Double[].class)); + resolver.registerSerializer(boolean[].class, new BooleanArraySerializer(fury)); + resolver.registerSerializer( + Boolean[].class, new ObjectArraySerializer<>(fury, Boolean[].class)); + resolver.registerSerializer(String[].class, new StringArraySerializer(fury)); } // ########################## utils ########################## @@ -674,25 +677,16 @@ public static PrimitiveArrayBufferObject byteArrayBufferObject(byte[] array) { static { primitiveInfo.put( - boolean.class, - new int[] {Platform.BOOLEAN_ARRAY_OFFSET, 1, Types.BOOL_ARRAY}); + boolean.class, new int[] {Platform.BOOLEAN_ARRAY_OFFSET, 1, Types.BOOL_ARRAY}); primitiveInfo.put(byte.class, new int[] {Platform.BYTE_ARRAY_OFFSET, 1, Types.BINARY}); primitiveInfo.put( char.class, new int[] {Platform.CHAR_ARRAY_OFFSET, 2, Fury.NOT_SUPPORT_XLANG}); + primitiveInfo.put(short.class, new int[] {Platform.SHORT_ARRAY_OFFSET, 2, Types.INT16_ARRAY}); + primitiveInfo.put(int.class, new int[] {Platform.INT_ARRAY_OFFSET, 4, Types.INT32_ARRAY}); + primitiveInfo.put(long.class, new int[] {Platform.LONG_ARRAY_OFFSET, 8, Types.INT64_ARRAY}); + primitiveInfo.put(float.class, new int[] {Platform.FLOAT_ARRAY_OFFSET, 4, Types.FLOAT32_ARRAY}); primitiveInfo.put( - short.class, - new int[] {Platform.SHORT_ARRAY_OFFSET, 2, Types.INT16_ARRAY}); - primitiveInfo.put( - int.class, new int[] {Platform.INT_ARRAY_OFFSET, 4, Types.INT32_ARRAY}); - primitiveInfo.put( - long.class, - new int[] {Platform.LONG_ARRAY_OFFSET, 8, Types.INT64_ARRAY}); - primitiveInfo.put( - float.class, - new int[] {Platform.FLOAT_ARRAY_OFFSET, 4, Types.FLOAT32_ARRAY}); - primitiveInfo.put( - double.class, - new int[] {Platform.DOUBLE_ARRAY_OFFSET, 8, Types.FLOAT64_ARRAY}); + double.class, new int[] {Platform.DOUBLE_ARRAY_OFFSET, 8, Types.FLOAT64_ARRAY}); } public abstract static class AbstractedNonexistentArrayClassSerializer extends Serializer { diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/OptionalSerializers.java b/java/fury-core/src/main/java/org/apache/fury/serializer/OptionalSerializers.java index 640d1857f3..fad0adf7f0 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/OptionalSerializers.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/OptionalSerializers.java @@ -25,6 +25,7 @@ import java.util.OptionalLong; import org.apache.fury.Fury; import org.apache.fury.memory.MemoryBuffer; +import org.apache.fury.resolver.ClassResolver; /** * Serializers for {@link Optional}, {@link OptionalInt}, {@link OptionalLong} and {@link @@ -122,9 +123,10 @@ public OptionalDouble read(MemoryBuffer buffer) { } public static void registerDefaultSerializers(Fury fury) { - fury.registerSerializer(Optional.class, new OptionalSerializer(fury)); - fury.registerSerializer(OptionalInt.class, new OptionalIntSerializer(fury)); - fury.registerSerializer(OptionalLong.class, new OptionalLongSerializer(fury)); - fury.registerSerializer(OptionalDouble.class, new OptionalDoubleSerializer(fury)); + ClassResolver resolver = fury.getClassResolver(); + resolver.registerSerializer(Optional.class, new OptionalSerializer(fury)); + resolver.registerSerializer(OptionalInt.class, new OptionalIntSerializer(fury)); + resolver.registerSerializer(OptionalLong.class, new OptionalLongSerializer(fury)); + resolver.registerSerializer(OptionalDouble.class, new OptionalDoubleSerializer(fury)); } } diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/PrimitiveSerializers.java b/java/fury-core/src/main/java/org/apache/fury/serializer/PrimitiveSerializers.java index dc547662ca..35932ddcb2 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/PrimitiveSerializers.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/PrimitiveSerializers.java @@ -27,6 +27,7 @@ import org.apache.fury.config.LongEncoding; import org.apache.fury.memory.MemoryBuffer; import org.apache.fury.memory.Platform; +import org.apache.fury.resolver.ClassResolver; import org.apache.fury.util.Preconditions; /** Serializers for java primitive types. */ @@ -292,21 +293,22 @@ public Double read(MemoryBuffer buffer) { public static void registerDefaultSerializers(Fury fury) { // primitive types will be boxed. - fury.registerSerializer(boolean.class, new BooleanSerializer(fury, boolean.class)); - fury.registerSerializer(byte.class, new ByteSerializer(fury, byte.class)); - fury.registerSerializer(short.class, new ShortSerializer(fury, short.class)); - fury.registerSerializer(char.class, new CharSerializer(fury, char.class)); - fury.registerSerializer(int.class, new IntSerializer(fury, int.class)); - fury.registerSerializer(long.class, new LongSerializer(fury, long.class)); - fury.registerSerializer(float.class, new FloatSerializer(fury, float.class)); - fury.registerSerializer(double.class, new DoubleSerializer(fury, double.class)); - fury.registerSerializer(Boolean.class, new BooleanSerializer(fury, Boolean.class)); - fury.registerSerializer(Byte.class, new ByteSerializer(fury, Byte.class)); - fury.registerSerializer(Short.class, new ShortSerializer(fury, Short.class)); - fury.registerSerializer(Character.class, new CharSerializer(fury, Character.class)); - fury.registerSerializer(Integer.class, new IntSerializer(fury, Integer.class)); - fury.registerSerializer(Long.class, new LongSerializer(fury, Long.class)); - fury.registerSerializer(Float.class, new FloatSerializer(fury, Float.class)); - fury.registerSerializer(Double.class, new DoubleSerializer(fury, Double.class)); + ClassResolver resolver = fury.getClassResolver(); + resolver.registerSerializer(boolean.class, new BooleanSerializer(fury, boolean.class)); + resolver.registerSerializer(byte.class, new ByteSerializer(fury, byte.class)); + resolver.registerSerializer(short.class, new ShortSerializer(fury, short.class)); + resolver.registerSerializer(char.class, new CharSerializer(fury, char.class)); + resolver.registerSerializer(int.class, new IntSerializer(fury, int.class)); + resolver.registerSerializer(long.class, new LongSerializer(fury, long.class)); + resolver.registerSerializer(float.class, new FloatSerializer(fury, float.class)); + resolver.registerSerializer(double.class, new DoubleSerializer(fury, double.class)); + resolver.registerSerializer(Boolean.class, new BooleanSerializer(fury, Boolean.class)); + resolver.registerSerializer(Byte.class, new ByteSerializer(fury, Byte.class)); + resolver.registerSerializer(Short.class, new ShortSerializer(fury, Short.class)); + resolver.registerSerializer(Character.class, new CharSerializer(fury, Character.class)); + resolver.registerSerializer(Integer.class, new IntSerializer(fury, Integer.class)); + resolver.registerSerializer(Long.class, new LongSerializer(fury, Long.class)); + resolver.registerSerializer(Float.class, new FloatSerializer(fury, Float.class)); + resolver.registerSerializer(Double.class, new DoubleSerializer(fury, Double.class)); } } diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/Serializers.java b/java/fury-core/src/main/java/org/apache/fury/serializer/Serializers.java index 9e1b55ca29..a1232f7038 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/Serializers.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/Serializers.java @@ -526,19 +526,20 @@ public Object read(MemoryBuffer buffer) { } public static void registerDefaultSerializers(Fury fury) { - fury.registerSerializer(Class.class, new ClassSerializer(fury)); - fury.registerSerializer(StringBuilder.class, new StringBuilderSerializer(fury)); - fury.registerSerializer(StringBuffer.class, new StringBufferSerializer(fury)); - fury.registerSerializer(BigInteger.class, new BigIntegerSerializer(fury)); - fury.registerSerializer(BigDecimal.class, new BigDecimalSerializer(fury)); - fury.registerSerializer(AtomicBoolean.class, new AtomicBooleanSerializer(fury)); - fury.registerSerializer(AtomicInteger.class, new AtomicIntegerSerializer(fury)); - fury.registerSerializer(AtomicLong.class, new AtomicLongSerializer(fury)); - fury.registerSerializer(AtomicReference.class, new AtomicReferenceSerializer(fury)); - fury.registerSerializer(Currency.class, new CurrencySerializer(fury)); - fury.registerSerializer(URI.class, new URISerializer(fury)); - fury.registerSerializer(Pattern.class, new RegexSerializer(fury)); - fury.registerSerializer(UUID.class, new UUIDSerializer(fury)); - fury.registerSerializer(Object.class, new EmptyObjectSerializer(fury)); + ClassResolver resolver = fury.getClassResolver(); + resolver.registerSerializer(Class.class, new ClassSerializer(fury)); + resolver.registerSerializer(StringBuilder.class, new StringBuilderSerializer(fury)); + resolver.registerSerializer(StringBuffer.class, new StringBufferSerializer(fury)); + resolver.registerSerializer(BigInteger.class, new BigIntegerSerializer(fury)); + resolver.registerSerializer(BigDecimal.class, new BigDecimalSerializer(fury)); + resolver.registerSerializer(AtomicBoolean.class, new AtomicBooleanSerializer(fury)); + resolver.registerSerializer(AtomicInteger.class, new AtomicIntegerSerializer(fury)); + resolver.registerSerializer(AtomicLong.class, new AtomicLongSerializer(fury)); + resolver.registerSerializer(AtomicReference.class, new AtomicReferenceSerializer(fury)); + resolver.registerSerializer(Currency.class, new CurrencySerializer(fury)); + resolver.registerSerializer(URI.class, new URISerializer(fury)); + resolver.registerSerializer(Pattern.class, new RegexSerializer(fury)); + resolver.registerSerializer(UUID.class, new UUIDSerializer(fury)); + resolver.registerSerializer(Object.class, new EmptyObjectSerializer(fury)); } } diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/TimeSerializers.java b/java/fury-core/src/main/java/org/apache/fury/serializer/TimeSerializers.java index 2a5de4be4d..d380db13c8 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/TimeSerializers.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/TimeSerializers.java @@ -41,6 +41,7 @@ import java.util.TimeZone; import org.apache.fury.Fury; import org.apache.fury.memory.MemoryBuffer; +import org.apache.fury.resolver.ClassResolver; import org.apache.fury.util.DateTimeUtils; /** Serializers for all time related types. */ @@ -620,22 +621,23 @@ public OffsetDateTime read(MemoryBuffer buffer) { } public static void registerDefaultSerializers(Fury fury) { - fury.registerSerializer(Date.class, new DateSerializer(fury)); - fury.registerSerializer(java.sql.Date.class, new SqlDateSerializer(fury)); - fury.registerSerializer(Time.class, new SqlTimeSerializer(fury)); - fury.registerSerializer(Timestamp.class, new TimestampSerializer(fury)); - fury.registerSerializer(LocalDate.class, new LocalDateSerializer(fury)); - fury.registerSerializer(LocalTime.class, new LocalTimeSerializer(fury)); - fury.registerSerializer(LocalDateTime.class, new LocalDateTimeSerializer(fury)); - fury.registerSerializer(Instant.class, new InstantSerializer(fury)); - fury.registerSerializer(Duration.class, new DurationSerializer(fury)); - fury.registerSerializer(ZoneOffset.class, new ZoneOffsetSerializer(fury)); - fury.registerSerializer(ZonedDateTime.class, new ZonedDateTimeSerializer(fury)); - fury.registerSerializer(Year.class, new YearSerializer(fury)); - fury.registerSerializer(YearMonth.class, new YearMonthSerializer(fury)); - fury.registerSerializer(MonthDay.class, new MonthDaySerializer(fury)); - fury.registerSerializer(Period.class, new PeriodSerializer(fury)); - fury.registerSerializer(OffsetTime.class, new OffsetTimeSerializer(fury)); - fury.registerSerializer(OffsetDateTime.class, new OffsetDateTimeSerializer(fury)); + ClassResolver resolver = fury.getClassResolver(); + resolver.registerSerializer(Date.class, new DateSerializer(fury)); + resolver.registerSerializer(java.sql.Date.class, new SqlDateSerializer(fury)); + resolver.registerSerializer(Time.class, new SqlTimeSerializer(fury)); + resolver.registerSerializer(Timestamp.class, new TimestampSerializer(fury)); + resolver.registerSerializer(LocalDate.class, new LocalDateSerializer(fury)); + resolver.registerSerializer(LocalTime.class, new LocalTimeSerializer(fury)); + resolver.registerSerializer(LocalDateTime.class, new LocalDateTimeSerializer(fury)); + resolver.registerSerializer(Instant.class, new InstantSerializer(fury)); + resolver.registerSerializer(Duration.class, new DurationSerializer(fury)); + resolver.registerSerializer(ZoneOffset.class, new ZoneOffsetSerializer(fury)); + resolver.registerSerializer(ZonedDateTime.class, new ZonedDateTimeSerializer(fury)); + resolver.registerSerializer(Year.class, new YearSerializer(fury)); + resolver.registerSerializer(YearMonth.class, new YearMonthSerializer(fury)); + resolver.registerSerializer(MonthDay.class, new MonthDaySerializer(fury)); + resolver.registerSerializer(Period.class, new PeriodSerializer(fury)); + resolver.registerSerializer(OffsetTime.class, new OffsetTimeSerializer(fury)); + resolver.registerSerializer(OffsetDateTime.class, new OffsetDateTimeSerializer(fury)); } } diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/CollectionSerializers.java b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/CollectionSerializers.java index 2dc7aabccd..6b0bbd0edb 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/CollectionSerializers.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/CollectionSerializers.java @@ -633,46 +633,48 @@ public void write(MemoryBuffer buffer, T value) { // TODO Support ArraySubListSerializer, SubListSerializer public static void registerDefaultSerializers(Fury fury) { - fury.registerSerializer(ArrayList.class, new ArrayListSerializer(fury)); + ClassResolver resolver = fury.getClassResolver(); + resolver.registerSerializer(ArrayList.class, new ArrayListSerializer(fury)); Class arrayAsListClass = Arrays.asList(1, 2).getClass(); - fury.registerSerializer(arrayAsListClass, new ArraysAsListSerializer(fury, arrayAsListClass)); - fury.registerSerializer( + resolver.registerSerializer( + arrayAsListClass, new ArraysAsListSerializer(fury, arrayAsListClass)); + resolver.registerSerializer( LinkedList.class, new CollectionSerializer(fury, LinkedList.class, true)); - fury.registerSerializer(HashSet.class, new HashSetSerializer(fury)); - fury.registerSerializer(LinkedHashSet.class, new LinkedHashSetSerializer(fury)); - fury.registerSerializer(TreeSet.class, new SortedSetSerializer<>(fury, TreeSet.class)); - fury.registerSerializer( + resolver.registerSerializer(HashSet.class, new HashSetSerializer(fury)); + resolver.registerSerializer(LinkedHashSet.class, new LinkedHashSetSerializer(fury)); + resolver.registerSerializer(TreeSet.class, new SortedSetSerializer<>(fury, TreeSet.class)); + resolver.registerSerializer( Collections.EMPTY_LIST.getClass(), new EmptyListSerializer(fury, (Class>) Collections.EMPTY_LIST.getClass())); - fury.registerSerializer( + resolver.registerSerializer( Collections.emptySortedSet().getClass(), new EmptySortedSetSerializer( fury, (Class>) Collections.emptySortedSet().getClass())); - fury.registerSerializer( + resolver.registerSerializer( Collections.EMPTY_SET.getClass(), new EmptySetSerializer(fury, (Class>) Collections.EMPTY_SET.getClass())); - fury.registerSerializer( + resolver.registerSerializer( Collections.singletonList(null).getClass(), new CollectionsSingletonListSerializer( fury, (Class>) Collections.singletonList(null).getClass())); - fury.registerSerializer( + resolver.registerSerializer( Collections.singleton(null).getClass(), new CollectionsSingletonSetSerializer( fury, (Class>) Collections.singleton(null).getClass())); - fury.registerSerializer( + resolver.registerSerializer( ConcurrentSkipListSet.class, new ConcurrentSkipListSetSerializer(fury, ConcurrentSkipListSet.class)); - fury.registerSerializer(Vector.class, new VectorSerializer(fury, Vector.class)); - fury.registerSerializer(ArrayDeque.class, new ArrayDequeSerializer(fury, ArrayDeque.class)); - fury.registerSerializer(BitSet.class, new BitSetSerializer(fury, BitSet.class)); - fury.registerSerializer( + resolver.registerSerializer(Vector.class, new VectorSerializer(fury, Vector.class)); + resolver.registerSerializer(ArrayDeque.class, new ArrayDequeSerializer(fury, ArrayDeque.class)); + resolver.registerSerializer(BitSet.class, new BitSetSerializer(fury, BitSet.class)); + resolver.registerSerializer( PriorityQueue.class, new PriorityQueueSerializer(fury, PriorityQueue.class)); - fury.registerSerializer( + resolver.registerSerializer( CopyOnWriteArrayList.class, new CopyOnWriteArrayListSerializer(fury, CopyOnWriteArrayList.class)); final Class setFromMapClass = Collections.newSetFromMap(new HashMap<>()).getClass(); - fury.registerSerializer(setFromMapClass, new SetFromMapSerializer(fury, setFromMapClass)); - fury.registerSerializer( + resolver.registerSerializer(setFromMapClass, new SetFromMapSerializer(fury, setFromMapClass)); + resolver.registerSerializer( ConcurrentHashMap.KeySetView.class, new ConcurrentHashMapKeySetView(fury, ConcurrentHashMap.KeySetView.class)); } diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/GuavaCollectionSerializers.java b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/GuavaCollectionSerializers.java index 7d5791ad68..d1d04b1fad 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/GuavaCollectionSerializers.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/GuavaCollectionSerializers.java @@ -38,6 +38,7 @@ import org.apache.fury.Fury; import org.apache.fury.memory.MemoryBuffer; import org.apache.fury.memory.Platform; +import org.apache.fury.resolver.ClassResolver; import org.apache.fury.util.unsafe._JDKAccess; /** Serializers for common guava types. */ @@ -360,65 +361,66 @@ public static void registerDefaultSerializers(Fury fury) { // inconsistent if peers load different version of guava. // For example: guava 20 return ImmutableBiMap for ImmutableMap.of(), but guava 27 return // ImmutableMap. + ClassResolver resolver = fury.getClassResolver(); Class cls = loadClass(pkg + ".RegularImmutableBiMap", ImmutableBiMap.of("k1", 1, "k2", 4).getClass()); - fury.registerSerializer(cls, new ImmutableBiMapSerializer(fury, cls)); + resolver.registerSerializer(cls, new ImmutableBiMapSerializer(fury, cls)); cls = loadClass(pkg + ".SingletonImmutableBiMap", ImmutableBiMap.of(1, 2).getClass()); - fury.registerSerializer(cls, new ImmutableBiMapSerializer(fury, cls)); + resolver.registerSerializer(cls, new ImmutableBiMapSerializer(fury, cls)); cls = loadClass(pkg + ".RegularImmutableMap", ImmutableMap.of("k1", 1, "k2", 2).getClass()); - fury.registerSerializer(cls, new ImmutableMapSerializer(fury, cls)); + resolver.registerSerializer(cls, new ImmutableMapSerializer(fury, cls)); cls = loadClass(pkg + ".RegularImmutableList", ImmutableList.of().getClass()); - fury.registerSerializer(cls, new RegularImmutableListSerializer(fury, cls)); + resolver.registerSerializer(cls, new RegularImmutableListSerializer(fury, cls)); cls = loadClass(pkg + ".SingletonImmutableList", ImmutableList.of(1).getClass()); - fury.registerSerializer(cls, new ImmutableListSerializer(fury, cls)); + resolver.registerSerializer(cls, new ImmutableListSerializer(fury, cls)); cls = loadClass(pkg + ".RegularImmutableSet", ImmutableSet.of(1, 2).getClass()); - fury.registerSerializer(cls, new ImmutableSetSerializer(fury, cls)); + resolver.registerSerializer(cls, new ImmutableSetSerializer(fury, cls)); cls = loadClass(pkg + ".SingletonImmutableSet", ImmutableSet.of(1).getClass()); - fury.registerSerializer(cls, new ImmutableSetSerializer(fury, cls)); + resolver.registerSerializer(cls, new ImmutableSetSerializer(fury, cls)); // sorted set/map doesn't support xlang. cls = loadClass(pkg + ".RegularImmutableSortedSet", ImmutableSortedSet.of(1, 2).getClass()); - fury.registerSerializer(cls, new ImmutableSortedSetSerializer<>(fury, cls)); + resolver.registerSerializer(cls, new ImmutableSortedSetSerializer<>(fury, cls)); cls = loadClass(pkg + ".ImmutableSortedMap", ImmutableSortedMap.of(1, 2).getClass()); - fury.registerSerializer(cls, new ImmutableSortedMapSerializer<>(fury, cls)); + resolver.registerSerializer(cls, new ImmutableSortedMapSerializer<>(fury, cls)); // Guava version before 19.0, of() return // EmptyImmutableSet/EmptyImmutableBiMap/EmptyImmutableSortedMap/EmptyImmutableSortedSet // we register if class exist or register empty to deserialize. if (checkClassExist(pkg + ".EmptyImmutableSet")) { cls = loadClass(pkg + ".EmptyImmutableSet", ImmutableSet.of().getClass()); - fury.registerSerializer(cls, new ImmutableSetSerializer(fury, cls)); + resolver.registerSerializer(cls, new ImmutableSetSerializer(fury, cls)); } else { class GuavaEmptySet {} cls = GuavaEmptySet.class; - fury.registerSerializer(cls, new ImmutableSetSerializer(fury, cls)); + resolver.registerSerializer(cls, new ImmutableSetSerializer(fury, cls)); } if (checkClassExist(pkg + ".EmptyImmutableBiMap")) { cls = loadClass(pkg + ".EmptyImmutableBiMap", ImmutableBiMap.of().getClass()); - fury.registerSerializer(cls, new ImmutableMapSerializer(fury, cls)); + resolver.registerSerializer(cls, new ImmutableMapSerializer(fury, cls)); } else { class GuavaEmptyBiMap {} cls = GuavaEmptyBiMap.class; - fury.registerSerializer(cls, new ImmutableMapSerializer(fury, cls)); + resolver.registerSerializer(cls, new ImmutableMapSerializer(fury, cls)); } if (checkClassExist(pkg + ".EmptyImmutableSortedSet")) { cls = loadClass(pkg + ".EmptyImmutableSortedSet", ImmutableSortedSet.of().getClass()); - fury.registerSerializer(cls, new ImmutableSortedSetSerializer(fury, cls)); + resolver.registerSerializer(cls, new ImmutableSortedSetSerializer(fury, cls)); } else { class GuavaEmptySortedSet {} cls = GuavaEmptySortedSet.class; - fury.registerSerializer(cls, new ImmutableSortedSetSerializer(fury, cls)); + resolver.registerSerializer(cls, new ImmutableSortedSetSerializer(fury, cls)); } if (checkClassExist(pkg + ".EmptyImmutableSortedMap")) { cls = loadClass(pkg + ".EmptyImmutableSortedMap", ImmutableSortedMap.of().getClass()); - fury.registerSerializer(cls, new ImmutableSortedMapSerializer(fury, cls)); + resolver.registerSerializer(cls, new ImmutableSortedMapSerializer(fury, cls)); } else { class GuavaEmptySortedMap {} cls = GuavaEmptySortedMap.class; - fury.registerSerializer(cls, new ImmutableSortedMapSerializer(fury, cls)); + resolver.registerSerializer(cls, new ImmutableSortedMapSerializer(fury, cls)); } } diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/ImmutableCollectionSerializers.java b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/ImmutableCollectionSerializers.java index 3667b8cc55..02fd0d2c98 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/ImmutableCollectionSerializers.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/ImmutableCollectionSerializers.java @@ -32,6 +32,7 @@ import org.apache.fury.Fury; import org.apache.fury.memory.MemoryBuffer; import org.apache.fury.memory.Platform; +import org.apache.fury.resolver.ClassResolver; import org.apache.fury.util.unsafe._JDKAccess; /** Serializers for jdk9+ java.util.ImmutableCollections. */ @@ -203,12 +204,13 @@ public Map onMapRead(Map map) { } public static void registerSerializers(Fury fury) { - fury.registerSerializer(List12, new ImmutableListSerializer(fury, List12)); - fury.registerSerializer(ListN, new ImmutableListSerializer(fury, ListN)); - fury.registerSerializer(SubList, new ImmutableListSerializer(fury, SubList)); - fury.registerSerializer(Set12, new ImmutableSetSerializer(fury, Set12)); - fury.registerSerializer(SetN, new ImmutableSetSerializer(fury, SetN)); - fury.registerSerializer(Map1, new ImmutableMapSerializer(fury, Map1)); - fury.registerSerializer(MapN, new ImmutableMapSerializer(fury, MapN)); + ClassResolver resolver = fury.getClassResolver(); + resolver.registerSerializer(List12, new ImmutableListSerializer(fury, List12)); + resolver.registerSerializer(ListN, new ImmutableListSerializer(fury, ListN)); + resolver.registerSerializer(SubList, new ImmutableListSerializer(fury, SubList)); + resolver.registerSerializer(Set12, new ImmutableSetSerializer(fury, Set12)); + resolver.registerSerializer(SetN, new ImmutableSetSerializer(fury, SetN)); + resolver.registerSerializer(Map1, new ImmutableMapSerializer(fury, Map1)); + resolver.registerSerializer(MapN, new ImmutableMapSerializer(fury, MapN)); } } diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapSerializers.java b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapSerializers.java index 7fc9e1a50c..d4c22502ae 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapSerializers.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapSerializers.java @@ -389,27 +389,28 @@ public void write(MemoryBuffer buffer, T value) { // TODO(chaokunyang) support ConcurrentSkipListMap.SubMap mo efficiently. public static void registerDefaultSerializers(Fury fury) { - fury.registerSerializer(HashMap.class, new HashMapSerializer(fury)); + ClassResolver resolver = fury.getClassResolver(); + resolver.registerSerializer(HashMap.class, new HashMapSerializer(fury)); fury.getClassResolver() .registerSerializer(LinkedHashMap.class, new LinkedHashMapSerializer(fury)); - fury.registerSerializer(TreeMap.class, new SortedMapSerializer<>(fury, TreeMap.class)); - fury.registerSerializer( + resolver.registerSerializer(TreeMap.class, new SortedMapSerializer<>(fury, TreeMap.class)); + resolver.registerSerializer( Collections.EMPTY_MAP.getClass(), new EmptyMapSerializer(fury, (Class>) Collections.EMPTY_MAP.getClass())); - fury.registerSerializer( + resolver.registerSerializer( Collections.emptySortedMap().getClass(), new EmptySortedMapSerializer( fury, (Class>) Collections.emptySortedMap().getClass())); - fury.registerSerializer( + resolver.registerSerializer( Collections.singletonMap(null, null).getClass(), new SingletonMapSerializer( fury, (Class>) Collections.singletonMap(null, null).getClass())); - fury.registerSerializer( + resolver.registerSerializer( ConcurrentHashMap.class, new ConcurrentHashMapSerializer(fury, ConcurrentHashMap.class)); - fury.registerSerializer( + resolver.registerSerializer( ConcurrentSkipListMap.class, new ConcurrentSkipListMapSerializer(fury, ConcurrentSkipListMap.class)); - fury.registerSerializer(EnumMap.class, new EnumMapSerializer(fury)); - fury.registerSerializer(LazyMap.class, new LazyMapSerializer(fury)); + resolver.registerSerializer(EnumMap.class, new EnumMapSerializer(fury)); + resolver.registerSerializer(LazyMap.class, new LazyMapSerializer(fury)); } } diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/SynchronizedSerializers.java b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/SynchronizedSerializers.java index 82d5bc4e86..d997d9640c 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/SynchronizedSerializers.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/SynchronizedSerializers.java @@ -40,6 +40,7 @@ import org.apache.fury.logging.LoggerFactory; import org.apache.fury.memory.MemoryBuffer; import org.apache.fury.memory.Platform; +import org.apache.fury.resolver.ClassResolver; import org.apache.fury.serializer.Serializer; import org.apache.fury.util.ExceptionUtils; @@ -199,8 +200,9 @@ static Tuple2, Function>[] synchronizedFactories() { */ public static void registerSerializers(Fury fury) { try { + ClassResolver resolver = fury.getClassResolver(); for (Tuple2, Function> factory : synchronizedFactories()) { - fury.registerSerializer(factory.f0, createSerializer(fury, factory)); + resolver.registerSerializer(factory.f0, createSerializer(fury, factory)); } } catch (Throwable e) { ExceptionUtils.ignore(e); diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/UnmodifiableSerializers.java b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/UnmodifiableSerializers.java index 3f54860a51..727b265e59 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/UnmodifiableSerializers.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/UnmodifiableSerializers.java @@ -39,6 +39,7 @@ import org.apache.fury.logging.LoggerFactory; import org.apache.fury.memory.MemoryBuffer; import org.apache.fury.memory.Platform; +import org.apache.fury.resolver.ClassResolver; import org.apache.fury.serializer.Serializer; import org.apache.fury.util.ExceptionUtils; import org.apache.fury.util.Preconditions; @@ -197,8 +198,9 @@ static Tuple2, Function>[] unmodifiableFactories() { */ public static void registerSerializers(Fury fury) { try { + ClassResolver resolver = fury.getClassResolver(); for (Tuple2, Function> factory : unmodifiableFactories()) { - fury.registerSerializer(factory.f0, createSerializer(fury, factory)); + resolver.registerSerializer(factory.f0, createSerializer(fury, factory)); } } catch (Throwable e) { ExceptionUtils.ignore(e); diff --git a/java/fury-core/src/main/java/org/apache/fury/type/Types.java b/java/fury-core/src/main/java/org/apache/fury/type/Types.java index 09c70df952..ceec6f1572 100644 --- a/java/fury-core/src/main/java/org/apache/fury/type/Types.java +++ b/java/fury-core/src/main/java/org/apache/fury/type/Types.java @@ -1,7 +1,6 @@ package org.apache.fury.type; public class Types { - public static final int NOT_SUPPORT_XLANG = 0; public static final int BOOL = 1; public static final int INT8 = 2; public static final int INT16 = 3; @@ -14,25 +13,31 @@ public class Types { public static final int FLOAT32 = 10; public static final int FLOAT64 = 11; public static final int STRING = 12; + public static final int ENUM = 13; - public static final int LIST = 14; - public static final int SET = 15; - public static final int MAP = 16; - public static final int DURATION = 17; - public static final int TIMESTAMP = 18; - public static final int DECIMAL = 19; - public static final int BINARY = 20; - public static final int ARRAY = 21; - public static final int BOOL_ARRAY = 22; - public static final int INT8_ARRAY = 23; - public static final int INT16_ARRAY = 24; - public static final int INT32_ARRAY = 25; - public static final int INT64_ARRAY = 26; - public static final int FLOAT16_ARRAY = 27; - public static final int FLOAT32_ARRAY = 28; - public static final int FLOAT64_ARRAY = 29; - public static final int TENSOR = 30; - public static final int SPARSE_TENSOR = 31; - public static final int ARROW_RECORD_BATCH = 32; - public static final int ARROW_TABLE = 33; + public static final int NS_ENUM = 14; + public static final int STRUCT = 15; + public static final int NS_STRUCT = 16; + public static final int EXT = 17; + public static final int NS_EXT = 18; + public static final int LIST = 19; + public static final int SET = 20; + public static final int MAP = 21; + public static final int DURATION = 22; + public static final int TIMESTAMP = 23; + public static final int DECIMAL = 24; + public static final int BINARY = 25; + public static final int ARRAY = 26; + public static final int BOOL_ARRAY = 27; + public static final int INT8_ARRAY = 28; + public static final int INT16_ARRAY = 29; + public static final int INT32_ARRAY = 30; + public static final int INT64_ARRAY = 31; + public static final int FLOAT16_ARRAY = 32; + public static final int FLOAT32_ARRAY = 33; + public static final int FLOAT64_ARRAY = 34; + public static final int TENSOR = 35; + public static final int SPARSE_TENSOR = 36; + public static final int ARROW_RECORD_BATCH = 37; + public static final int ARROW_TABLE = 38; } diff --git a/java/fury-format/src/main/java/org/apache/fury/format/type/DataTypes.java b/java/fury-format/src/main/java/org/apache/fury/format/type/DataTypes.java index 7ee8b8583f..7f78aa9d53 100644 --- a/java/fury-format/src/main/java/org/apache/fury/format/type/DataTypes.java +++ b/java/fury-format/src/main/java/org/apache/fury/format/type/DataTypes.java @@ -47,172 +47,180 @@ /** Arrow data type utils. */ public class DataTypes { - public static Field PRIMITIVE_BOOLEAN_ARRAY_FIELD = primitiveArrayField(org.apache.arrow.vector.types.pojo.ArrowType.Bool.INSTANCE); + public static Field PRIMITIVE_BOOLEAN_ARRAY_FIELD = + primitiveArrayField(org.apache.arrow.vector.types.pojo.ArrowType.Bool.INSTANCE); public static Field PRIMITIVE_BYTE_ARRAY_FIELD = primitiveArrayField(intType(8)); public static Field PRIMITIVE_SHORT_ARRAY_FIELD = primitiveArrayField(intType(16)); public static Field PRIMITIVE_INT_ARRAY_FIELD = primitiveArrayField(intType(32)); public static Field PRIMITIVE_LONG_ARRAY_FIELD = primitiveArrayField(intType(64)); public static Field PRIMITIVE_FLOAT_ARRAY_FIELD = - primitiveArrayField(new org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)); + primitiveArrayField( + new org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint( + FloatingPointPrecision.SINGLE)); public static Field PRIMITIVE_DOUBLE_ARRAY_FIELD = - primitiveArrayField(new org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)); + primitiveArrayField( + new org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint( + FloatingPointPrecision.DOUBLE)); // Array item field default name public static final String ARRAY_ITEM_NAME = "item"; - private static final org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeVisitor typeWidthVisitor = - new DefaultTypeVisitor() { + private static final org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeVisitor + typeWidthVisitor = + new DefaultTypeVisitor() { - @Override - public Integer visit(org.apache.arrow.vector.types.pojo.ArrowType.Struct type) { - return -1; - } - - @Override - public Integer visit(org.apache.arrow.vector.types.pojo.ArrowType.List type) { - return -1; - } + @Override + public Integer visit(org.apache.arrow.vector.types.pojo.ArrowType.Struct type) { + return -1; + } - @Override - public Integer visit(org.apache.arrow.vector.types.pojo.ArrowType.Map type) { - return -1; - } + @Override + public Integer visit(org.apache.arrow.vector.types.pojo.ArrowType.List type) { + return -1; + } - @Override - public Integer visit(org.apache.arrow.vector.types.pojo.ArrowType.Bool type) { - return 1; - } + @Override + public Integer visit(org.apache.arrow.vector.types.pojo.ArrowType.Map type) { + return -1; + } - @Override - public Integer visit(org.apache.arrow.vector.types.pojo.ArrowType.Int type) { - return type.getBitWidth() / 8; - } + @Override + public Integer visit(org.apache.arrow.vector.types.pojo.ArrowType.Bool type) { + return 1; + } - @Override - public Integer visit(org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint type) { - switch (type.getPrecision()) { - case SINGLE: - return 4; - case DOUBLE: - return 8; - default: - return unsupported(type); - } - } + @Override + public Integer visit(org.apache.arrow.vector.types.pojo.ArrowType.Int type) { + return type.getBitWidth() / 8; + } - @Override - public Integer visit(org.apache.arrow.vector.types.pojo.ArrowType.Date type) { - return 4; - } + @Override + public Integer visit(org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint type) { + switch (type.getPrecision()) { + case SINGLE: + return 4; + case DOUBLE: + return 8; + default: + return unsupported(type); + } + } - @Override - public Integer visit(org.apache.arrow.vector.types.pojo.ArrowType.Timestamp type) { - return 8; - } + @Override + public Integer visit(org.apache.arrow.vector.types.pojo.ArrowType.Date type) { + return 4; + } - @Override - public Integer visit(org.apache.arrow.vector.types.pojo.ArrowType.Binary type) { - return -1; - } + @Override + public Integer visit(org.apache.arrow.vector.types.pojo.ArrowType.Timestamp type) { + return 8; + } - @Override - public Integer visit(org.apache.arrow.vector.types.pojo.ArrowType.Decimal type) { - return -1; - } + @Override + public Integer visit(org.apache.arrow.vector.types.pojo.ArrowType.Binary type) { + return -1; + } - @Override - public Integer visit(org.apache.arrow.vector.types.pojo.ArrowType.Utf8 type) { - return -1; - } - }; + @Override + public Integer visit(org.apache.arrow.vector.types.pojo.ArrowType.Decimal type) { + return -1; + } - private static final org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeVisitor typeIdVisitor = - new DefaultTypeVisitor() { + @Override + public Integer visit(org.apache.arrow.vector.types.pojo.ArrowType.Utf8 type) { + return -1; + } + }; - @Override - public ArrowType visit(org.apache.arrow.vector.types.pojo.ArrowType.Bool type) { - return ArrowType.BOOL; - } + private static final org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeVisitor + typeIdVisitor = + new DefaultTypeVisitor() { - @Override - public ArrowType visit(org.apache.arrow.vector.types.pojo.ArrowType.Int type) { - if (type.getIsSigned()) { - int byteWidth = type.getBitWidth() / 8; - switch (byteWidth) { - case 1: - return ArrowType.INT8; - case 2: - return ArrowType.INT16; - case 4: - return ArrowType.INT32; - case 8: - return ArrowType.INT64; - default: - return unsupported(type); + @Override + public ArrowType visit(org.apache.arrow.vector.types.pojo.ArrowType.Bool type) { + return ArrowType.BOOL; } - } - return unsupported(type); - } - @Override - public ArrowType visit(org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint type) { - switch (type.getPrecision()) { - case SINGLE: - return ArrowType.FLOAT; - case DOUBLE: - return ArrowType.DOUBLE; - default: + @Override + public ArrowType visit(org.apache.arrow.vector.types.pojo.ArrowType.Int type) { + if (type.getIsSigned()) { + int byteWidth = type.getBitWidth() / 8; + switch (byteWidth) { + case 1: + return ArrowType.INT8; + case 2: + return ArrowType.INT16; + case 4: + return ArrowType.INT32; + case 8: + return ArrowType.INT64; + default: + return unsupported(type); + } + } return unsupported(type); - } - } + } - @Override - public ArrowType visit(org.apache.arrow.vector.types.pojo.ArrowType.Date type) { - switch (type.getUnit()) { - case DAY: - return ArrowType.DATE32; - case MILLISECOND: - return ArrowType.DATE64; - default: - return unsupported(type); - } - } + @Override + public ArrowType visit( + org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint type) { + switch (type.getPrecision()) { + case SINGLE: + return ArrowType.FLOAT; + case DOUBLE: + return ArrowType.DOUBLE; + default: + return unsupported(type); + } + } - @Override - public ArrowType visit(org.apache.arrow.vector.types.pojo.ArrowType.Timestamp type) { - return ArrowType.TIMESTAMP; - } + @Override + public ArrowType visit(org.apache.arrow.vector.types.pojo.ArrowType.Date type) { + switch (type.getUnit()) { + case DAY: + return ArrowType.DATE32; + case MILLISECOND: + return ArrowType.DATE64; + default: + return unsupported(type); + } + } - @Override - public ArrowType visit(org.apache.arrow.vector.types.pojo.ArrowType.Binary type) { - return ArrowType.BINARY; - } + @Override + public ArrowType visit(org.apache.arrow.vector.types.pojo.ArrowType.Timestamp type) { + return ArrowType.TIMESTAMP; + } - @Override - public ArrowType visit(org.apache.arrow.vector.types.pojo.ArrowType.Decimal type) { - return ArrowType.DECIMAL; - } + @Override + public ArrowType visit(org.apache.arrow.vector.types.pojo.ArrowType.Binary type) { + return ArrowType.BINARY; + } - @Override - public ArrowType visit(org.apache.arrow.vector.types.pojo.ArrowType.Utf8 type) { - return ArrowType.STRING; - } + @Override + public ArrowType visit(org.apache.arrow.vector.types.pojo.ArrowType.Decimal type) { + return ArrowType.DECIMAL; + } - @Override - public ArrowType visit(org.apache.arrow.vector.types.pojo.ArrowType.Struct type) { - return ArrowType.STRUCT; - } + @Override + public ArrowType visit(org.apache.arrow.vector.types.pojo.ArrowType.Utf8 type) { + return ArrowType.STRING; + } - @Override - public ArrowType visit(org.apache.arrow.vector.types.pojo.ArrowType.List type) { - return ArrowType.LIST; - } + @Override + public ArrowType visit(org.apache.arrow.vector.types.pojo.ArrowType.Struct type) { + return ArrowType.STRUCT; + } - @Override - public ArrowType visit(org.apache.arrow.vector.types.pojo.ArrowType.Map type) { - return ArrowType.MAP; - } - }; + @Override + public ArrowType visit(org.apache.arrow.vector.types.pojo.ArrowType.List type) { + return ArrowType.LIST; + } + + @Override + public ArrowType visit(org.apache.arrow.vector.types.pojo.ArrowType.Map type) { + return ArrowType.MAP; + } + }; public static int getTypeWidth(org.apache.arrow.vector.types.pojo.ArrowType type) { return type.accept(typeWidthVisitor); @@ -251,11 +259,13 @@ public static org.apache.arrow.vector.types.pojo.ArrowType.Int int64() { } public static org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint float32() { - return new org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE); + return new org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint( + FloatingPointPrecision.SINGLE); } public static org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint float64() { - return new org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE); + return new org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint( + FloatingPointPrecision.DOUBLE); } public static org.apache.arrow.vector.types.pojo.ArrowType.Date date32() { @@ -282,7 +292,8 @@ public static org.apache.arrow.vector.types.pojo.ArrowType.Decimal decimal() { return decimal(DecimalUtils.MAX_PRECISION, DecimalUtils.MAX_SCALE); } - public static org.apache.arrow.vector.types.pojo.ArrowType.Decimal decimal(int precision, int scale) { + public static org.apache.arrow.vector.types.pojo.ArrowType.Decimal decimal( + int precision, int scale) { return new org.apache.arrow.vector.types.pojo.ArrowType.Decimal(precision, scale); } @@ -295,7 +306,11 @@ public static Field field(String name, FieldType fieldType) { return field(name, fieldType, Collections.emptyList()); } - public static Field field(String name, boolean nullable, org.apache.arrow.vector.types.pojo.ArrowType type, Field... children) { + public static Field field( + String name, + boolean nullable, + org.apache.arrow.vector.types.pojo.ArrowType type, + Field... children) { return field(name, new FieldType(nullable, type, null), children); } @@ -303,11 +318,16 @@ public static Field field(String name, FieldType fieldType, Field... children) { return field(name, fieldType, Arrays.asList(children)); } - public static Field field(String name, boolean nullable, org.apache.arrow.vector.types.pojo.ArrowType type, List children) { + public static Field field( + String name, + boolean nullable, + org.apache.arrow.vector.types.pojo.ArrowType type, + List children) { return field(name, new FieldType(nullable, type, null), children); } - public static Field field(String name, org.apache.arrow.vector.types.pojo.ArrowType type, Field... children) { + public static Field field( + String name, org.apache.arrow.vector.types.pojo.ArrowType type, Field... children) { return field(name, true, type, children); } @@ -315,7 +335,8 @@ public static Field field(String name, FieldType fieldType, List children return new ExtField(name, fieldType, children); } - public static Field notNullField(String name, org.apache.arrow.vector.types.pojo.ArrowType type, Field... children) { + public static Field notNullField( + String name, org.apache.arrow.vector.types.pojo.ArrowType type, Field... children) { return field(name, false, type, children); } @@ -328,7 +349,8 @@ public static Field primitiveArrayField(org.apache.arrow.vector.types.pojo.Arrow return primitiveArrayField("", type); } - public static Field primitiveArrayField(String name, org.apache.arrow.vector.types.pojo.ArrowType type) { + public static Field primitiveArrayField( + String name, org.apache.arrow.vector.types.pojo.ArrowType type) { return field( name, FieldType.nullable(org.apache.arrow.vector.types.pojo.ArrowType.List.INSTANCE), @@ -363,7 +385,9 @@ public static Field arrayField(Field valueField) { public static Field arrayField(String name, Field valueField) { return field( - name, FieldType.nullable(org.apache.arrow.vector.types.pojo.ArrowType.List.INSTANCE), Collections.singletonList(valueField)); + name, + FieldType.nullable(org.apache.arrow.vector.types.pojo.ArrowType.List.INSTANCE), + Collections.singletonList(valueField)); } public static Field arrayElementField(Field field) { @@ -371,11 +395,16 @@ public static Field arrayElementField(Field field) { } /* ========================= map field utils start ========================= */ - public static Field mapField(org.apache.arrow.vector.types.pojo.ArrowType keyType, org.apache.arrow.vector.types.pojo.ArrowType itemType) { + public static Field mapField( + org.apache.arrow.vector.types.pojo.ArrowType keyType, + org.apache.arrow.vector.types.pojo.ArrowType itemType) { return mapField("", keyType, itemType); } - public static Field mapField(String name, org.apache.arrow.vector.types.pojo.ArrowType keyType, org.apache.arrow.vector.types.pojo.ArrowType itemType) { + public static Field mapField( + String name, + org.apache.arrow.vector.types.pojo.ArrowType keyType, + org.apache.arrow.vector.types.pojo.ArrowType itemType) { return mapField( name, field(MapVector.KEY_NAME, false, keyType), @@ -390,7 +419,8 @@ public static Field mapField(String name, Field keyField, Field itemField) { Preconditions.checkArgument(!keyField.isNullable(), "Map's keys must be non-nullable"); // Map's key-item pairs must be non-nullable structs Field valueField = structField(false, keyField, itemField); - return field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Map(false), valueField); + return field( + name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Map(false), valueField); } public static Field keyFieldForMap(Field mapField) { @@ -448,11 +478,13 @@ public static Field structField(boolean nullable, Field... fields) { } public static Field structField(String name, boolean nullable, Field... fields) { - return field(name, nullable, org.apache.arrow.vector.types.pojo.ArrowType.Struct.INSTANCE, fields); + return field( + name, nullable, org.apache.arrow.vector.types.pojo.ArrowType.Struct.INSTANCE, fields); } public static Field structField(String name, boolean nullable, List fields) { - return field(name, nullable, org.apache.arrow.vector.types.pojo.ArrowType.Struct.INSTANCE, fields); + return field( + name, nullable, org.apache.arrow.vector.types.pojo.ArrowType.Struct.INSTANCE, fields); } /* ========================= struct field utils end ========================= */ From 3eba2d971f4f8ef600030d49cf3f834b9d7b2a3f Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 14 Jul 2024 15:42:09 +0800 Subject: [PATCH 09/18] refactor type system --- .../src/main/java/org/apache/fury/Fury.java | 99 ++++++++++++++-- .../org/apache/fury/resolver/ClassInfo.java | 4 +- .../apache/fury/resolver/ClassResolver.java | 2 +- .../apache/fury/resolver/XtypeResolver.java | 110 +++++++++++++++--- .../fury/serializer/StructSerializer.java | 5 +- 5 files changed, 183 insertions(+), 37 deletions(-) diff --git a/java/fury-core/src/main/java/org/apache/fury/Fury.java b/java/fury-core/src/main/java/org/apache/fury/Fury.java index 316487342f..4feca16b14 100644 --- a/java/fury-core/src/main/java/org/apache/fury/Fury.java +++ b/java/fury-core/src/main/java/org/apache/fury/Fury.java @@ -56,6 +56,7 @@ import org.apache.fury.serializer.SerializerFactory; import org.apache.fury.serializer.StringSerializer; import org.apache.fury.type.Generics; +import org.apache.fury.type.Types; import org.apache.fury.util.ExceptionUtils; import org.apache.fury.util.Preconditions; import org.apache.fury.util.StringUtils; @@ -185,17 +186,29 @@ public void register(Class cls, String namespace, String typeName) { @Override public void registerSerializer(Class type, Class serializerClass) { - classResolver.registerSerializer(type, serializerClass); + if (language == Language.JAVA) { + classResolver.registerSerializer(type, serializerClass); + } else { + xtypeResolver.registerSerializer(type, serializerClass); + } } @Override public void registerSerializer(Class type, Serializer serializer) { - classResolver.registerSerializer(type, serializer); + if (language == Language.JAVA) { + classResolver.registerSerializer(type, serializer); + } else { + xtypeResolver.registerSerializer(type, serializer); + } } @Override public void registerSerializer(Class type, Function> serializerCreator) { - classResolver.registerSerializer(type, serializerCreator.apply(this)); + if (language == Language.JAVA) { + classResolver.registerSerializer(type, serializerCreator.apply(this)); + } else { + xtypeResolver.registerSerializer(type, serializerCreator.apply(this)); + } } @Override @@ -447,11 +460,47 @@ public void writeNonRef(MemoryBuffer buffer, Object obj, ClassInfo classInfo) { public void xwriteRef(MemoryBuffer buffer, Object obj) { if (!refResolver.writeRefOrNull(buffer, obj)) { - ClassInfo classInfo = classResolver.getOrUpdateClassInfo(obj.getClass()); - buffer.writeVarUint32Small7(classInfo.getXtypeId()); - depth++; - classInfo.getSerializer().xwrite(buffer, obj); - depth--; + ClassInfo classInfo = xtypeResolver.writeClassInfo(buffer, obj); + switch (classInfo.getXtypeId()) { + case Types.BOOL: + buffer.writeBoolean((Boolean) obj); + break; + case Types.INT8: + buffer.writeByte((Byte) obj); + break; + case Types.INT16: + buffer.writeInt16((Short) obj); + break; + case Types.INT32: + buffer.writeInt32((Integer) obj); + break; + case Types.VAR_INT32: + buffer.writeVarInt32((Integer) obj); + break; + case Types.INT64: + buffer.writeInt64((Long) obj); + break; + case Types.VAR_INT64: + buffer.writeVarInt64((Long) obj); + break; + case Types.SLI_INT64: + buffer.writeSliInt64((Long) obj); + break; + case Types.FLOAT32: + buffer.writeFloat32((Float) obj); + break; + case Types.FLOAT64: + buffer.writeFloat64((Double) obj); + break; + case Types.STRING: + stringSerializer.writeString(buffer, (String) obj); + break; + // TODO(add fastpath for other types) + default: + depth++; + classInfo.getSerializer().xwrite(buffer, obj); + depth--; + } } } @@ -907,11 +956,37 @@ public Object xreadNonRef(MemoryBuffer buffer, Serializer serializer) { } public Object xreadNonRef(MemoryBuffer buffer, ClassInfo classInfo) { - depth++; assert classInfo != null; - Object o = classInfo.getSerializer().xread(buffer); - depth--; - return o; + switch (classInfo.getXtypeId()) { + case Types.BOOL: + return buffer.readBoolean(); + case Types.INT8: + return buffer.readByte(); + case Types.INT16: + return buffer.readInt16(); + case Types.INT32: + return buffer.readInt32(); + case Types.VAR_INT32: + return buffer.readVarUint32(); + case Types.INT64: + return buffer.readInt64(); + case Types.VAR_INT64: + return buffer.readVarInt64(); + case Types.SLI_INT64: + return buffer.readSliInt64(); + case Types.FLOAT32: + return buffer.readFloat32(); + case Types.FLOAT64: + return buffer.readFloat64(); + case Types.STRING: + return stringSerializer.readString(buffer); + // TODO(add fastpath for other types) + default: + depth++; + Object o = classInfo.getSerializer().xread(buffer); + depth--; + return o; + } } @Override diff --git a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassInfo.java b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassInfo.java index 868847e1d3..e6ee4e313e 100644 --- a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassInfo.java +++ b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassInfo.java @@ -41,7 +41,7 @@ public class ClassInfo { final MetaStringBytes packageNameBytes; final MetaStringBytes classNameBytes; final boolean isDynamicGeneratedClass; - short xtypeId; + int xtypeId; Serializer serializer; // use primitive to avoid boxing // class id must be less than Integer.MAX_VALUE/2 since we use bit 0 as class id flag. @@ -127,7 +127,7 @@ public short getClassId() { return classId; } - public short getXtypeId() { + public int getXtypeId() { return xtypeId; } diff --git a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java index 2969e4ba36..b16450734b 100644 --- a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java +++ b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java @@ -1657,7 +1657,7 @@ private ClassInfo readClassInfoFromBytes( return classInfo; } - private ClassInfo loadBytesToClassInfo( + ClassInfo loadBytesToClassInfo( MetaStringBytes packageBytes, MetaStringBytes simpleClassNameBytes) { ClassNameBytes classNameBytes = new ClassNameBytes(packageBytes.hashCode, simpleClassNameBytes.hashCode); diff --git a/java/fury-core/src/main/java/org/apache/fury/resolver/XtypeResolver.java b/java/fury-core/src/main/java/org/apache/fury/resolver/XtypeResolver.java index 119e8a491d..397a995fb0 100644 --- a/java/fury-core/src/main/java/org/apache/fury/resolver/XtypeResolver.java +++ b/java/fury-core/src/main/java/org/apache/fury/resolver/XtypeResolver.java @@ -18,14 +18,17 @@ import org.apache.fury.memory.MemoryBuffer; import org.apache.fury.memory.Platform; import org.apache.fury.serializer.EnumSerializer; +import org.apache.fury.serializer.Serializer; import org.apache.fury.serializer.StructSerializer; import org.apache.fury.type.Types; import org.apache.fury.util.Preconditions; +// TODO(chaokunyang) Abstract type resolver for java/xlang type resolution. public class XtypeResolver { - private static final float loadFactor = 0.25f; + private static final float loadFactor = 0.5f; private final ClassResolver classResolver; + private final MetaStringResolver metaStringResolver; private int xtypeIdGenerator = 64; private final LongMap xtypeIdToClassMap = new LongMap<>(8, loadFactor); @@ -33,14 +36,10 @@ public class XtypeResolver { public XtypeResolver(ClassResolver classResolver) { this.classResolver = classResolver; + this.metaStringResolver = classResolver.getMetaStringResolver(); registerDefaultTypes(); } - public ClassInfo readClassInfo(MemoryBuffer buffer) { - int typeId = buffer.readVarUint32Small7(); - return xtypeIdToClassMap.get(typeId); - } - public void register(Class type) { while (registeredTypeIds.contains(xtypeIdGenerator)) { xtypeIdGenerator++; @@ -48,20 +47,40 @@ public void register(Class type) { register(type, xtypeIdGenerator++); } - public void register(Class type, int xtypeId) { + public void register(Class type, int typeId) { // We can relax this limit in the future, but currently we keep it in small range for future // extension. - Preconditions.checkArgument(xtypeId < Short.MAX_VALUE, "Too big type id %s", xtypeId); - ClassInfo classInfo = xtypeIdToClassMap.get(xtypeId); + Preconditions.checkArgument(typeId < Short.MAX_VALUE, "Too big type id %s", typeId); + ClassInfo classInfo = classResolver.getClassInfo(type, false); + Serializer serializer = null; if (classInfo != null) { - throw new IllegalArgumentException( - String.format("Type %s has been registered with id %s", type, classInfo.xtypeId)); + serializer = classInfo.serializer; + if (classInfo.xtypeId != 0) { + throw new IllegalArgumentException( + String.format("Type %s has been registered with id %s", type, classInfo.xtypeId)); + } } + int xtypeId = typeId; if (type.isEnum()) { - classResolver.registerSerializer( - type, new EnumSerializer<>(classResolver.getFury(), type.asSubclass(Enum.class))); + xtypeId = xtypeId << 8 + Types.ENUM; + } else { - classResolver.registerSerializer(type, new StructSerializer<>(classResolver.getFury(), type)); + if (serializer != null) { + if (serializer instanceof StructSerializer) { + xtypeId = xtypeId << 8 + Types.STRUCT; + } else { + xtypeId = xtypeId << 8 + Types.EXT; + } + } + } + if (serializer == null) { + if (type.isEnum()) { + classResolver.registerSerializer( + type, new EnumSerializer<>(classResolver.getFury(), type.asSubclass(Enum.class))); + } else { + classResolver.registerSerializer( + type, new StructSerializer<>(classResolver.getFury(), type)); + } } registeredTypeIds.add(xtypeId); classInfo = classResolver.getClassInfo(type); @@ -70,9 +89,10 @@ public void register(Class type, int xtypeId) { } public void register(Class cls, String namespace, String typeName) { - Preconditions.checkArgument(!typeName.contains("."), - "Typename %s should not contains ., please put it into namespace", typeName); - + Preconditions.checkArgument( + !typeName.contains("."), + "Typename %s should not contains `.`, please put it into namespace", + typeName); } private void registerDefaultTypes(int xtypeId, Class defaultType, Class... otherTypes) { @@ -87,7 +107,7 @@ private void internalRegister(Class type, int xtypeId) { ClassInfo classInfo = classResolver.getClassInfo(type); Preconditions.checkArgument( classInfo.xtypeId == 0, "Type %s has be registered with id %s", type, classInfo.xtypeId); - classInfo.xtypeId = (short) xtypeId; + classInfo.xtypeId = xtypeId; } private void registerDefaultTypes() { @@ -118,4 +138,58 @@ private void registerDefaultTypes() { registerDefaultTypes(Types.SET, HashSet.class); registerDefaultTypes(Types.MAP, HashMap.class); } + + public void registerSerializer(Class type, Class serializerClass) { + classResolver.registerSerializer(type, serializerClass); + + if (classResolver.getClassInfo(type).xtypeId == 0) { + register(type); + } + } + + public void registerSerializer(Class type, Serializer serializer) { + classResolver.registerSerializer(type, serializer); + } + + public ClassInfo writeClassInfo(MemoryBuffer buffer, Object obj) { + ClassInfo classInfo = classResolver.getOrUpdateClassInfo(obj.getClass()); + int xtypeId = classInfo.getXtypeId(); + buffer.writeVarUint32Small7((byte) xtypeId); + switch ((byte) xtypeId) { + case Types.ENUM: + case Types.STRUCT: + case Types.EXT: + buffer.writeVarUint32Small7(xtypeId >>> 8); + break; + case Types.NS_ENUM: + case Types.NS_STRUCT: + case Types.NS_EXT: + // if it's null, it's a bug. + assert classInfo.packageNameBytes != null; + metaStringResolver.writeMetaStringBytes(buffer, classInfo.packageNameBytes); + assert classInfo.classNameBytes != null; + metaStringResolver.writeMetaStringBytes(buffer, classInfo.classNameBytes); + } + return classInfo; + } + + public ClassInfo readClassInfo(MemoryBuffer buffer) { + int xtypeId = buffer.readVarUint32Small14(); + switch (xtypeId) { + case Types.ENUM: + return xtypeIdToClassMap.get(xtypeId << 8 + Types.ENUM); + case Types.STRUCT: + return xtypeIdToClassMap.get(xtypeId << 8 + Types.STRUCT); + case Types.EXT: + return xtypeIdToClassMap.get(xtypeId << 8 + Types.EXT); + case Types.NS_ENUM: + case Types.NS_STRUCT: + case Types.NS_EXT: + MetaStringBytes packageBytes = metaStringResolver.readMetaStringBytes(buffer); + MetaStringBytes simpleClassNameBytes = metaStringResolver.readMetaStringBytes(buffer); + return classResolver.loadBytesToClassInfo(packageBytes, simpleClassNameBytes); + default: + return xtypeIdToClassMap.get(xtypeId); + } + } } diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/StructSerializer.java b/java/fury-core/src/main/java/org/apache/fury/serializer/StructSerializer.java index 9a1d4a9eff..04e2a43afa 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/StructSerializer.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/StructSerializer.java @@ -219,10 +219,7 @@ int computeFieldHash(int hash, GenericType fieldGeneric) { } else { try { ClassInfo classInfo = fury.getClassResolver().getClassInfo(fieldGeneric.getCls()); - short xtypeId = classInfo.getXtypeId(); - if (xtypeId == Fury.NOT_SUPPORT_XLANG) { - return hash; - } + int xtypeId = classInfo.getXtypeId(); id = Math.abs(xtypeId); } catch (Exception e) { return hash; From ddb57666713a3fe51fbe720a45b5270ec625d5ff Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 3 Nov 2024 22:52:29 +0800 Subject: [PATCH 10/18] fix fury --- java/fury-core/src/main/java/org/apache/fury/Fury.java | 8 +++++++- .../main/java/org/apache/fury/resolver/XtypeResolver.java | 2 +- .../src/main/java/org/apache/fury/type/ScalaTypes.java | 1 + 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/java/fury-core/src/main/java/org/apache/fury/Fury.java b/java/fury-core/src/main/java/org/apache/fury/Fury.java index 6dd1535816..7757bf3a2b 100644 --- a/java/fury-core/src/main/java/org/apache/fury/Fury.java +++ b/java/fury-core/src/main/java/org/apache/fury/Fury.java @@ -197,7 +197,13 @@ public void register(Class cls, int id, boolean createSerializer) { /** register class with given type tag which will be used for cross-language serialization. */ public void register(Class cls, String typeName) { Preconditions.checkArgument(language != Language.JAVA); - register(cls, "", typeName); + int idx = typeName.lastIndexOf('.'); + String namespace = ""; + if (idx > 0) { + namespace = typeName.substring(0, idx); + typeName = typeName.substring(idx); + } + register(cls, namespace, typeName); } public void register(Class cls, String namespace, String typeName) { diff --git a/java/fury-core/src/main/java/org/apache/fury/resolver/XtypeResolver.java b/java/fury-core/src/main/java/org/apache/fury/resolver/XtypeResolver.java index 397a995fb0..986e9cf876 100644 --- a/java/fury-core/src/main/java/org/apache/fury/resolver/XtypeResolver.java +++ b/java/fury-core/src/main/java/org/apache/fury/resolver/XtypeResolver.java @@ -76,7 +76,7 @@ public void register(Class type, int typeId) { if (serializer == null) { if (type.isEnum()) { classResolver.registerSerializer( - type, new EnumSerializer<>(classResolver.getFury(), type.asSubclass(Enum.class))); + type, new EnumSerializer(classResolver.getFury(), (Class) type)); } else { classResolver.registerSerializer( type, new StructSerializer<>(classResolver.getFury(), type)); diff --git a/java/fury-core/src/main/java/org/apache/fury/type/ScalaTypes.java b/java/fury-core/src/main/java/org/apache/fury/type/ScalaTypes.java index 8aa1b09637..6d044bb3d6 100644 --- a/java/fury-core/src/main/java/org/apache/fury/type/ScalaTypes.java +++ b/java/fury-core/src/main/java/org/apache/fury/type/ScalaTypes.java @@ -28,6 +28,7 @@ /** Scala types utils using reflection without dependency on scala library. */ @SuppressWarnings({"unchecked", "rawtypes"}) public class ScalaTypes { + private static final Class SCALA_MAP_TYPE; private static final Class SCALA_SEQ_TYPE; private static final Class SCALA_SET_TYPE; From 3871b19188560879736b3be7a454f76cfb5ac874 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 23 Nov 2024 21:22:21 +0800 Subject: [PATCH 11/18] rewrite xlang type dispatch --- .../org/apache/fury/resolver/ClassInfo.java | 8 - .../apache/fury/resolver/ClassResolver.java | 4 + .../apache/fury/resolver/XtypeResolver.java | 170 +++++++++++++----- .../apache/fury/serializer/Serializers.java | 2 +- .../main/java/org/apache/fury/type/Types.java | 2 + 5 files changed, 132 insertions(+), 54 deletions(-) diff --git a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassInfo.java b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassInfo.java index 8bc171a6c1..35e2393119 100644 --- a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassInfo.java +++ b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassInfo.java @@ -132,14 +132,6 @@ public int getXtypeId() { return xtypeId; } - public MetaStringBytes getPackageNameBytes() { - return packageNameBytes; - } - - public MetaStringBytes getClassNameBytes() { - return classNameBytes; - } - @SuppressWarnings("unchecked") public Serializer getSerializer() { return (Serializer) serializer; diff --git a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java index 97787767e8..1807b28e36 100644 --- a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java +++ b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java @@ -1159,6 +1159,10 @@ public ClassInfo getClassInfo(Class cls, boolean createClassInfoIfNotFound) { } } + void setClassInfo(Class cls, ClassInfo classInfo) { + classInfoMap.put(cls, classInfo); + } + @Internal public ClassInfo getOrUpdateClassInfo(Class cls) { ClassInfo classInfo = classInfoCache; diff --git a/java/fury-core/src/main/java/org/apache/fury/resolver/XtypeResolver.java b/java/fury-core/src/main/java/org/apache/fury/resolver/XtypeResolver.java index 986e9cf876..53e8403ea0 100644 --- a/java/fury-core/src/main/java/org/apache/fury/resolver/XtypeResolver.java +++ b/java/fury-core/src/main/java/org/apache/fury/resolver/XtypeResolver.java @@ -1,5 +1,10 @@ package org.apache.fury.resolver; +import static org.apache.fury.meta.Encoders.GENERIC_ENCODER; +import static org.apache.fury.meta.Encoders.PACKAGE_DECODER; +import static org.apache.fury.meta.Encoders.TYPE_NAME_DECODER; +import static org.apache.fury.resolver.ClassResolver.NO_CLASS_ID; + import java.math.BigDecimal; import java.math.BigInteger; import java.sql.Timestamp; @@ -10,6 +15,8 @@ import java.util.Date; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -17,6 +24,8 @@ import org.apache.fury.collection.LongMap; import org.apache.fury.memory.MemoryBuffer; import org.apache.fury.memory.Platform; +import org.apache.fury.meta.Encoders; +import org.apache.fury.meta.MetaString; import org.apache.fury.serializer.EnumSerializer; import org.apache.fury.serializer.Serializer; import org.apache.fury.serializer.StructSerializer; @@ -26,11 +35,15 @@ // TODO(chaokunyang) Abstract type resolver for java/xlang type resolution. public class XtypeResolver { private static final float loadFactor = 0.5f; + // Most systems won't have so many types for serialization. + private static final int MAX_TYPE_ID = 4096; private final ClassResolver classResolver; private final MetaStringResolver metaStringResolver; private int xtypeIdGenerator = 64; + // Use ClassInfo[] or LongMap? + // ClassInfo[] is faster, but we can't have bigger type id. private final LongMap xtypeIdToClassMap = new LongMap<>(8, loadFactor); private final Set registeredTypeIds = new HashSet<>(); @@ -48,9 +61,10 @@ public void register(Class type) { } public void register(Class type, int typeId) { - // We can relax this limit in the future, but currently we keep it in small range for future - // extension. - Preconditions.checkArgument(typeId < Short.MAX_VALUE, "Too big type id %s", typeId); + // ClassInfo[] has length of max type id. If the type id is too big, Fury will waste many + // memory. + // We can relax this limit in the future. + Preconditions.checkArgument(typeId < MAX_TYPE_ID, "Too big type id %s", typeId); ClassInfo classInfo = classResolver.getClassInfo(type, false); Serializer serializer = null; if (classInfo != null) { @@ -59,6 +73,14 @@ public void register(Class type, int typeId) { throw new IllegalArgumentException( String.format("Type %s has been registered with id %s", type, classInfo.xtypeId)); } + String prevNamespace = decodeNamespace(classInfo.packageNameBytes); + String prevTypeName = decodeTypeName(classInfo.classNameBytes); + if (!type.getSimpleName().equals(prevTypeName)) { + throw new IllegalArgumentException( + String.format( + "Type %s has been registered with namespace %s type %s", + type, prevNamespace, prevTypeName)); + } } int xtypeId = typeId; if (type.isEnum()) { @@ -73,6 +95,71 @@ public void register(Class type, int typeId) { } } } + register(type, serializer, xtypeId); + } + + public void register(Class type, String namespace, String typeName) { + Preconditions.checkArgument( + !typeName.contains("."), + "Typename %s should not contains `.`, please put it into namespace", + typeName); + ClassInfo classInfo = classResolver.getClassInfo(type, false); + Serializer serializer = null; + if (classInfo != null) { + serializer = classInfo.serializer; + if (classInfo.classNameBytes != null) { + String prevNamespace = decodeNamespace(classInfo.packageNameBytes); + String prevTypeName = decodeTypeName(classInfo.classNameBytes); + if (!namespace.equals(prevNamespace) || typeName.equals(prevTypeName)) { + throw new IllegalArgumentException( + String.format( + "Type %s has been registered with namespace %s type %s", + type, prevNamespace, prevTypeName)); + } + } + } + short xtypeId = -1; + if (type.isEnum()) { + xtypeId = Types.NS_ENUM; + } else { + if (serializer != null) { + if (serializer instanceof StructSerializer) { + xtypeId = Types.NS_STRUCT; + } else { + xtypeId = Types.NS_EXT; + } + } + } + MetaStringBytes fullClassNameBytes = + metaStringResolver.getOrCreateMetaStringBytes( + GENERIC_ENCODER.encode(type.getName(), MetaString.Encoding.UTF_8)); + MetaStringBytes nsBytes = + metaStringResolver.getOrCreateMetaStringBytes(Encoders.encodePackage(namespace)); + MetaStringBytes classNameBytes = + metaStringResolver.getOrCreateMetaStringBytes(Encoders.encodeTypeName(typeName)); + ClassInfo info = + new ClassInfo( + type, + fullClassNameBytes, + nsBytes, + classNameBytes, + false, + serializer, + NO_CLASS_ID, + xtypeId); + classResolver.setClassInfo(type, info); + register(type, serializer, xtypeId); + } + + private String decodeNamespace(MetaStringBytes packageNameBytes) { + return packageNameBytes.decode(PACKAGE_DECODER); + } + + private String decodeTypeName(MetaStringBytes classNameBytes) { + return classNameBytes.decode(TYPE_NAME_DECODER); + } + + private void register(Class type, Serializer serializer, int xtypeId) { if (serializer == null) { if (type.isEnum()) { classResolver.registerSerializer( @@ -83,31 +170,29 @@ public void register(Class type, int typeId) { } } registeredTypeIds.add(xtypeId); - classInfo = classResolver.getClassInfo(type); + ClassInfo classInfo = classResolver.getClassInfo(type); xtypeIdToClassMap.put(xtypeId, classInfo); - internalRegister(type, xtypeId); + classInfo.xtypeId = xtypeId; } - public void register(Class cls, String namespace, String typeName) { - Preconditions.checkArgument( - !typeName.contains("."), - "Typename %s should not contains `.`, please put it into namespace", - typeName); + public void registerSerializer(Class type, Class serializerClass) { + checkClassRegistration(type); + classResolver.registerSerializer(type, serializerClass); } - private void registerDefaultTypes(int xtypeId, Class defaultType, Class... otherTypes) { - internalRegister(defaultType, xtypeId); - xtypeIdToClassMap.put(xtypeId, classResolver.getClassInfo(defaultType)); - for (Class otherType : otherTypes) { - internalRegister(otherType, xtypeId); - } + public void registerSerializer(Class type, Serializer serializer) { + checkClassRegistration(type); + classResolver.registerSerializer(type, serializer); } - private void internalRegister(Class type, int xtypeId) { - ClassInfo classInfo = classResolver.getClassInfo(type); + private void checkClassRegistration(Class type) { + ClassInfo classInfo = classResolver.getClassInfo(type, false); Preconditions.checkArgument( - classInfo.xtypeId == 0, "Type %s has be registered with id %s", type, classInfo.xtypeId); - classInfo.xtypeId = xtypeId; + classInfo != null + && (classInfo.xtypeId != 0 + || !type.getSimpleName().equals(decodeTypeName(classInfo.classNameBytes))), + "Type %s should be registered with id or namespace+typename before register serializer", + type); } private void registerDefaultTypes() { @@ -135,55 +220,50 @@ private void registerDefaultTypes() { registerDefaultTypes(Types.FLOAT32_ARRAY, float[].class); registerDefaultTypes(Types.FLOAT64_ARRAY, double[].class); registerDefaultTypes(Types.LIST, ArrayList.class, Object[].class); - registerDefaultTypes(Types.SET, HashSet.class); - registerDefaultTypes(Types.MAP, HashMap.class); + registerDefaultTypes(Types.SET, HashSet.class, LinkedHashSet.class); + registerDefaultTypes(Types.MAP, HashMap.class, LinkedHashMap.class); } - public void registerSerializer(Class type, Class serializerClass) { - classResolver.registerSerializer(type, serializerClass); - - if (classResolver.getClassInfo(type).xtypeId == 0) { - register(type); + private void registerDefaultTypes(int xtypeId, Class defaultType, Class... otherTypes) { + internalRegister(defaultType, xtypeId); + xtypeIdToClassMap.put(xtypeId, classResolver.getClassInfo(defaultType)); + for (Class otherType : otherTypes) { + internalRegister(otherType, xtypeId); } } - public void registerSerializer(Class type, Serializer serializer) { - classResolver.registerSerializer(type, serializer); + private void internalRegister(Class type, int xtypeId) { + ClassInfo classInfo = classResolver.getClassInfo(type); + Preconditions.checkArgument( + classInfo.xtypeId == 0, "Type %s has be registered with id %s", type, classInfo.xtypeId); + classInfo.xtypeId = xtypeId; } public ClassInfo writeClassInfo(MemoryBuffer buffer, Object obj) { ClassInfo classInfo = classResolver.getOrUpdateClassInfo(obj.getClass()); int xtypeId = classInfo.getXtypeId(); - buffer.writeVarUint32Small7((byte) xtypeId); - switch ((byte) xtypeId) { - case Types.ENUM: - case Types.STRUCT: - case Types.EXT: - buffer.writeVarUint32Small7(xtypeId >>> 8); - break; + byte internalTypeId = (byte) xtypeId; + buffer.writeVarUint32Small7(xtypeId); + switch (internalTypeId) { case Types.NS_ENUM: case Types.NS_STRUCT: case Types.NS_EXT: - // if it's null, it's a bug. assert classInfo.packageNameBytes != null; metaStringResolver.writeMetaStringBytes(buffer, classInfo.packageNameBytes); assert classInfo.classNameBytes != null; metaStringResolver.writeMetaStringBytes(buffer, classInfo.classNameBytes); + break; } return classInfo; } public ClassInfo readClassInfo(MemoryBuffer buffer) { - int xtypeId = buffer.readVarUint32Small14(); - switch (xtypeId) { - case Types.ENUM: - return xtypeIdToClassMap.get(xtypeId << 8 + Types.ENUM); - case Types.STRUCT: - return xtypeIdToClassMap.get(xtypeId << 8 + Types.STRUCT); - case Types.EXT: - return xtypeIdToClassMap.get(xtypeId << 8 + Types.EXT); + long xtypeId = buffer.readVarUint32Small14(); + byte internalTypeId = (byte) xtypeId; + switch (internalTypeId) { case Types.NS_ENUM: case Types.NS_STRUCT: + case Types.NS_COMPATIBLE_STRUCT: case Types.NS_EXT: MetaStringBytes packageBytes = metaStringResolver.readMetaStringBytes(buffer); MetaStringBytes simpleClassNameBytes = metaStringResolver.readMetaStringBytes(buffer); diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/Serializers.java b/java/fury-core/src/main/java/org/apache/fury/serializer/Serializers.java index 47152b565b..e5b01225c1 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/Serializers.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/Serializers.java @@ -183,7 +183,7 @@ public CrossLanguageCompatibleSerializer(Fury fury, Class cls) { } public CrossLanguageCompatibleSerializer( - Fury fury, Class cls, boolean needToWriteRef, boolean immutable) { + Fury fury, Class cls, boolean needToWriteRef, boolean immutable) { super(fury, cls, needToWriteRef, immutable); } diff --git a/java/fury-core/src/main/java/org/apache/fury/type/Types.java b/java/fury-core/src/main/java/org/apache/fury/type/Types.java index ceec6f1572..6b30cb487b 100644 --- a/java/fury-core/src/main/java/org/apache/fury/type/Types.java +++ b/java/fury-core/src/main/java/org/apache/fury/type/Types.java @@ -17,7 +17,9 @@ public class Types { public static final int ENUM = 13; public static final int NS_ENUM = 14; public static final int STRUCT = 15; + public static final int COMPATIBLE_STRUCT = 15; public static final int NS_STRUCT = 16; + public static final int NS_COMPATIBLE_STRUCT = 46; public static final int EXT = 17; public static final int NS_EXT = 18; public static final int LIST = 19; From efdcde44592bb39a65df154ab268b0566daa1833 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 23 Nov 2024 22:23:35 +0800 Subject: [PATCH 12/18] fix compile error --- .../apache/fury/AbstractThreadSafeFury.java | 6 +++--- .../src/main/java/org/apache/fury/Fury.java | 2 +- .../apache/fury/resolver/ClassResolver.java | 12 +----------- .../serializer/FuryCopyableSerializer.java | 19 ------------------- .../apache/fury/serializer/Serializer.java | 14 +------------- .../collection/SubListSerializers.java | 6 ++++-- 6 files changed, 10 insertions(+), 49 deletions(-) diff --git a/java/fury-core/src/main/java/org/apache/fury/AbstractThreadSafeFury.java b/java/fury-core/src/main/java/org/apache/fury/AbstractThreadSafeFury.java index 493fad2193..3e2aff67ff 100644 --- a/java/fury-core/src/main/java/org/apache/fury/AbstractThreadSafeFury.java +++ b/java/fury-core/src/main/java/org/apache/fury/AbstractThreadSafeFury.java @@ -43,18 +43,18 @@ public void register(Class cls, int id) { } @Override - public void register(Class cls, Short id, boolean createSerializer) { + public void register(Class cls, int id, boolean createSerializer) { registerCallback(fury -> fury.register(cls, id, createSerializer)); } @Override public void register(Class cls, String typeName) { - processCallback(fury -> fury.register(cls, typeName)); + registerCallback(fury -> fury.register(cls, typeName)); } @Override public void register(Class cls, String namespace, String typeName) { - processCallback(fury -> fury.register(cls, namespace, typeName)); + registerCallback(fury -> fury.register(cls, namespace, typeName)); } @Override diff --git a/java/fury-core/src/main/java/org/apache/fury/Fury.java b/java/fury-core/src/main/java/org/apache/fury/Fury.java index 7757bf3a2b..ba2e9bb6c7 100644 --- a/java/fury-core/src/main/java/org/apache/fury/Fury.java +++ b/java/fury-core/src/main/java/org/apache/fury/Fury.java @@ -201,7 +201,7 @@ public void register(Class cls, String typeName) { String namespace = ""; if (idx > 0) { namespace = typeName.substring(0, idx); - typeName = typeName.substring(idx); + typeName = typeName.substring(idx + 1); } register(cls, namespace, typeName); } diff --git a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java index 1807b28e36..c9ad12ae57 100644 --- a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java +++ b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java @@ -631,15 +631,6 @@ public void registerSerializer(Class type, Serializer serializer) { && fury.getLanguage() == Language.JAVA) { register(type); } - if (fury.getConfig().getLanguage() != Language.JAVA) { - if (isSet(type)) { - getClassInfo(type).xtypeId = Types.SET; - } else if (isCollection(type)) { - getClassInfo(type).xtypeId = Types.LIST; - } else if (isMap(type)) { - getClassInfo(type).xtypeId = Types.MAP; - } - } addSerializer(type, serializer); } @@ -734,7 +725,6 @@ public void clearSerializer(Class cls) { /** Ass serializer for specified class. */ private void addSerializer(Class type, Serializer serializer) { Preconditions.checkNotNull(serializer); - short typeId = serializer.getXtypeId(); // 1. Try to get ClassInfo from `registeredId2ClassInfo` and // `classInfoMap` or create a new `ClassInfo`. ClassInfo classInfo; @@ -753,7 +743,7 @@ private void addSerializer(Class type, Serializer serializer) { } if (classInfo == null || classId != classInfo.classId) { - classInfo = new ClassInfo(this, type, null, classId, typeId); + classInfo = new ClassInfo(this, type, null, classId, (short) 0); classInfoMap.put(type, classInfo); if (registered) { registeredId2ClassInfo[classId] = classInfo; diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/FuryCopyableSerializer.java b/java/fury-core/src/main/java/org/apache/fury/serializer/FuryCopyableSerializer.java index a17c35be91..a720b44be0 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/FuryCopyableSerializer.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/FuryCopyableSerializer.java @@ -48,25 +48,6 @@ public T read(MemoryBuffer buffer) { return serializer.read(buffer); } - /** - * Returns {@link Fury#NOT_SUPPORT_CROSS_LANGUAGE} if the serializer doesn't support - * cross-language serialization. Return a number in range (0, 32767) if the serializer support - * cross-language serialization and native serialization data is the same with cross-language - * serialization. Return a negative short in range [-32768, 0) if the serializer support - * cross-language serialization and native serialization data is not the same with cross-language - * serialization. - */ - @Override - public short getXtypeId() { - return serializer.getXtypeId(); - } - - /** Returns a type tag used for setup type mapping between languages. */ - @Override - public String getCrossLanguageTypeTag() { - return serializer.getCrossLanguageTypeTag(); - } - @Override public void xwrite(MemoryBuffer buffer, T value) { serializer.xwrite(buffer, value); diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/Serializer.java b/java/fury-core/src/main/java/org/apache/fury/serializer/Serializer.java index ab482ca006..720120acf1 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/Serializer.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/Serializer.java @@ -27,8 +27,7 @@ /** * Serialize/deserializer objects into binary. Note that this class is designed as an abstract class - * instead of interface to reduce virtual method call cost of {@link #needToWriteRef}/{@link - * #getXtypeId}. + * instead of interface to reduce virtual method call cost of {@link #needToWriteRef}. * * @param type of objects being serializing/deserializing */ @@ -63,17 +62,6 @@ public T read(MemoryBuffer buffer) { throw new UnsupportedOperationException(); } - /** - * Returns {@link Fury#NOT_SUPPORT_XLANG} if the serializer doesn't support cross-language - * serialization. Return a number in range (0, 32767) if the serializer support cross-language - * serialization and native serialization data is the same with cross-language serialization. - * Return a negative short in range [-32768, 0) if the serializer support cross-language - * serialization and native serialization data is not the same with cross-language serialization. - */ - public short getXtypeId() { - return Fury.NOT_SUPPORT_XLANG; - } - public void xwrite(MemoryBuffer buffer, T value) { throw new UnsupportedOperationException(); } diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/SubListSerializers.java b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/SubListSerializers.java index 60353e7866..0f3fa73da2 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/SubListSerializers.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/SubListSerializers.java @@ -28,6 +28,7 @@ import org.apache.fury.logging.LoggerFactory; import org.apache.fury.memory.MemoryBuffer; import org.apache.fury.reflect.ReflectionUtils; +import org.apache.fury.resolver.ClassResolver; import org.apache.fury.serializer.ObjectSerializer; @SuppressWarnings({"rawtypes", "unchecked"}) @@ -74,14 +75,15 @@ class ImmutableSubListStub implements Stub {} } public static void registerSerializers(Fury fury, boolean preserveView) { + ClassResolver classResolver = fury.getClassResolver(); for (Class cls : new Class[] { SubListClass, RandomAccessSubListClass, ArrayListSubListClass, ImmutableSubListClass }) { if (fury.trackingRef() && preserveView && fury.getConfig().getLanguage() == Language.JAVA) { - fury.registerSerializer(cls, new SubListViewSerializer(fury, cls)); + classResolver.registerSerializer(cls, new SubListViewSerializer(fury, cls)); } else { - fury.registerSerializer(cls, new SubListSerializer(fury, (Class) cls)); + classResolver.registerSerializer(cls, new SubListSerializer(fury, (Class) cls)); } } } From a4b4e263d0ddafed96a125e543a80060c80dbfdf Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 24 Nov 2024 01:17:32 +0800 Subject: [PATCH 13/18] java xlang serialization type dispatch --- .../src/main/java/org/apache/fury/Fury.java | 2 +- .../exception/ClassUnregisteredException.java | 12 + .../SerializerUnregisteredException.java | 8 + .../apache/fury/resolver/ClassNameBytes.java | 25 ++ .../apache/fury/resolver/ClassResolver.java | 27 +- .../apache/fury/resolver/XtypeResolver.java | 268 ++++++++++++++---- .../java/org/apache/fury/type/TypeUtils.java | 9 + 7 files changed, 263 insertions(+), 88 deletions(-) create mode 100644 java/fury-core/src/main/java/org/apache/fury/exception/ClassUnregisteredException.java create mode 100644 java/fury-core/src/main/java/org/apache/fury/exception/SerializerUnregisteredException.java create mode 100644 java/fury-core/src/main/java/org/apache/fury/resolver/ClassNameBytes.java diff --git a/java/fury-core/src/main/java/org/apache/fury/Fury.java b/java/fury-core/src/main/java/org/apache/fury/Fury.java index ba2e9bb6c7..b86432ffaa 100644 --- a/java/fury-core/src/main/java/org/apache/fury/Fury.java +++ b/java/fury-core/src/main/java/org/apache/fury/Fury.java @@ -146,7 +146,7 @@ public Fury(FuryBuilder builder, ClassLoader classLoader) { metaStringResolver = new MetaStringResolver(); classResolver = new ClassResolver(this); classResolver.initialize(); - xtypeResolver = new XtypeResolver(classResolver); + xtypeResolver = new XtypeResolver(this); serializationContext = new SerializationContext(config); this.classLoader = classLoader; generics = new Generics(this); diff --git a/java/fury-core/src/main/java/org/apache/fury/exception/ClassUnregisteredException.java b/java/fury-core/src/main/java/org/apache/fury/exception/ClassUnregisteredException.java new file mode 100644 index 0000000000..e438d92913 --- /dev/null +++ b/java/fury-core/src/main/java/org/apache/fury/exception/ClassUnregisteredException.java @@ -0,0 +1,12 @@ +package org.apache.fury.exception; + +public class ClassUnregisteredException extends FuryException { + + public ClassUnregisteredException(Class cls) { + this(cls.getName()); + } + + public ClassUnregisteredException(String qualifiedName) { + super(String.format("Class %s is not registered", qualifiedName)); + } +} diff --git a/java/fury-core/src/main/java/org/apache/fury/exception/SerializerUnregisteredException.java b/java/fury-core/src/main/java/org/apache/fury/exception/SerializerUnregisteredException.java new file mode 100644 index 0000000000..69a882ebdc --- /dev/null +++ b/java/fury-core/src/main/java/org/apache/fury/exception/SerializerUnregisteredException.java @@ -0,0 +1,8 @@ +package org.apache.fury.exception; + +public class SerializerUnregisteredException extends FuryException { + + public SerializerUnregisteredException(String qualifiedName) { + super(String.format("Class %s is not registered with a serializer", qualifiedName)); + } +} diff --git a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassNameBytes.java b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassNameBytes.java new file mode 100644 index 0000000000..f1840bb706 --- /dev/null +++ b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassNameBytes.java @@ -0,0 +1,25 @@ +package org.apache.fury.resolver; + +class ClassNameBytes { + private final long packageHash; + private final long classNameHash; + + ClassNameBytes(long packageHash, long classNameHash) { + this.packageHash = packageHash; + this.classNameHash = classNameHash; + } + + @Override + public boolean equals(Object o) { + // ClassNameBytes is used internally, skip + ClassNameBytes that = (ClassNameBytes) o; + return packageHash == that.packageHash && classNameHash == that.classNameHash; + } + + @Override + public int hashCode() { + int result = 31 + (int) (packageHash ^ (packageHash >>> 32)); + result = result * 31 + (int) (classNameHash ^ (classNameHash >>> 32)); + return result; + } +} diff --git a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java index c9ad12ae57..a268201d50 100644 --- a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java +++ b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java @@ -151,7 +151,6 @@ import org.apache.fury.type.GenericType; import org.apache.fury.type.ScalaTypes; import org.apache.fury.type.TypeUtils; -import org.apache.fury.type.Types; import org.apache.fury.util.GraalvmSupport; import org.apache.fury.util.Preconditions; import org.apache.fury.util.function.Functions; @@ -212,7 +211,7 @@ public class ClassResolver { private static final String SET_META__CONTEXT_MSG = "Meta context must be set before serialization, " + "please set meta context by SerializationContext.setMetaContext"; - private static final ClassInfo NIL_CLASS_INFO = + static final ClassInfo NIL_CLASS_INFO = new ClassInfo(null, null, null, null, false, null, NO_CLASS_ID, NOT_SUPPORT_XLANG); private final Fury fury; @@ -1840,30 +1839,6 @@ public void resetRead() {} public void resetWrite() {} - private static class ClassNameBytes { - private final long packageHash; - private final long classNameHash; - - private ClassNameBytes(long packageHash, long classNameHash) { - this.packageHash = packageHash; - this.classNameHash = classNameHash; - } - - @Override - public boolean equals(Object o) { - // ClassNameBytes is used internally, skip - ClassNameBytes that = (ClassNameBytes) o; - return packageHash == that.packageHash && classNameHash == that.classNameHash; - } - - @Override - public int hashCode() { - int result = 31 + (int) (packageHash ^ (packageHash >>> 32)); - result = result * 31 + (int) (classNameHash ^ (classNameHash >>> 32)); - return result; - } - } - public GenericType buildGenericType(TypeRef typeRef) { return GenericType.build( typeRef.getType(), diff --git a/java/fury-core/src/main/java/org/apache/fury/resolver/XtypeResolver.java b/java/fury-core/src/main/java/org/apache/fury/resolver/XtypeResolver.java index 53e8403ea0..70078dc4e1 100644 --- a/java/fury-core/src/main/java/org/apache/fury/resolver/XtypeResolver.java +++ b/java/fury-core/src/main/java/org/apache/fury/resolver/XtypeResolver.java @@ -1,9 +1,12 @@ package org.apache.fury.resolver; +import static org.apache.fury.Fury.NOT_SUPPORT_XLANG; import static org.apache.fury.meta.Encoders.GENERIC_ENCODER; import static org.apache.fury.meta.Encoders.PACKAGE_DECODER; +import static org.apache.fury.meta.Encoders.PACKAGE_ENCODER; import static org.apache.fury.meta.Encoders.TYPE_NAME_DECODER; import static org.apache.fury.resolver.ClassResolver.NO_CLASS_ID; +import static org.apache.fury.type.TypeUtils.qualifiedName; import java.math.BigDecimal; import java.math.BigInteger; @@ -21,25 +24,54 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import org.apache.fury.Fury; +import org.apache.fury.collection.IdentityMap; import org.apache.fury.collection.LongMap; +import org.apache.fury.collection.ObjectMap; +import org.apache.fury.config.Config; +import org.apache.fury.exception.ClassUnregisteredException; +import org.apache.fury.exception.SerializerUnregisteredException; +import org.apache.fury.logging.Logger; +import org.apache.fury.logging.LoggerFactory; import org.apache.fury.memory.MemoryBuffer; import org.apache.fury.memory.Platform; import org.apache.fury.meta.Encoders; import org.apache.fury.meta.MetaString; +import org.apache.fury.reflect.ReflectionUtils; +import org.apache.fury.serializer.ArraySerializers.ObjectArraySerializer; import org.apache.fury.serializer.EnumSerializer; +import org.apache.fury.serializer.NonexistentClass; +import org.apache.fury.serializer.NonexistentClassSerializers; import org.apache.fury.serializer.Serializer; +import org.apache.fury.serializer.Serializers; import org.apache.fury.serializer.StructSerializer; +import org.apache.fury.serializer.collection.CollectionSerializer; +import org.apache.fury.serializer.collection.MapSerializer; +import org.apache.fury.type.TypeUtils; import org.apache.fury.type.Types; import org.apache.fury.util.Preconditions; +@SuppressWarnings({"unchecked", "rawtypes"}) // TODO(chaokunyang) Abstract type resolver for java/xlang type resolution. public class XtypeResolver { + private static final Logger LOG = LoggerFactory.getLogger(XtypeResolver.class); + private static final float loadFactor = 0.5f; // Most systems won't have so many types for serialization. private static final int MAX_TYPE_ID = 4096; + private final Config config; + private final Fury fury; private final ClassResolver classResolver; + private final ClassInfoHolder classInfoCache = new ClassInfoHolder(ClassResolver.NIL_CLASS_INFO); private final MetaStringResolver metaStringResolver; + // IdentityMap has better lookup performance, when loadFactor is 0.05f, performance is better + private final IdentityMap, ClassInfo> classInfoMap = new IdentityMap<>(64, loadFactor); + // Every deserialization for unregistered class will query it, performance is important. + private final ObjectMap compositeClassNameBytes2ClassInfo = + new ObjectMap<>(16, loadFactor); + private final ObjectMap qualifiedType2ClassInfo = + new ObjectMap<>(16, loadFactor); private int xtypeIdGenerator = 64; // Use ClassInfo[] or LongMap? @@ -47,9 +79,11 @@ public class XtypeResolver { private final LongMap xtypeIdToClassMap = new LongMap<>(8, loadFactor); private final Set registeredTypeIds = new HashSet<>(); - public XtypeResolver(ClassResolver classResolver) { - this.classResolver = classResolver; - this.metaStringResolver = classResolver.getMetaStringResolver(); + public XtypeResolver(Fury fury) { + this.config = fury.getConfig(); + this.fury = fury; + this.classResolver = fury.getClassResolver(); + this.metaStringResolver = fury.getMetaStringResolver(); registerDefaultTypes(); } @@ -65,7 +99,7 @@ public void register(Class type, int typeId) { // memory. // We can relax this limit in the future. Preconditions.checkArgument(typeId < MAX_TYPE_ID, "Too big type id %s", typeId); - ClassInfo classInfo = classResolver.getClassInfo(type, false); + ClassInfo classInfo = classInfoMap.get(type); Serializer serializer = null; if (classInfo != null) { serializer = classInfo.serializer; @@ -95,7 +129,12 @@ public void register(Class type, int typeId) { } } } - register(type, serializer, xtypeId); + register( + type, + serializer, + ReflectionUtils.getPackage(type), + ReflectionUtils.getClassNameWithoutPackage(type), + xtypeId); } public void register(Class type, String namespace, String typeName) { @@ -103,7 +142,7 @@ public void register(Class type, String namespace, String typeName) { !typeName.contains("."), "Typename %s should not contains `.`, please put it into namespace", typeName); - ClassInfo classInfo = classResolver.getClassInfo(type, false); + ClassInfo classInfo = classInfoMap.get(type); Serializer serializer = null; if (classInfo != null) { serializer = classInfo.serializer; @@ -118,18 +157,52 @@ public void register(Class type, String namespace, String typeName) { } } } - short xtypeId = -1; - if (type.isEnum()) { - xtypeId = Types.NS_ENUM; + short xtypeId; + if (serializer != null) { + if (serializer instanceof StructSerializer) { + xtypeId = Types.NS_STRUCT; + } else if (serializer instanceof EnumSerializer) { + xtypeId = Types.NS_ENUM; + } else { + xtypeId = Types.NS_EXT; + } } else { - if (serializer != null) { - if (serializer instanceof StructSerializer) { - xtypeId = Types.NS_STRUCT; - } else { - xtypeId = Types.NS_EXT; - } + if (type.isEnum()) { + xtypeId = Types.NS_ENUM; + } else { + xtypeId = Types.NS_STRUCT; + } + } + register(type, serializer, namespace, typeName, xtypeId); + } + + private void register( + Class type, Serializer serializer, String namespace, String typeName, int xtypeId) { + ClassInfo classInfo = newClassInfo(type, serializer, namespace, typeName, (short) xtypeId); + qualifiedType2ClassInfo.put(qualifiedName(namespace, typeName), classInfo); + if (serializer == null) { + if (type.isEnum()) { + classInfo.serializer = new EnumSerializer(fury, (Class) type); + } else { + classInfo.serializer = new StructSerializer(fury, type); } } + classInfoMap.put(type, classInfo); + registeredTypeIds.add(xtypeId); + xtypeIdToClassMap.put(xtypeId, classInfo); + } + + private ClassInfo newClassInfo(Class type, Serializer serializer, short xtypeId) { + return newClassInfo( + type, + serializer, + ReflectionUtils.getPackage(type), + ReflectionUtils.getClassNameWithoutPackage(type), + xtypeId); + } + + private ClassInfo newClassInfo( + Class type, Serializer serializer, String namespace, String typeName, short xtypeId) { MetaStringBytes fullClassNameBytes = metaStringResolver.getOrCreateMetaStringBytes( GENERIC_ENCODER.encode(type.getName(), MetaString.Encoding.UTF_8)); @@ -137,18 +210,8 @@ public void register(Class type, String namespace, String typeName) { metaStringResolver.getOrCreateMetaStringBytes(Encoders.encodePackage(namespace)); MetaStringBytes classNameBytes = metaStringResolver.getOrCreateMetaStringBytes(Encoders.encodeTypeName(typeName)); - ClassInfo info = - new ClassInfo( - type, - fullClassNameBytes, - nsBytes, - classNameBytes, - false, - serializer, - NO_CLASS_ID, - xtypeId); - classResolver.setClassInfo(type, info); - register(type, serializer, xtypeId); + return new ClassInfo( + type, fullClassNameBytes, nsBytes, classNameBytes, false, serializer, NO_CLASS_ID, xtypeId); } private String decodeNamespace(MetaStringBytes packageNameBytes) { @@ -159,40 +222,61 @@ private String decodeTypeName(MetaStringBytes classNameBytes) { return classNameBytes.decode(TYPE_NAME_DECODER); } - private void register(Class type, Serializer serializer, int xtypeId) { - if (serializer == null) { - if (type.isEnum()) { - classResolver.registerSerializer( - type, new EnumSerializer(classResolver.getFury(), (Class) type)); - } else { - classResolver.registerSerializer( - type, new StructSerializer<>(classResolver.getFury(), type)); - } - } - registeredTypeIds.add(xtypeId); - ClassInfo classInfo = classResolver.getClassInfo(type); - xtypeIdToClassMap.put(xtypeId, classInfo); - classInfo.xtypeId = xtypeId; - } - public void registerSerializer(Class type, Class serializerClass) { - checkClassRegistration(type); - classResolver.registerSerializer(type, serializerClass); + ClassInfo classInfo = checkClassRegistration(type); + classInfo.serializer = Serializers.newSerializer(fury, type, serializerClass); } public void registerSerializer(Class type, Serializer serializer) { - checkClassRegistration(type); - classResolver.registerSerializer(type, serializer); + ClassInfo classInfo = checkClassRegistration(type); + classInfo.serializer = serializer; } - private void checkClassRegistration(Class type) { - ClassInfo classInfo = classResolver.getClassInfo(type, false); + private ClassInfo checkClassRegistration(Class type) { + ClassInfo classInfo = classInfoMap.get(type); Preconditions.checkArgument( classInfo != null && (classInfo.xtypeId != 0 || !type.getSimpleName().equals(decodeTypeName(classInfo.classNameBytes))), "Type %s should be registered with id or namespace+typename before register serializer", type); + return classInfo; + } + + public ClassInfo getClassInfo(Class cls, ClassInfoHolder classInfoHolder) { + ClassInfo classInfo = classInfoHolder.classInfo; + if (classInfo.getCls() != cls) { + classInfo = classInfoMap.get(cls); + if (classInfo == null) { + classInfo = buildClassInfo(cls); + } + classInfoHolder.classInfo = classInfo; + } + assert classInfo.serializer != null; + return classInfo; + } + + private ClassInfo buildClassInfo(Class cls) { + Serializer serializer; + int xtypeId; + if (classResolver.isSet(cls)) { + serializer = new CollectionSerializer(fury, cls); + xtypeId = Types.SET; + } else if (classResolver.isCollection(cls)) { + serializer = new CollectionSerializer(fury, cls); + xtypeId = Types.LIST; + } else if (cls.isArray() && !TypeUtils.getArrayComponent(cls).isPrimitive()) { + serializer = new ObjectArraySerializer(fury, cls); + xtypeId = Types.LIST; + } else if (classResolver.isMap(cls)) { + serializer = new MapSerializer(fury, cls); + xtypeId = Types.MAP; + } else { + throw new ClassUnregisteredException(cls); + } + ClassInfo info = newClassInfo(cls, serializer, (short) xtypeId); + classInfoMap.put(cls, info); + return info; } private void registerDefaultTypes() { @@ -225,22 +309,18 @@ private void registerDefaultTypes() { } private void registerDefaultTypes(int xtypeId, Class defaultType, Class... otherTypes) { - internalRegister(defaultType, xtypeId); - xtypeIdToClassMap.put(xtypeId, classResolver.getClassInfo(defaultType)); + ClassInfo classInfo = + newClassInfo(defaultType, classResolver.getSerializer(defaultType), (short) xtypeId); + classInfoMap.put(defaultType, classInfo); + xtypeIdToClassMap.put(xtypeId, classInfo); for (Class otherType : otherTypes) { - internalRegister(otherType, xtypeId); + classInfo = newClassInfo(otherType, classResolver.getSerializer(otherType), (short) xtypeId); + classInfoMap.put(otherType, classInfo); } } - private void internalRegister(Class type, int xtypeId) { - ClassInfo classInfo = classResolver.getClassInfo(type); - Preconditions.checkArgument( - classInfo.xtypeId == 0, "Type %s has be registered with id %s", type, classInfo.xtypeId); - classInfo.xtypeId = xtypeId; - } - public ClassInfo writeClassInfo(MemoryBuffer buffer, Object obj) { - ClassInfo classInfo = classResolver.getOrUpdateClassInfo(obj.getClass()); + ClassInfo classInfo = getClassInfo(obj.getClass(), classInfoCache); int xtypeId = classInfo.getXtypeId(); byte internalTypeId = (byte) xtypeId; buffer.writeVarUint32Small7(xtypeId); @@ -267,9 +347,75 @@ public ClassInfo readClassInfo(MemoryBuffer buffer) { case Types.NS_EXT: MetaStringBytes packageBytes = metaStringResolver.readMetaStringBytes(buffer); MetaStringBytes simpleClassNameBytes = metaStringResolver.readMetaStringBytes(buffer); - return classResolver.loadBytesToClassInfo(packageBytes, simpleClassNameBytes); + return loadBytesToClassInfo(internalTypeId, packageBytes, simpleClassNameBytes); default: return xtypeIdToClassMap.get(xtypeId); } } + + private ClassInfo loadBytesToClassInfo( + int internalTypeId, MetaStringBytes packageBytes, MetaStringBytes simpleClassNameBytes) { + ClassNameBytes classNameBytes = + new ClassNameBytes(packageBytes.hashCode, simpleClassNameBytes.hashCode); + ClassInfo classInfo = compositeClassNameBytes2ClassInfo.get(classNameBytes); + if (classInfo == null) { + classInfo = + populateBytesToClassInfo( + internalTypeId, classNameBytes, packageBytes, simpleClassNameBytes); + } + return classInfo; + } + + private ClassInfo populateBytesToClassInfo( + int typeId, + ClassNameBytes classNameBytes, + MetaStringBytes packageBytes, + MetaStringBytes simpleClassNameBytes) { + String namespace = packageBytes.decode(PACKAGE_DECODER); + String typeName = simpleClassNameBytes.decode(TYPE_NAME_DECODER); + String qualifiedName = qualifiedName(namespace, typeName); + ClassInfo classInfo = qualifiedType2ClassInfo.get(qualifiedName); + if (classInfo == null) { + String msg = String.format("Class %s not registered", qualifiedName); + Class type = null; + if (config.deserializeNonexistentClass()) { + LOG.warn(msg); + switch (typeId) { + case Types.NS_ENUM: + case Types.NS_STRUCT: + case Types.NS_COMPATIBLE_STRUCT: + type = + NonexistentClass.getNonexistentClass( + qualifiedName, isEnum(typeId), 0, config.isMetaShareEnabled()); + break; + case Types.NS_EXT: + throw new SerializerUnregisteredException(qualifiedName); + } + } else { + throw new ClassUnregisteredException(qualifiedName); + } + MetaStringBytes fullClassNameBytes = + metaStringResolver.getOrCreateMetaStringBytes( + PACKAGE_ENCODER.encode(qualifiedName, MetaString.Encoding.UTF_8)); + classInfo = + new ClassInfo( + type, + fullClassNameBytes, + packageBytes, + simpleClassNameBytes, + false, + null, + NO_CLASS_ID, + NOT_SUPPORT_XLANG); + if (NonexistentClass.class.isAssignableFrom(TypeUtils.getComponentIfArray(type))) { + classInfo.serializer = NonexistentClassSerializers.getSerializer(fury, qualifiedName, type); + } + } + compositeClassNameBytes2ClassInfo.put(classNameBytes, classInfo); + return classInfo; + } + + private boolean isEnum(int internalTypeId) { + return internalTypeId == Types.ENUM || internalTypeId == Types.NS_ENUM; + } } diff --git a/java/fury-core/src/main/java/org/apache/fury/type/TypeUtils.java b/java/fury-core/src/main/java/org/apache/fury/type/TypeUtils.java index 73861e5c24..b7462554b5 100644 --- a/java/fury-core/src/main/java/org/apache/fury/type/TypeUtils.java +++ b/java/fury-core/src/main/java/org/apache/fury/type/TypeUtils.java @@ -51,6 +51,7 @@ import org.apache.fury.reflect.TypeParameter; import org.apache.fury.reflect.TypeRef; import org.apache.fury.util.Preconditions; +import org.apache.fury.util.StringUtils; /** Type utils for common type inference and extraction. */ @SuppressWarnings({"UnstableApiUsage", "unchecked"}) @@ -734,4 +735,12 @@ public static boolean isEnumArray(Class clz) { } return getArrayComponent(clz).isEnum(); } + + public static String qualifiedName(String pkg, String className) { + if (StringUtils.isBlank(pkg)) { + return className; + } else { + return pkg + "." + className; + } + } } From b3db2c786f3241c8eac202aeb2ba92cbdf23c30a Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 24 Nov 2024 01:17:58 +0800 Subject: [PATCH 14/18] add header --- .../apache/fury/resolver/ClassNameBytes.java | 19 +++++++++++++++++++ .../apache/fury/resolver/XtypeResolver.java | 19 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassNameBytes.java b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassNameBytes.java index f1840bb706..ab2c01dd8a 100644 --- a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassNameBytes.java +++ b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassNameBytes.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.fury.resolver; class ClassNameBytes { diff --git a/java/fury-core/src/main/java/org/apache/fury/resolver/XtypeResolver.java b/java/fury-core/src/main/java/org/apache/fury/resolver/XtypeResolver.java index 70078dc4e1..2b9f394b4a 100644 --- a/java/fury-core/src/main/java/org/apache/fury/resolver/XtypeResolver.java +++ b/java/fury-core/src/main/java/org/apache/fury/resolver/XtypeResolver.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.fury.resolver; import static org.apache.fury.Fury.NOT_SUPPORT_XLANG; From c9d9ff0de1cb9accb8222099daccf803647d0d69 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 24 Nov 2024 01:19:13 +0800 Subject: [PATCH 15/18] add license header for exception --- .../exception/ClassUnregisteredException.java | 19 +++++++++++++++ .../SerializerUnregisteredException.java | 23 +++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/java/fury-core/src/main/java/org/apache/fury/exception/ClassUnregisteredException.java b/java/fury-core/src/main/java/org/apache/fury/exception/ClassUnregisteredException.java index e438d92913..ca8874f5fa 100644 --- a/java/fury-core/src/main/java/org/apache/fury/exception/ClassUnregisteredException.java +++ b/java/fury-core/src/main/java/org/apache/fury/exception/ClassUnregisteredException.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.fury.exception; public class ClassUnregisteredException extends FuryException { diff --git a/java/fury-core/src/main/java/org/apache/fury/exception/SerializerUnregisteredException.java b/java/fury-core/src/main/java/org/apache/fury/exception/SerializerUnregisteredException.java index 69a882ebdc..967867bcbf 100644 --- a/java/fury-core/src/main/java/org/apache/fury/exception/SerializerUnregisteredException.java +++ b/java/fury-core/src/main/java/org/apache/fury/exception/SerializerUnregisteredException.java @@ -1,7 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.fury.exception; public class SerializerUnregisteredException extends FuryException { + public SerializerUnregisteredException(Class cls) { + this(cls.getName()); + } + public SerializerUnregisteredException(String qualifiedName) { super(String.format("Class %s is not registered with a serializer", qualifiedName)); } From 79159ce0cf69190db0c78f56b89de181df99869d Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 24 Nov 2024 09:28:49 +0800 Subject: [PATCH 16/18] add date support --- .../src/main/java/org/apache/fury/resolver/XtypeResolver.java | 2 ++ java/fury-core/src/main/java/org/apache/fury/type/Types.java | 1 + 2 files changed, 3 insertions(+) diff --git a/java/fury-core/src/main/java/org/apache/fury/resolver/XtypeResolver.java b/java/fury-core/src/main/java/org/apache/fury/resolver/XtypeResolver.java index 2b9f394b4a..0095d1768f 100644 --- a/java/fury-core/src/main/java/org/apache/fury/resolver/XtypeResolver.java +++ b/java/fury-core/src/main/java/org/apache/fury/resolver/XtypeResolver.java @@ -32,6 +32,7 @@ import java.sql.Timestamp; import java.time.Duration; import java.time.Instant; +import java.time.LocalDate; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.Date; @@ -325,6 +326,7 @@ private void registerDefaultTypes() { registerDefaultTypes(Types.LIST, ArrayList.class, Object[].class); registerDefaultTypes(Types.SET, HashSet.class, LinkedHashSet.class); registerDefaultTypes(Types.MAP, HashMap.class, LinkedHashMap.class); + registerDefaultTypes(Types.DATE, LocalDate.class); } private void registerDefaultTypes(int xtypeId, Class defaultType, Class... otherTypes) { diff --git a/java/fury-core/src/main/java/org/apache/fury/type/Types.java b/java/fury-core/src/main/java/org/apache/fury/type/Types.java index 6b30cb487b..57e2188cbd 100644 --- a/java/fury-core/src/main/java/org/apache/fury/type/Types.java +++ b/java/fury-core/src/main/java/org/apache/fury/type/Types.java @@ -42,4 +42,5 @@ public class Types { public static final int SPARSE_TENSOR = 36; public static final int ARROW_RECORD_BATCH = 37; public static final int ARROW_TABLE = 38; + public static final int DATE = 39; } From bc15ed7ea8f881907191362a20efb2076c17a15f Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sun, 24 Nov 2024 16:43:13 +0800 Subject: [PATCH 17/18] refine type system --- .../apache/fury/resolver/XtypeResolver.java | 2 +- .../main/java/org/apache/fury/type/Types.java | 57 ++++++++++--------- 2 files changed, 31 insertions(+), 28 deletions(-) diff --git a/java/fury-core/src/main/java/org/apache/fury/resolver/XtypeResolver.java b/java/fury-core/src/main/java/org/apache/fury/resolver/XtypeResolver.java index 0095d1768f..bc68973c39 100644 --- a/java/fury-core/src/main/java/org/apache/fury/resolver/XtypeResolver.java +++ b/java/fury-core/src/main/java/org/apache/fury/resolver/XtypeResolver.java @@ -326,7 +326,7 @@ private void registerDefaultTypes() { registerDefaultTypes(Types.LIST, ArrayList.class, Object[].class); registerDefaultTypes(Types.SET, HashSet.class, LinkedHashSet.class); registerDefaultTypes(Types.MAP, HashMap.class, LinkedHashMap.class); - registerDefaultTypes(Types.DATE, LocalDate.class); + registerDefaultTypes(Types.LOCAL_DATE, LocalDate.class); } private void registerDefaultTypes(int xtypeId, Class defaultType, Class... otherTypes) { diff --git a/java/fury-core/src/main/java/org/apache/fury/type/Types.java b/java/fury-core/src/main/java/org/apache/fury/type/Types.java index 57e2188cbd..96694d677e 100644 --- a/java/fury-core/src/main/java/org/apache/fury/type/Types.java +++ b/java/fury-core/src/main/java/org/apache/fury/type/Types.java @@ -13,34 +13,37 @@ public class Types { public static final int FLOAT32 = 10; public static final int FLOAT64 = 11; public static final int STRING = 12; - public static final int ENUM = 13; public static final int NS_ENUM = 14; public static final int STRUCT = 15; - public static final int COMPATIBLE_STRUCT = 15; - public static final int NS_STRUCT = 16; - public static final int NS_COMPATIBLE_STRUCT = 46; - public static final int EXT = 17; - public static final int NS_EXT = 18; - public static final int LIST = 19; - public static final int SET = 20; - public static final int MAP = 21; - public static final int DURATION = 22; - public static final int TIMESTAMP = 23; - public static final int DECIMAL = 24; - public static final int BINARY = 25; - public static final int ARRAY = 26; - public static final int BOOL_ARRAY = 27; - public static final int INT8_ARRAY = 28; - public static final int INT16_ARRAY = 29; - public static final int INT32_ARRAY = 30; - public static final int INT64_ARRAY = 31; - public static final int FLOAT16_ARRAY = 32; - public static final int FLOAT32_ARRAY = 33; - public static final int FLOAT64_ARRAY = 34; - public static final int TENSOR = 35; - public static final int SPARSE_TENSOR = 36; - public static final int ARROW_RECORD_BATCH = 37; - public static final int ARROW_TABLE = 38; - public static final int DATE = 39; + public static final int POLYMORPHIC_STRUCT = 16; + public static final int COMPATIBLE_STRUCT = 17; + public static final int POLYMORPHIC_COMPATIBLE_STRUCT = 18; + public static final int NS_STRUCT = 19; + public static final int NS_POLYMORPHIC_STRUCT = 20; + public static final int NS_COMPATIBLE_STRUCT = 21; + public static final int NS_POLYMORPHIC_COMPATIBLE_STRUCT = 22; + public static final int EXT = 23; + public static final int POLYMORPHIC_EXT = 24; + public static final int NS_EXT = 25; + public static final int NS_POLYMORPHIC_EXT = 26; + public static final int LIST = 27; + public static final int SET = 28; + public static final int MAP = 29; + public static final int DURATION = 30; + public static final int TIMESTAMP = 31; + public static final int LOCAL_DATE = 32; + public static final int DECIMAL = 33; + public static final int BINARY = 34; + public static final int ARRAY = 35; + public static final int BOOL_ARRAY = 36; + public static final int INT8_ARRAY = 37; + public static final int INT16_ARRAY = 38; + public static final int INT32_ARRAY = 39; + public static final int INT64_ARRAY = 40; + public static final int FLOAT16_ARRAY = 41; + public static final int FLOAT32_ARRAY = 42; + public static final int FLOAT64_ARRAY = 43; + public static final int ARROW_RECORD_BATCH = 44; + public static final int ARROW_TABLE = 45; } From 3ef5e35ee16e5bd7fb695016577802b2722d2359 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Mon, 25 Nov 2024 01:10:55 +0800 Subject: [PATCH 18/18] add type name hash check --- .../org/apache/fury/resolver/ClassInfo.java | 11 +++++++++++ .../apache/fury/resolver/XtypeResolver.java | 18 +++++------------- .../fury/serializer/StructSerializer.java | 7 ++++++- .../main/java/org/apache/fury/type/Types.java | 11 +++++++++++ 4 files changed, 33 insertions(+), 14 deletions(-) diff --git a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassInfo.java b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassInfo.java index 35e2393119..6acd0b59d9 100644 --- a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassInfo.java +++ b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassInfo.java @@ -20,6 +20,8 @@ package org.apache.fury.resolver; import static org.apache.fury.meta.Encoders.GENERIC_ENCODER; +import static org.apache.fury.meta.Encoders.PACKAGE_DECODER; +import static org.apache.fury.meta.Encoders.TYPE_NAME_DECODER; import org.apache.fury.collection.Tuple2; import org.apache.fury.config.Language; @@ -142,6 +144,15 @@ void setSerializer(ClassResolver resolver, Serializer serializer) { needToWriteClassDef = serializer != null && resolver.needToWriteClassDef(serializer); } + + public String decodeNamespace() { + return packageNameBytes.decode(PACKAGE_DECODER); + } + + public String decodeTypeName() { + return classNameBytes.decode(TYPE_NAME_DECODER); + } + @Override public String toString() { return "ClassInfo{" diff --git a/java/fury-core/src/main/java/org/apache/fury/resolver/XtypeResolver.java b/java/fury-core/src/main/java/org/apache/fury/resolver/XtypeResolver.java index bc68973c39..5771cf8bcf 100644 --- a/java/fury-core/src/main/java/org/apache/fury/resolver/XtypeResolver.java +++ b/java/fury-core/src/main/java/org/apache/fury/resolver/XtypeResolver.java @@ -127,8 +127,8 @@ public void register(Class type, int typeId) { throw new IllegalArgumentException( String.format("Type %s has been registered with id %s", type, classInfo.xtypeId)); } - String prevNamespace = decodeNamespace(classInfo.packageNameBytes); - String prevTypeName = decodeTypeName(classInfo.classNameBytes); + String prevNamespace = classInfo.decodeNamespace(); + String prevTypeName = classInfo.decodeTypeName(); if (!type.getSimpleName().equals(prevTypeName)) { throw new IllegalArgumentException( String.format( @@ -167,8 +167,8 @@ public void register(Class type, String namespace, String typeName) { if (classInfo != null) { serializer = classInfo.serializer; if (classInfo.classNameBytes != null) { - String prevNamespace = decodeNamespace(classInfo.packageNameBytes); - String prevTypeName = decodeTypeName(classInfo.classNameBytes); + String prevNamespace = classInfo.decodeNamespace(); + String prevTypeName = classInfo.decodeTypeName(); if (!namespace.equals(prevNamespace) || typeName.equals(prevTypeName)) { throw new IllegalArgumentException( String.format( @@ -234,14 +234,6 @@ private ClassInfo newClassInfo( type, fullClassNameBytes, nsBytes, classNameBytes, false, serializer, NO_CLASS_ID, xtypeId); } - private String decodeNamespace(MetaStringBytes packageNameBytes) { - return packageNameBytes.decode(PACKAGE_DECODER); - } - - private String decodeTypeName(MetaStringBytes classNameBytes) { - return classNameBytes.decode(TYPE_NAME_DECODER); - } - public void registerSerializer(Class type, Class serializerClass) { ClassInfo classInfo = checkClassRegistration(type); classInfo.serializer = Serializers.newSerializer(fury, type, serializerClass); @@ -257,7 +249,7 @@ private ClassInfo checkClassRegistration(Class type) { Preconditions.checkArgument( classInfo != null && (classInfo.xtypeId != 0 - || !type.getSimpleName().equals(decodeTypeName(classInfo.classNameBytes))), + || !type.getSimpleName().equals(classInfo.decodeTypeName())), "Type %s should be registered with id or namespace+typename before register serializer", type); return classInfo; diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/StructSerializer.java b/java/fury-core/src/main/java/org/apache/fury/serializer/StructSerializer.java index 9e36c745b2..a0f30f3bb2 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/StructSerializer.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/StructSerializer.java @@ -40,6 +40,7 @@ import org.apache.fury.type.Descriptor; import org.apache.fury.type.GenericType; import org.apache.fury.type.Generics; +import org.apache.fury.type.TypeUtils; import org.apache.fury.type.Types; import org.apache.fury.util.ExceptionUtils; import org.apache.fury.util.Preconditions; @@ -220,7 +221,11 @@ int computeFieldHash(int hash, GenericType fieldGeneric) { try { ClassInfo classInfo = fury.getClassResolver().getClassInfo(fieldGeneric.getCls()); int xtypeId = classInfo.getXtypeId(); - id = Math.abs(xtypeId); + if (Types.isStructType((byte) xtypeId)) { + id = TypeUtils.computeStringHash(classInfo.decodeNamespace() + classInfo.decodeTypeName()); + } else { + id = Math.abs(xtypeId); + } } catch (Exception e) { return hash; } diff --git a/java/fury-core/src/main/java/org/apache/fury/type/Types.java b/java/fury-core/src/main/java/org/apache/fury/type/Types.java index 96694d677e..59062391e3 100644 --- a/java/fury-core/src/main/java/org/apache/fury/type/Types.java +++ b/java/fury-core/src/main/java/org/apache/fury/type/Types.java @@ -46,4 +46,15 @@ public class Types { public static final int FLOAT64_ARRAY = 43; public static final int ARROW_RECORD_BATCH = 44; public static final int ARROW_TABLE = 45; + + public static boolean isStructType(int value) { + return value == STRUCT || + value == POLYMORPHIC_STRUCT || + value == COMPATIBLE_STRUCT || + value == POLYMORPHIC_COMPATIBLE_STRUCT || + value == NS_STRUCT || + value == NS_POLYMORPHIC_STRUCT || + value == NS_COMPATIBLE_STRUCT || + value == NS_POLYMORPHIC_COMPATIBLE_STRUCT; + } }