diff --git a/docs/specification/xlang_serialization_spec.md b/docs/specification/xlang_serialization_spec.md index 7fd991ceb9..c18ad2289c 100644 --- a/docs/specification/xlang_serialization_spec.md +++ b/docs/specification/xlang_serialization_spec.md @@ -68,6 +68,11 @@ Note: - Unsigned int/long are not added here, since not every language support those types. +### Polymorphisms + +For polymorphism, if one non-final class is registered, and only one subclass is registered, then we can take all +elements in List/Map have same type, thus reduce runtime check cost. + ### Type disambiguation Due to differences between type systems of languages, those types can't be mapped one-to-one between languages. When diff --git a/java/fury-core/src/main/java/org/apache/fury/AbstractThreadSafeFury.java b/java/fury-core/src/main/java/org/apache/fury/AbstractThreadSafeFury.java index 23a6aa407e..3e2aff67ff 100644 --- a/java/fury-core/src/main/java/org/apache/fury/AbstractThreadSafeFury.java +++ b/java/fury-core/src/main/java/org/apache/fury/AbstractThreadSafeFury.java @@ -38,15 +38,25 @@ public void register(Class cls, boolean createSerializer) { } @Override - public void register(Class cls, Short id) { + public void register(Class cls, int id) { registerCallback(fury -> fury.register(cls, id)); } @Override - public void register(Class cls, Short id, boolean createSerializer) { + public void register(Class cls, int id, boolean createSerializer) { registerCallback(fury -> fury.register(cls, id, createSerializer)); } + @Override + public void register(Class cls, String typeName) { + registerCallback(fury -> fury.register(cls, typeName)); + } + + @Override + public void register(Class cls, String namespace, String typeName) { + registerCallback(fury -> fury.register(cls, namespace, typeName)); + } + @Override public void registerSerializer(Class type, Class serializerClass) { registerCallback(fury -> fury.registerSerializer(type, serializerClass)); diff --git a/java/fury-core/src/main/java/org/apache/fury/BaseFury.java b/java/fury-core/src/main/java/org/apache/fury/BaseFury.java index 96a04be021..81c6e8662d 100644 --- a/java/fury-core/src/main/java/org/apache/fury/BaseFury.java +++ b/java/fury-core/src/main/java/org/apache/fury/BaseFury.java @@ -33,14 +33,21 @@ public interface BaseFury { /** - * register class. + * Register class and allocate an auto-grown ID for this class. Note that the registration order + * is important. If registration order is inconsistent, the allocated ID will be different, and + * the deserialization will failed. * * @param cls class to register. */ void register(Class cls); + /** register class with given id. */ + void register(Class cls, int id); + /** - * Register class. + * Register class and allocate an auto-grown ID for this class. Note that the registration order + * is important. If registration order is inconsistent, the allocated ID will be different, and + * the deserialization will failed. * * @param cls class to register. * @param createSerializer whether to create serializer, if true and codegen enabled, this will @@ -48,9 +55,6 @@ public interface BaseFury { */ void register(Class cls, boolean createSerializer); - /** register class with given id. */ - void register(Class cls, Short id); - /** * Register class with specified id. * @@ -59,10 +63,21 @@ public interface BaseFury { * @param createSerializer whether to create serializer, if true and codegen enabled, this will * generate the serializer code too. */ - void register(Class cls, Short id, boolean createSerializer); + void register(Class cls, int id, boolean createSerializer); + + /** register class with given type name which will be used for cross-language serialization. */ + void register(Class cls, String typeName); + + /** + * register class with given type namespace and name which will be used for cross-language + * serialization. + */ + void register(Class cls, String namespace, String typeName); /** - * Register a Serializer. + * Register a Serializer for a class, and allocate an auto-grown ID for this class if it's not + * registered yet. Note that the registration order is important. If registration order is + * inconsistent, the allocated ID will be different, and the deserialization will failed. * * @param type class needed to be serialized/deserialized. * @param serializerClass serializer class can be created with {@link Serializers#newSerializer}. @@ -70,10 +85,18 @@ public interface BaseFury { */ void registerSerializer(Class type, Class serializerClass); + /** + * Register a Serializer for a class, and allocate an auto-grown ID for this class if it's not + * registered yet. Note that the registration order is important. If registration order is + * inconsistent, the allocated ID will be different, and the deserialization will failed. + */ void registerSerializer(Class type, Serializer serializer); /** - * Register a Serializer created by serializerCreator when fury created. + * Register a Serializer created by serializerCreator when fury created. And allocate an + * auto-grown ID for this class if it's not registered yet. Note that the registration order is + * important. If registration order is inconsistent, the allocated ID will be different, and the + * deserialization will failed. * * @param type class needed to be serialized/deserialized. * @param serializerCreator serializer creator with param {@link Fury} diff --git a/java/fury-core/src/main/java/org/apache/fury/Fury.java b/java/fury-core/src/main/java/org/apache/fury/Fury.java index e10e989ab0..b86432ffaa 100644 --- a/java/fury-core/src/main/java/org/apache/fury/Fury.java +++ b/java/fury-core/src/main/java/org/apache/fury/Fury.java @@ -27,7 +27,6 @@ import java.util.Arrays; import java.util.HashMap; import java.util.Iterator; -import java.util.List; import java.util.function.Consumer; import java.util.function.Function; import javax.annotation.concurrent.NotThreadSafe; @@ -53,10 +52,10 @@ import org.apache.fury.resolver.NoRefResolver; import org.apache.fury.resolver.RefResolver; import org.apache.fury.resolver.SerializationContext; +import org.apache.fury.resolver.XtypeResolver; import org.apache.fury.serializer.ArraySerializers; import org.apache.fury.serializer.BufferCallback; import org.apache.fury.serializer.BufferObject; -import org.apache.fury.serializer.OpaqueObjects; import org.apache.fury.serializer.PrimitiveSerializers.LongSerializer; import org.apache.fury.serializer.Serializer; import org.apache.fury.serializer.SerializerFactory; @@ -64,7 +63,7 @@ import org.apache.fury.serializer.collection.CollectionSerializers.ArrayListSerializer; import org.apache.fury.serializer.collection.MapSerializers.HashMapSerializer; import org.apache.fury.type.Generics; -import org.apache.fury.type.Type; +import org.apache.fury.type.Types; import org.apache.fury.util.ExceptionUtils; import org.apache.fury.util.Preconditions; import org.apache.fury.util.StringUtils; @@ -90,8 +89,7 @@ public final class Fury implements BaseFury { public static final byte NOT_NULL_VALUE_FLAG = -1; // this flag indicates that the object is a referencable and first write. public static final byte REF_VALUE_FLAG = 0; - public static final byte NOT_SUPPORT_CROSS_LANGUAGE = 0; - public static final short FURY_TYPE_TAG_ID = Type.FURY_TYPE_TAG.getId(); + public static final byte NOT_SUPPORT_XLANG = 0; private static final byte isNilFlag = 1; private static final byte isLittleEndianFlag = 1 << 1; private static final byte isCrossLanguageFlag = 1 << 2; @@ -106,12 +104,12 @@ public final class Fury implements BaseFury { private final boolean shareMeta; private final RefResolver refResolver; private final ClassResolver classResolver; + private final XtypeResolver xtypeResolver; private final MetaStringResolver metaStringResolver; private final SerializationContext serializationContext; private final ClassLoader classLoader; private final JITContext jitContext; private MemoryBuffer buffer; - private final List nativeObjects; private final StringSerializer stringSerializer; private final ArrayListSerializer arrayListSerializer; private final HashMapSerializer hashMapSerializer; @@ -148,9 +146,9 @@ public Fury(FuryBuilder builder, ClassLoader classLoader) { metaStringResolver = new MetaStringResolver(); classResolver = new ClassResolver(this); classResolver.initialize(); + xtypeResolver = new XtypeResolver(this); serializationContext = new SerializationContext(config); this.classLoader = classLoader; - nativeObjects = new ArrayList<>(); generics = new Generics(this); stringSerializer = new StringSerializer(this); arrayListSerializer = new ArrayListSerializer(this); @@ -162,42 +160,82 @@ public Fury(FuryBuilder builder, ClassLoader classLoader) { @Override public void register(Class cls) { - classResolver.register(cls); + if (language == Language.JAVA) { + classResolver.register(cls); + } else { + xtypeResolver.register(cls); + } } @Override - public void register(Class cls, boolean createSerializer) { - classResolver.register(cls, createSerializer); + public void register(Class cls, int id) { + if (language == Language.JAVA) { + classResolver.register(cls, id); + } else { + xtypeResolver.register(cls, id); + } } @Override - public void register(Class cls, Short id) { - classResolver.register(cls, id); + public void register(Class cls, boolean createSerializer) { + if (language == Language.JAVA) { + classResolver.register(cls, createSerializer); + } else { + xtypeResolver.register(cls); + } } @Override - public void register(Class cls, Short id, boolean createSerializer) { - classResolver.register(cls, id, createSerializer); + public void register(Class cls, int id, boolean createSerializer) { + if (language == Language.JAVA) { + classResolver.register(cls, id, createSerializer); + } else { + xtypeResolver.register(cls, id); + } } /** register class with given type tag which will be used for cross-language serialization. */ - public void register(Class cls, String typeTag) { - classResolver.register(cls, typeTag); + public void register(Class cls, String typeName) { + Preconditions.checkArgument(language != Language.JAVA); + int idx = typeName.lastIndexOf('.'); + String namespace = ""; + if (idx > 0) { + namespace = typeName.substring(0, idx); + typeName = typeName.substring(idx + 1); + } + register(cls, namespace, typeName); + } + + public void register(Class cls, String namespace, String typeName) { + Preconditions.checkArgument(language != Language.JAVA); + xtypeResolver.register(cls, namespace, typeName); } @Override public void registerSerializer(Class type, Class serializerClass) { - classResolver.registerSerializer(type, serializerClass); + if (language == Language.JAVA) { + classResolver.registerSerializer(type, serializerClass); + } else { + xtypeResolver.registerSerializer(type, serializerClass); + } } @Override public void registerSerializer(Class type, Serializer serializer) { - classResolver.registerSerializer(type, serializer); + if (language == Language.JAVA) { + classResolver.registerSerializer(type, serializer); + } else { + xtypeResolver.registerSerializer(type, serializer); + } } @Override public void registerSerializer(Class type, Function> serializerCreator) { - classResolver.registerSerializer(type, serializerCreator.apply(this)); + if (language == Language.JAVA) { + classResolver.registerSerializer(type, serializerCreator.apply(this)); + } else { + xtypeResolver.registerSerializer(type, serializerCreator.apply(this)); + } } @Override @@ -269,7 +307,7 @@ public MemoryBuffer serialize(MemoryBuffer buffer, Object obj, BufferCallback ca write(buffer, obj); } else { buffer.writeByte((byte) Language.JAVA.ordinal()); - xserializeInternal(buffer, obj); + xwriteRef(buffer, obj); } return buffer; } catch (StackOverflowError t) { @@ -337,8 +375,9 @@ public void resetBuffer() { private void write(MemoryBuffer buffer, Object obj) { int startOffset = buffer.writerIndex(); + boolean shareMeta = config.isMetaShareEnabled(); if (shareMeta) { - buffer.writeInt32(-1); // preserve 4-byte for nativeObjects start offsets. + buffer.writeInt32(-1); // preserve 4-byte for meta start offsets. } // reduce caller stack if (!refResolver.writeRefOrNull(buffer, obj)) { @@ -353,22 +392,6 @@ private void write(MemoryBuffer buffer, Object obj) { } } - private void xserializeInternal(MemoryBuffer buffer, Object obj) { - int startOffset = buffer.writerIndex(); - buffer.writeInt32(-1); // preserve 4-byte for nativeObjects start offsets. - buffer.writeInt32(-1); // preserve 4-byte for nativeObjects size - xwriteRef(buffer, obj); - buffer.putInt32(startOffset, buffer.writerIndex()); - buffer.putInt32(startOffset + 4, nativeObjects.size()); - refResolver.resetWrite(); - // fury write opaque object classname which cause later write of classname only write an id. - classResolver.resetWrite(); - metaStringResolver.resetWrite(); - for (Object nativeObject : nativeObjects) { - writeRef(buffer, nativeObject); - } - } - /** Serialize a nullable referencable object to buffer. */ public void writeRef(MemoryBuffer buffer, Object obj) { if (!refResolver.writeRefOrNull(buffer, obj)) { @@ -476,72 +499,69 @@ public void writeNonRef(MemoryBuffer buffer, Object obj, ClassInfo classInfo) { depth--; } - public void writeNonRef(MemoryBuffer buffer, T obj, Serializer serializer) { - depth++; - serializer.write(buffer, obj); - depth--; - } - public void xwriteRef(MemoryBuffer buffer, Object obj) { if (!refResolver.writeRefOrNull(buffer, obj)) { - xwriteNonRef(buffer, obj, null); + ClassInfo classInfo = xtypeResolver.writeClassInfo(buffer, obj); + switch (classInfo.getXtypeId()) { + case Types.BOOL: + buffer.writeBoolean((Boolean) obj); + break; + case Types.INT8: + buffer.writeByte((Byte) obj); + break; + case Types.INT16: + buffer.writeInt16((Short) obj); + break; + case Types.INT32: + buffer.writeInt32((Integer) obj); + break; + case Types.VAR_INT32: + buffer.writeVarInt32((Integer) obj); + break; + case Types.INT64: + buffer.writeInt64((Long) obj); + break; + case Types.VAR_INT64: + buffer.writeVarInt64((Long) obj); + break; + case Types.SLI_INT64: + buffer.writeSliInt64((Long) obj); + break; + case Types.FLOAT32: + buffer.writeFloat32((Float) obj); + break; + case Types.FLOAT64: + buffer.writeFloat64((Double) obj); + break; + case Types.STRING: + stringSerializer.writeString(buffer, (String) obj); + break; + // TODO(add fastpath for other types) + default: + depth++; + classInfo.getSerializer().xwrite(buffer, obj); + depth--; + } } } public void xwriteRef(MemoryBuffer buffer, T obj, Serializer serializer) { if (serializer.needToWriteRef()) { if (!refResolver.writeRefOrNull(buffer, obj)) { - xwriteNonRef(buffer, obj, serializer); + depth++; + serializer.xwrite(buffer, obj); + depth--; } } else { if (obj == null) { buffer.writeByte(Fury.NULL_FLAG); } else { buffer.writeByte(Fury.NOT_NULL_VALUE_FLAG); - xwriteNonRef(buffer, obj, serializer); - } - } - } - - public void xwriteRefByNullableSerializer( - MemoryBuffer buffer, T obj, Serializer serializer) { - if (serializer == null) { - xwriteRef(buffer, obj); - } else { - xwriteRef(buffer, obj, serializer); - } - } - - public void xwriteNonRef(MemoryBuffer buffer, T obj, Serializer serializer) { - depth++; - @SuppressWarnings("unchecked") - Class cls = (Class) obj.getClass(); - if (serializer == null) { - serializer = classResolver.getSerializer(cls); - } - short typeId = serializer.getXtypeId(); - buffer.writeInt16(typeId); - if (typeId != NOT_SUPPORT_CROSS_LANGUAGE) { - if (typeId == FURY_TYPE_TAG_ID) { - classResolver.xwriteTypeTag(buffer, cls); - } - if (typeId < NOT_SUPPORT_CROSS_LANGUAGE) { - classResolver.xwriteClass(buffer, cls); + depth++; + serializer.xwrite(buffer, obj); + depth--; } - serializer.xwrite(buffer, obj); - } else { - // Write classname so it can be used for debugging which object doesn't support - // cross-language. - // TODO add a config to disable this to reduce space cost. - classResolver.xwriteClass(buffer, cls); - // serializer may increase reference id multi times internally, thus peer cross-language later - // fields/objects deserialization will use wrong reference id since we skip opaque objects - // deserialization. - // So we stash native objects and serialize all those object at the last. - buffer.writeVarUint32(nativeObjects.size()); - nativeObjects.add(obj); } - depth--; } /** Write not null data to buffer. */ @@ -785,7 +805,7 @@ public Object deserialize(MemoryBuffer buffer, Iterable outOfBandB } Object obj; if (isTargetXLang) { - obj = xdeserializeInternal(buffer); + obj = xreadRef(buffer); } else { if (shareMeta) { readClassDefs(buffer); @@ -830,28 +850,6 @@ public Object deserialize(FuryReadableChannel channel, Iterable ou return deserialize(buf, outOfBandBuffers); } - private Object xdeserializeInternal(MemoryBuffer buffer) { - Object obj; - int nativeObjectsStartOffset = buffer.readInt32(); - int nativeObjectsSize = buffer.readInt32(); - int endReaderIndex = nativeObjectsStartOffset; - if (peerLanguage == Language.JAVA) { - int readerIndex = buffer.readerIndex(); - buffer.readerIndex(nativeObjectsStartOffset); - for (int i = 0; i < nativeObjectsSize; i++) { - nativeObjects.add(readRef(buffer)); - } - endReaderIndex = buffer.readerIndex(); - buffer.readerIndex(readerIndex); - refResolver.resetRead(); - classResolver.resetRead(); - metaStringResolver.resetRead(); - } - obj = xreadRef(buffer); - buffer.readerIndex(endReaderIndex); - return obj; - } - /** Deserialize nullable referencable object from buffer. */ public Object readRef(MemoryBuffer buffer) { RefResolver refResolver = this.refResolver; @@ -966,7 +964,7 @@ public Object xreadRef(MemoryBuffer buffer) { RefResolver refResolver = this.refResolver; int nextReadRefId = refResolver.tryPreserveRefId(buffer); if (nextReadRefId >= NOT_NULL_VALUE_FLAG) { - Object o = xreadNonRef(buffer, null); + Object o = xreadNonRef(buffer, xtypeResolver.readClassInfo(buffer)); refResolver.setReadObject(nextReadRefId, o); return o; } else { @@ -995,52 +993,44 @@ public Object xreadRef(MemoryBuffer buffer, Serializer serializer) { } } - public Object xreadRefByNullableSerializer(MemoryBuffer buffer, Serializer serializer) { - if (serializer == null) { - return xreadRef(buffer); - } else { - return xreadRef(buffer, serializer); - } - } - public Object xreadNonRef(MemoryBuffer buffer, Serializer serializer) { depth++; - short typeId = buffer.readInt16(); - ClassResolver classResolver = this.classResolver; - if (typeId != NOT_SUPPORT_CROSS_LANGUAGE) { - Class cls = null; - if (typeId == FURY_TYPE_TAG_ID) { - cls = classResolver.readClassByTypeTag(buffer); - } - if (typeId < NOT_SUPPORT_CROSS_LANGUAGE) { - if (peerLanguage != Language.JAVA) { - classResolver.xreadClassName(buffer); - cls = classResolver.getClassByTypeId((short) -typeId); - } else { - cls = classResolver.xreadClass(buffer); - } - } else { - if (typeId != FURY_TYPE_TAG_ID) { - cls = classResolver.getClassByTypeId(typeId); - } - } - Preconditions.checkNotNull(cls); - if (serializer == null) { - serializer = classResolver.getSerializer(cls); - } - // TODO check serializer consistent with `classResolver.getSerializer(cls)` when serializer - // not null; - Object o = serializer.xread(buffer); - depth--; - return o; - } else { - String className = classResolver.xreadClassName(buffer); - int ordinal = buffer.readVarUint32(); - if (peerLanguage != Language.JAVA) { - return OpaqueObjects.of(peerLanguage, className, ordinal); - } else { - return nativeObjects.get(ordinal); - } + Object o = serializer.xread(buffer); + depth--; + return o; + } + + public Object xreadNonRef(MemoryBuffer buffer, ClassInfo classInfo) { + assert classInfo != null; + switch (classInfo.getXtypeId()) { + case Types.BOOL: + return buffer.readBoolean(); + case Types.INT8: + return buffer.readByte(); + case Types.INT16: + return buffer.readInt16(); + case Types.INT32: + return buffer.readInt32(); + case Types.VAR_INT32: + return buffer.readVarUint32(); + case Types.INT64: + return buffer.readInt64(); + case Types.VAR_INT64: + return buffer.readVarInt64(); + case Types.SLI_INT64: + return buffer.readSliInt64(); + case Types.FLOAT32: + return buffer.readFloat32(); + case Types.FLOAT64: + return buffer.readFloat64(); + case Types.STRING: + return stringSerializer.readString(buffer); + // TODO(add fastpath for other types) + default: + depth++; + Object o = classInfo.getSerializer().xread(buffer); + depth--; + return o; } } @@ -1063,7 +1053,7 @@ public void serializeJavaObject(MemoryBuffer buffer, Object obj) { } if (config.isMetaShareEnabled()) { int startOffset = buffer.writerIndex(); - buffer.writeInt32(-1); // preserve 4-byte for nativeObjects start offsets. + buffer.writeInt32(-1); // preserve 4-byte for meta start offsets. if (!refResolver.writeRefOrNull(buffer, obj)) { ClassInfo classInfo = classResolver.getOrUpdateClassInfo(obj.getClass()); writeData(buffer, classInfo, obj); @@ -1459,7 +1449,6 @@ public void reset() { classResolver.reset(); metaStringResolver.reset(); serializationContext.reset(); - nativeObjects.clear(); peerOutOfBandEnabled = false; bufferCallback = null; depth = 0; @@ -1471,7 +1460,6 @@ public void resetWrite() { classResolver.resetWrite(); metaStringResolver.resetWrite(); serializationContext.resetWrite(); - nativeObjects.clear(); bufferCallback = null; depth = 0; } @@ -1481,7 +1469,6 @@ public void resetRead() { classResolver.resetRead(); metaStringResolver.resetRead(); serializationContext.resetRead(); - nativeObjects.clear(); peerOutOfBandEnabled = false; depth = 0; classDefEndOffset = -1; diff --git a/java/fury-core/src/main/java/org/apache/fury/exception/ClassUnregisteredException.java b/java/fury-core/src/main/java/org/apache/fury/exception/ClassUnregisteredException.java new file mode 100644 index 0000000000..ca8874f5fa --- /dev/null +++ b/java/fury-core/src/main/java/org/apache/fury/exception/ClassUnregisteredException.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fury.exception; + +public class ClassUnregisteredException extends FuryException { + + public ClassUnregisteredException(Class cls) { + this(cls.getName()); + } + + public ClassUnregisteredException(String qualifiedName) { + super(String.format("Class %s is not registered", qualifiedName)); + } +} diff --git a/java/fury-core/src/main/java/org/apache/fury/exception/SerializerUnregisteredException.java b/java/fury-core/src/main/java/org/apache/fury/exception/SerializerUnregisteredException.java new file mode 100644 index 0000000000..967867bcbf --- /dev/null +++ b/java/fury-core/src/main/java/org/apache/fury/exception/SerializerUnregisteredException.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fury.exception; + +public class SerializerUnregisteredException extends FuryException { + + public SerializerUnregisteredException(Class cls) { + this(cls.getName()); + } + + public SerializerUnregisteredException(String qualifiedName) { + super(String.format("Class %s is not registered with a serializer", qualifiedName)); + } +} diff --git a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassInfo.java b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassInfo.java index 57efd230f2..6acd0b59d9 100644 --- a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassInfo.java +++ b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassInfo.java @@ -20,7 +20,8 @@ package org.apache.fury.resolver; import static org.apache.fury.meta.Encoders.GENERIC_ENCODER; -import static org.apache.fury.meta.Encoders.PACKAGE_ENCODER; +import static org.apache.fury.meta.Encoders.PACKAGE_DECODER; +import static org.apache.fury.meta.Encoders.TYPE_NAME_DECODER; import org.apache.fury.collection.Tuple2; import org.apache.fury.config.Language; @@ -42,7 +43,7 @@ public class ClassInfo { final MetaStringBytes packageNameBytes; final MetaStringBytes classNameBytes; final boolean isDynamicGeneratedClass; - final MetaStringBytes typeTagBytes; + int xtypeId; Serializer serializer; // use primitive to avoid boxing // class id must be less than Integer.MAX_VALUE/2 since we use bit 0 as class id flag. @@ -56,15 +57,15 @@ public class ClassInfo { MetaStringBytes packageNameBytes, MetaStringBytes classNameBytes, boolean isDynamicGeneratedClass, - MetaStringBytes typeTagBytes, Serializer serializer, - short classId) { + short classId, + short xtypeId) { this.cls = cls; this.fullClassNameBytes = fullClassNameBytes; this.packageNameBytes = packageNameBytes; this.classNameBytes = classNameBytes; this.isDynamicGeneratedClass = isDynamicGeneratedClass; - this.typeTagBytes = typeTagBytes; + this.xtypeId = xtypeId; this.serializer = serializer; this.classId = classId; if (cls != null && classId == ClassResolver.NO_CLASS_ID) { @@ -75,9 +76,9 @@ public class ClassInfo { ClassInfo( ClassResolver classResolver, Class cls, - String tag, Serializer serializer, - short classId) { + short classId, + short xtypeId) { this.cls = cls; this.serializer = serializer; needToWriteClassDef = serializer != null && classResolver.needToWriteClassDef(serializer); @@ -104,13 +105,7 @@ public class ClassInfo { this.packageNameBytes = null; this.classNameBytes = null; } - if (tag != null) { - this.typeTagBytes = - metaStringResolver.getOrCreateMetaStringBytes( - PACKAGE_ENCODER.encode(tag, Encoding.UTF_8)); - } else { - this.typeTagBytes = null; - } + this.xtypeId = xtypeId; this.classId = classId; if (cls != null) { boolean isLambda = Functions.isLambda(cls); @@ -135,12 +130,8 @@ public short getClassId() { return classId; } - public MetaStringBytes getPackageNameBytes() { - return packageNameBytes; - } - - public MetaStringBytes getClassNameBytes() { - return classNameBytes; + public int getXtypeId() { + return xtypeId; } @SuppressWarnings("unchecked") @@ -153,6 +144,15 @@ void setSerializer(ClassResolver resolver, Serializer serializer) { needToWriteClassDef = serializer != null && resolver.needToWriteClassDef(serializer); } + + public String decodeNamespace() { + return packageNameBytes.decode(PACKAGE_DECODER); + } + + public String decodeTypeName() { + return classNameBytes.decode(TYPE_NAME_DECODER); + } + @Override public String toString() { return "ClassInfo{" diff --git a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassNameBytes.java b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassNameBytes.java new file mode 100644 index 0000000000..ab2c01dd8a --- /dev/null +++ b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassNameBytes.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fury.resolver; + +class ClassNameBytes { + private final long packageHash; + private final long classNameHash; + + ClassNameBytes(long packageHash, long classNameHash) { + this.packageHash = packageHash; + this.classNameHash = classNameHash; + } + + @Override + public boolean equals(Object o) { + // ClassNameBytes is used internally, skip + ClassNameBytes that = (ClassNameBytes) o; + return packageHash == that.packageHash && classNameHash == that.classNameHash; + } + + @Override + public int hashCode() { + int result = 31 + (int) (packageHash ^ (packageHash >>> 32)); + result = result * 31 + (int) (classNameHash ^ (classNameHash >>> 32)); + return result; + } +} diff --git a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java index dbaf4bb403..a268201d50 100644 --- a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java +++ b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java @@ -19,6 +19,7 @@ package org.apache.fury.resolver; +import static org.apache.fury.Fury.NOT_SUPPORT_XLANG; import static org.apache.fury.meta.ClassDef.SIZE_TWO_BYTES_FLAG; import static org.apache.fury.meta.Encoders.PACKAGE_DECODER; import static org.apache.fury.meta.Encoders.PACKAGE_ENCODER; @@ -205,14 +206,13 @@ public class ClassResolver { public static final short CLASS_CLASS_ID = (short) (PRIMITIVE_DOUBLE_ARRAY_CLASS_ID + 6); public static final short EMPTY_OBJECT_ID = (short) (PRIMITIVE_DOUBLE_ARRAY_CLASS_ID + 7); // use a lower load factor to minimize hash collision - private static final float loadFactor = 0.25f; private static final float furyMapLoadFactor = 0.25f; private static final int estimatedNumRegistered = 150; private static final String SET_META__CONTEXT_MSG = "Meta context must be set before serialization, " + "please set meta context by SerializationContext.setMetaContext"; - private static final ClassInfo NIL_CLASS_INFO = - new ClassInfo(null, null, null, null, false, null, null, ClassResolver.NO_CLASS_ID); + static final ClassInfo NIL_CLASS_INFO = + new ClassInfo(null, null, null, null, false, null, NO_CLASS_ID, NOT_SUPPORT_XLANG); private final Fury fury; private ClassInfo[] registeredId2ClassInfo = new ClassInfo[] {}; @@ -221,13 +221,9 @@ public class ClassResolver { private final IdentityMap, ClassInfo> classInfoMap = new IdentityMap<>(estimatedNumRegistered, furyMapLoadFactor); private ClassInfo classInfoCache; - private final ObjectMap> classNameBytes2Class = - new ObjectMap<>(16, furyMapLoadFactor); // Every deserialization for unregistered class will query it, performance is important. private final ObjectMap compositeClassNameBytes2ClassInfo = new ObjectMap<>(16, furyMapLoadFactor); - private final HashMap> typeIdToClassXLangMap = new HashMap<>(8, loadFactor); - private final HashMap> typeTagToClassXLangMap = new HashMap<>(8, loadFactor); private final MetaStringResolver metaStringResolver; private final boolean metaContextShareEnabled; private final Map, ClassDef> classDefMap = new HashMap<>(); @@ -436,8 +432,7 @@ public void register(Class cls, String typeTag) { + "Fury#register(Class) or Fury.register(Class, Short)"); } register(cls); - Preconditions.checkArgument(!typeTagToClassXLangMap.containsKey(typeTag)); - addSerializer(cls, new StructSerializer<>(fury, cls, typeTag)); + addSerializer(cls, new StructSerializer<>(fury, cls)); } /** @@ -476,7 +471,7 @@ public void register(Class cls, int classId) { if (classInfo != null) { classInfo.classId = id; } else { - classInfo = new ClassInfo(this, cls, null, null, id); + classInfo = new ClassInfo(this, cls, null, id, NOT_SUPPORT_XLANG); // make `extRegistry.registeredClassIdMap` and `classInfoMap` share same classInfo // instances. classInfoMap.put(cls, classInfo); @@ -491,7 +486,7 @@ public void register(String className, int classId) { register(loadClass(className, false, 0, false), classId); } - public void register(Class cls, Short id, boolean createSerializer) { + public void register(Class cls, int id, boolean createSerializer) { register(cls, id); if (createSerializer) { createSerializerAhead(cls); @@ -729,18 +724,6 @@ public void clearSerializer(Class cls) { /** Ass serializer for specified class. */ private void addSerializer(Class type, Serializer serializer) { Preconditions.checkNotNull(serializer); - String typeTag = null; - short typeId = serializer.getXtypeId(); - if (typeId != Fury.NOT_SUPPORT_CROSS_LANGUAGE) { - if (typeId > Fury.NOT_SUPPORT_CROSS_LANGUAGE) { - typeIdToClassXLangMap.put(typeId, type); - } - if (typeId == Fury.FURY_TYPE_TAG_ID) { - typeTag = serializer.getCrossLanguageTypeTag(); - typeTagToClassXLangMap.put(typeTag, type); - } - } - // 1. Try to get ClassInfo from `registeredId2ClassInfo` and // `classInfoMap` or create a new `ClassInfo`. ClassInfo classInfo; @@ -758,8 +741,8 @@ private void addSerializer(Class type, Serializer serializer) { classInfo = classInfoMap.get(type); } - if (classInfo == null || typeTag != null || classId != classInfo.classId) { - classInfo = new ClassInfo(this, type, typeTag, null, classId); + if (classInfo == null || classId != classInfo.classId) { + classInfo = new ClassInfo(this, type, null, classId, (short) 0); classInfoMap.put(type, classInfo); if (registered) { registeredId2ClassInfo[classId] = classInfo; @@ -968,6 +951,21 @@ public boolean isCollection(Class cls) { } } + public boolean isSet(Class cls) { + if (Set.class.isAssignableFrom(cls)) { + return true; + } + if (fury.getConfig().isScalaOptimizationEnabled()) { + // Scala map is scala iterable too. + if (ScalaTypes.getScalaMapType().isAssignableFrom(cls)) { + return false; + } + return ScalaTypes.getScalaSetType().isAssignableFrom(cls); + } else { + return false; + } + } + public boolean isMap(Class cls) { return Map.class.isAssignableFrom(cls) || (fury.getConfig().isScalaOptimizationEnabled() @@ -1150,6 +1148,10 @@ public ClassInfo getClassInfo(Class cls, boolean createClassInfoIfNotFound) { } } + void setClassInfo(Class cls, ClassInfo classInfo) { + classInfoMap.put(cls, classInfo); + } + @Internal public ClassInfo getOrUpdateClassInfo(Class cls) { ClassInfo classInfo = classInfoCache; @@ -1435,7 +1437,7 @@ private ClassInfo getMetaSharedClassInfo(ClassDef classDef, Class clz) { Class cls = clz; Short classId = extRegistry.registeredClassIdMap.get(cls); ClassInfo classInfo = - new ClassInfo(this, cls, null, null, classId == null ? NO_CLASS_ID : classId); + new ClassInfo(this, cls, null, classId == null ? NO_CLASS_ID : classId, NOT_SUPPORT_XLANG); if (NonexistentClass.class.isAssignableFrom(TypeUtils.getComponentIfArray(cls))) { if (cls == NonexistentMetaShared.class) { classInfo.setSerializer(this, new NonexistentClassSerializer(fury, classDef)); @@ -1603,7 +1605,9 @@ public void writeClassInternal(MemoryBuffer buffer, Class cls) { Short classId = extRegistry.registeredClassIdMap.get(cls); // Don't create serializer in case the object for class is non-serializable, // Or class is abstract or interface. - classInfo = new ClassInfo(this, cls, null, null, classId == null ? NO_CLASS_ID : classId); + classInfo = + new ClassInfo( + this, cls, null, classId == null ? NO_CLASS_ID : classId, NOT_SUPPORT_XLANG); classInfoMap.put(cls, classInfo); } writeClassInternal(buffer, classInfo); @@ -1745,7 +1749,7 @@ private ClassInfo readClassInfoFromBytes( return classInfo; } - private ClassInfo loadBytesToClassInfo( + ClassInfo loadBytesToClassInfo( MetaStringBytes packageBytes, MetaStringBytes simpleClassNameBytes) { ClassNameBytes classNameBytes = new ClassNameBytes(packageBytes.hashCode, simpleClassNameBytes.hashCode); @@ -1775,8 +1779,8 @@ private ClassInfo populateBytesToClassInfo( simpleClassNameBytes, false, null, - null, - NO_CLASS_ID); + NO_CLASS_ID, + NOT_SUPPORT_XLANG); if (NonexistentClass.class.isAssignableFrom(TypeUtils.getComponentIfArray(cls))) { classInfo.serializer = NonexistentClassSerializers.getSerializer(fury, classSpec.entireClassName, cls); @@ -1791,39 +1795,10 @@ private ClassInfo populateBytesToClassInfo( return classInfo; } - public void xwriteClass(MemoryBuffer buffer, Class cls) { - metaStringResolver.writeMetaStringBytes(buffer, getOrUpdateClassInfo(cls).fullClassNameBytes); - } - - public void xwriteTypeTag(MemoryBuffer buffer, Class cls) { - metaStringResolver.writeMetaStringBytes(buffer, getOrUpdateClassInfo(cls).typeTagBytes); - } - - public Class xreadClass(MemoryBuffer buffer) { - MetaStringBytes byteString = metaStringResolver.readMetaStringBytes(buffer); - Class cls = classNameBytes2Class.get(byteString); - if (cls == null) { - Preconditions.checkNotNull(byteString); - String className = byteString.decode(Encoders.GENERIC_DECODER); - cls = loadClass(className); - classNameBytes2Class.put(byteString, cls); - } - currentReadClass = cls; - return cls; - } - - public String xreadClassName(MemoryBuffer buffer) { - return metaStringResolver.readMetaString(buffer); - } - public Class getCurrentReadClass() { return currentReadClass; } - private Class loadClass(String className) { - return loadClass(className, false, 0); - } - private Class loadClass(ClassSpec classSpec) { return loadClass(classSpec.entireClassName, classSpec.isEnum, classSpec.dimension); } @@ -1864,39 +1839,6 @@ public void resetRead() {} public void resetWrite() {} - public Class getClassByTypeId(short typeId) { - return typeIdToClassXLangMap.get(typeId); - } - - public Class readClassByTypeTag(MemoryBuffer buffer) { - String tag = metaStringResolver.readMetaString(buffer); - return typeTagToClassXLangMap.get(tag); - } - - private static class ClassNameBytes { - private final long packageHash; - private final long classNameHash; - - private ClassNameBytes(long packageHash, long classNameHash) { - this.packageHash = packageHash; - this.classNameHash = classNameHash; - } - - @Override - public boolean equals(Object o) { - // ClassNameBytes is used internally, skip - ClassNameBytes that = (ClassNameBytes) o; - return packageHash == that.packageHash && classNameHash == that.classNameHash; - } - - @Override - public int hashCode() { - int result = 31 + (int) (packageHash ^ (packageHash >>> 32)); - result = result * 31 + (int) (classNameHash ^ (classNameHash >>> 32)); - return result; - } - } - public GenericType buildGenericType(TypeRef typeRef) { return GenericType.build( typeRef.getType(), @@ -1926,12 +1868,12 @@ public GenericType getObjectGenericType() { } public ClassInfo newClassInfo(Class cls, Serializer serializer, short classId) { - return new ClassInfo(this, cls, null, serializer, classId); + return new ClassInfo(this, cls, serializer, classId, NOT_SUPPORT_XLANG); } // Invoked by fury JIT. public ClassInfo nilClassInfo() { - return new ClassInfo(this, null, null, null, NO_CLASS_ID); + return new ClassInfo(this, null, null, NO_CLASS_ID, NOT_SUPPORT_XLANG); } public ClassInfoHolder nilClassInfoHolder() { diff --git a/java/fury-core/src/main/java/org/apache/fury/resolver/MetaStringResolver.java b/java/fury-core/src/main/java/org/apache/fury/resolver/MetaStringResolver.java index af24ce8e37..7ee442d323 100644 --- a/java/fury-core/src/main/java/org/apache/fury/resolver/MetaStringResolver.java +++ b/java/fury-core/src/main/java/org/apache/fury/resolver/MetaStringResolver.java @@ -82,6 +82,7 @@ public void writeMetaStringBytesWithFlag(MemoryBuffer buffer, MetaStringBytes by } dynamicWrittenMetaString[id] = byteString; int length = byteString.bytes.length; + // last bit `1` indicates class is written by name instead of registered id. buffer.writeVarUint32Small7(length << 2 | 0b1); if (length > SMALL_STRING_THRESHOLD) { buffer.writeInt64(byteString.hashCode); @@ -90,6 +91,7 @@ public void writeMetaStringBytesWithFlag(MemoryBuffer buffer, MetaStringBytes by } buffer.writeBytes(byteString.bytes); } else { + // last bit `1` indicates class is written by name instead of registered id. buffer.writeVarUint32Small7(((id + 1) << 2) | 0b11); } } diff --git a/java/fury-core/src/main/java/org/apache/fury/resolver/XtypeResolver.java b/java/fury-core/src/main/java/org/apache/fury/resolver/XtypeResolver.java new file mode 100644 index 0000000000..5771cf8bcf --- /dev/null +++ b/java/fury-core/src/main/java/org/apache/fury/resolver/XtypeResolver.java @@ -0,0 +1,434 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fury.resolver; + +import static org.apache.fury.Fury.NOT_SUPPORT_XLANG; +import static org.apache.fury.meta.Encoders.GENERIC_ENCODER; +import static org.apache.fury.meta.Encoders.PACKAGE_DECODER; +import static org.apache.fury.meta.Encoders.PACKAGE_ENCODER; +import static org.apache.fury.meta.Encoders.TYPE_NAME_DECODER; +import static org.apache.fury.resolver.ClassResolver.NO_CLASS_ID; +import static org.apache.fury.type.TypeUtils.qualifiedName; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.sql.Timestamp; +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.fury.Fury; +import org.apache.fury.collection.IdentityMap; +import org.apache.fury.collection.LongMap; +import org.apache.fury.collection.ObjectMap; +import org.apache.fury.config.Config; +import org.apache.fury.exception.ClassUnregisteredException; +import org.apache.fury.exception.SerializerUnregisteredException; +import org.apache.fury.logging.Logger; +import org.apache.fury.logging.LoggerFactory; +import org.apache.fury.memory.MemoryBuffer; +import org.apache.fury.memory.Platform; +import org.apache.fury.meta.Encoders; +import org.apache.fury.meta.MetaString; +import org.apache.fury.reflect.ReflectionUtils; +import org.apache.fury.serializer.ArraySerializers.ObjectArraySerializer; +import org.apache.fury.serializer.EnumSerializer; +import org.apache.fury.serializer.NonexistentClass; +import org.apache.fury.serializer.NonexistentClassSerializers; +import org.apache.fury.serializer.Serializer; +import org.apache.fury.serializer.Serializers; +import org.apache.fury.serializer.StructSerializer; +import org.apache.fury.serializer.collection.CollectionSerializer; +import org.apache.fury.serializer.collection.MapSerializer; +import org.apache.fury.type.TypeUtils; +import org.apache.fury.type.Types; +import org.apache.fury.util.Preconditions; + +@SuppressWarnings({"unchecked", "rawtypes"}) +// TODO(chaokunyang) Abstract type resolver for java/xlang type resolution. +public class XtypeResolver { + private static final Logger LOG = LoggerFactory.getLogger(XtypeResolver.class); + + private static final float loadFactor = 0.5f; + // Most systems won't have so many types for serialization. + private static final int MAX_TYPE_ID = 4096; + + private final Config config; + private final Fury fury; + private final ClassResolver classResolver; + private final ClassInfoHolder classInfoCache = new ClassInfoHolder(ClassResolver.NIL_CLASS_INFO); + private final MetaStringResolver metaStringResolver; + // IdentityMap has better lookup performance, when loadFactor is 0.05f, performance is better + private final IdentityMap, ClassInfo> classInfoMap = new IdentityMap<>(64, loadFactor); + // Every deserialization for unregistered class will query it, performance is important. + private final ObjectMap compositeClassNameBytes2ClassInfo = + new ObjectMap<>(16, loadFactor); + private final ObjectMap qualifiedType2ClassInfo = + new ObjectMap<>(16, loadFactor); + private int xtypeIdGenerator = 64; + + // Use ClassInfo[] or LongMap? + // ClassInfo[] is faster, but we can't have bigger type id. + private final LongMap xtypeIdToClassMap = new LongMap<>(8, loadFactor); + private final Set registeredTypeIds = new HashSet<>(); + + public XtypeResolver(Fury fury) { + this.config = fury.getConfig(); + this.fury = fury; + this.classResolver = fury.getClassResolver(); + this.metaStringResolver = fury.getMetaStringResolver(); + registerDefaultTypes(); + } + + public void register(Class type) { + while (registeredTypeIds.contains(xtypeIdGenerator)) { + xtypeIdGenerator++; + } + register(type, xtypeIdGenerator++); + } + + public void register(Class type, int typeId) { + // ClassInfo[] has length of max type id. If the type id is too big, Fury will waste many + // memory. + // We can relax this limit in the future. + Preconditions.checkArgument(typeId < MAX_TYPE_ID, "Too big type id %s", typeId); + ClassInfo classInfo = classInfoMap.get(type); + Serializer serializer = null; + if (classInfo != null) { + serializer = classInfo.serializer; + if (classInfo.xtypeId != 0) { + throw new IllegalArgumentException( + String.format("Type %s has been registered with id %s", type, classInfo.xtypeId)); + } + String prevNamespace = classInfo.decodeNamespace(); + String prevTypeName = classInfo.decodeTypeName(); + if (!type.getSimpleName().equals(prevTypeName)) { + throw new IllegalArgumentException( + String.format( + "Type %s has been registered with namespace %s type %s", + type, prevNamespace, prevTypeName)); + } + } + int xtypeId = typeId; + if (type.isEnum()) { + xtypeId = xtypeId << 8 + Types.ENUM; + + } else { + if (serializer != null) { + if (serializer instanceof StructSerializer) { + xtypeId = xtypeId << 8 + Types.STRUCT; + } else { + xtypeId = xtypeId << 8 + Types.EXT; + } + } + } + register( + type, + serializer, + ReflectionUtils.getPackage(type), + ReflectionUtils.getClassNameWithoutPackage(type), + xtypeId); + } + + public void register(Class type, String namespace, String typeName) { + Preconditions.checkArgument( + !typeName.contains("."), + "Typename %s should not contains `.`, please put it into namespace", + typeName); + ClassInfo classInfo = classInfoMap.get(type); + Serializer serializer = null; + if (classInfo != null) { + serializer = classInfo.serializer; + if (classInfo.classNameBytes != null) { + String prevNamespace = classInfo.decodeNamespace(); + String prevTypeName = classInfo.decodeTypeName(); + if (!namespace.equals(prevNamespace) || typeName.equals(prevTypeName)) { + throw new IllegalArgumentException( + String.format( + "Type %s has been registered with namespace %s type %s", + type, prevNamespace, prevTypeName)); + } + } + } + short xtypeId; + if (serializer != null) { + if (serializer instanceof StructSerializer) { + xtypeId = Types.NS_STRUCT; + } else if (serializer instanceof EnumSerializer) { + xtypeId = Types.NS_ENUM; + } else { + xtypeId = Types.NS_EXT; + } + } else { + if (type.isEnum()) { + xtypeId = Types.NS_ENUM; + } else { + xtypeId = Types.NS_STRUCT; + } + } + register(type, serializer, namespace, typeName, xtypeId); + } + + private void register( + Class type, Serializer serializer, String namespace, String typeName, int xtypeId) { + ClassInfo classInfo = newClassInfo(type, serializer, namespace, typeName, (short) xtypeId); + qualifiedType2ClassInfo.put(qualifiedName(namespace, typeName), classInfo); + if (serializer == null) { + if (type.isEnum()) { + classInfo.serializer = new EnumSerializer(fury, (Class) type); + } else { + classInfo.serializer = new StructSerializer(fury, type); + } + } + classInfoMap.put(type, classInfo); + registeredTypeIds.add(xtypeId); + xtypeIdToClassMap.put(xtypeId, classInfo); + } + + private ClassInfo newClassInfo(Class type, Serializer serializer, short xtypeId) { + return newClassInfo( + type, + serializer, + ReflectionUtils.getPackage(type), + ReflectionUtils.getClassNameWithoutPackage(type), + xtypeId); + } + + private ClassInfo newClassInfo( + Class type, Serializer serializer, String namespace, String typeName, short xtypeId) { + MetaStringBytes fullClassNameBytes = + metaStringResolver.getOrCreateMetaStringBytes( + GENERIC_ENCODER.encode(type.getName(), MetaString.Encoding.UTF_8)); + MetaStringBytes nsBytes = + metaStringResolver.getOrCreateMetaStringBytes(Encoders.encodePackage(namespace)); + MetaStringBytes classNameBytes = + metaStringResolver.getOrCreateMetaStringBytes(Encoders.encodeTypeName(typeName)); + return new ClassInfo( + type, fullClassNameBytes, nsBytes, classNameBytes, false, serializer, NO_CLASS_ID, xtypeId); + } + + public void registerSerializer(Class type, Class serializerClass) { + ClassInfo classInfo = checkClassRegistration(type); + classInfo.serializer = Serializers.newSerializer(fury, type, serializerClass); + } + + public void registerSerializer(Class type, Serializer serializer) { + ClassInfo classInfo = checkClassRegistration(type); + classInfo.serializer = serializer; + } + + private ClassInfo checkClassRegistration(Class type) { + ClassInfo classInfo = classInfoMap.get(type); + Preconditions.checkArgument( + classInfo != null + && (classInfo.xtypeId != 0 + || !type.getSimpleName().equals(classInfo.decodeTypeName())), + "Type %s should be registered with id or namespace+typename before register serializer", + type); + return classInfo; + } + + public ClassInfo getClassInfo(Class cls, ClassInfoHolder classInfoHolder) { + ClassInfo classInfo = classInfoHolder.classInfo; + if (classInfo.getCls() != cls) { + classInfo = classInfoMap.get(cls); + if (classInfo == null) { + classInfo = buildClassInfo(cls); + } + classInfoHolder.classInfo = classInfo; + } + assert classInfo.serializer != null; + return classInfo; + } + + private ClassInfo buildClassInfo(Class cls) { + Serializer serializer; + int xtypeId; + if (classResolver.isSet(cls)) { + serializer = new CollectionSerializer(fury, cls); + xtypeId = Types.SET; + } else if (classResolver.isCollection(cls)) { + serializer = new CollectionSerializer(fury, cls); + xtypeId = Types.LIST; + } else if (cls.isArray() && !TypeUtils.getArrayComponent(cls).isPrimitive()) { + serializer = new ObjectArraySerializer(fury, cls); + xtypeId = Types.LIST; + } else if (classResolver.isMap(cls)) { + serializer = new MapSerializer(fury, cls); + xtypeId = Types.MAP; + } else { + throw new ClassUnregisteredException(cls); + } + ClassInfo info = newClassInfo(cls, serializer, (short) xtypeId); + classInfoMap.put(cls, info); + return info; + } + + private void registerDefaultTypes() { + registerDefaultTypes(Types.BOOL, Boolean.class, boolean.class, AtomicBoolean.class); + registerDefaultTypes(Types.INT8, Byte.class, byte.class); + registerDefaultTypes(Types.INT16, Short.class, short.class); + registerDefaultTypes(Types.INT32, Integer.class, int.class, AtomicInteger.class); + registerDefaultTypes(Types.INT64, Long.class, long.class, AtomicLong.class); + registerDefaultTypes(Types.FLOAT32, Float.class, float.class); + registerDefaultTypes(Types.FLOAT64, Double.class, double.class); + registerDefaultTypes(Types.STRING, String.class); + registerDefaultTypes(Types.DURATION, Duration.class); + registerDefaultTypes( + Types.TIMESTAMP, Instant.class, Date.class, Timestamp.class, LocalDateTime.class); + registerDefaultTypes(Types.DECIMAL, BigDecimal.class, BigInteger.class); + registerDefaultTypes( + Types.BINARY, + byte[].class, + Platform.HEAP_BYTE_BUFFER_CLASS, + Platform.DIRECT_BYTE_BUFFER_CLASS); + registerDefaultTypes(Types.BOOL_ARRAY, boolean[].class); + registerDefaultTypes(Types.INT16_ARRAY, short[].class); + registerDefaultTypes(Types.INT32_ARRAY, int[].class); + registerDefaultTypes(Types.INT64_ARRAY, long[].class); + registerDefaultTypes(Types.FLOAT32_ARRAY, float[].class); + registerDefaultTypes(Types.FLOAT64_ARRAY, double[].class); + registerDefaultTypes(Types.LIST, ArrayList.class, Object[].class); + registerDefaultTypes(Types.SET, HashSet.class, LinkedHashSet.class); + registerDefaultTypes(Types.MAP, HashMap.class, LinkedHashMap.class); + registerDefaultTypes(Types.LOCAL_DATE, LocalDate.class); + } + + private void registerDefaultTypes(int xtypeId, Class defaultType, Class... otherTypes) { + ClassInfo classInfo = + newClassInfo(defaultType, classResolver.getSerializer(defaultType), (short) xtypeId); + classInfoMap.put(defaultType, classInfo); + xtypeIdToClassMap.put(xtypeId, classInfo); + for (Class otherType : otherTypes) { + classInfo = newClassInfo(otherType, classResolver.getSerializer(otherType), (short) xtypeId); + classInfoMap.put(otherType, classInfo); + } + } + + public ClassInfo writeClassInfo(MemoryBuffer buffer, Object obj) { + ClassInfo classInfo = getClassInfo(obj.getClass(), classInfoCache); + int xtypeId = classInfo.getXtypeId(); + byte internalTypeId = (byte) xtypeId; + buffer.writeVarUint32Small7(xtypeId); + switch (internalTypeId) { + case Types.NS_ENUM: + case Types.NS_STRUCT: + case Types.NS_EXT: + assert classInfo.packageNameBytes != null; + metaStringResolver.writeMetaStringBytes(buffer, classInfo.packageNameBytes); + assert classInfo.classNameBytes != null; + metaStringResolver.writeMetaStringBytes(buffer, classInfo.classNameBytes); + break; + } + return classInfo; + } + + public ClassInfo readClassInfo(MemoryBuffer buffer) { + long xtypeId = buffer.readVarUint32Small14(); + byte internalTypeId = (byte) xtypeId; + switch (internalTypeId) { + case Types.NS_ENUM: + case Types.NS_STRUCT: + case Types.NS_COMPATIBLE_STRUCT: + case Types.NS_EXT: + MetaStringBytes packageBytes = metaStringResolver.readMetaStringBytes(buffer); + MetaStringBytes simpleClassNameBytes = metaStringResolver.readMetaStringBytes(buffer); + return loadBytesToClassInfo(internalTypeId, packageBytes, simpleClassNameBytes); + default: + return xtypeIdToClassMap.get(xtypeId); + } + } + + private ClassInfo loadBytesToClassInfo( + int internalTypeId, MetaStringBytes packageBytes, MetaStringBytes simpleClassNameBytes) { + ClassNameBytes classNameBytes = + new ClassNameBytes(packageBytes.hashCode, simpleClassNameBytes.hashCode); + ClassInfo classInfo = compositeClassNameBytes2ClassInfo.get(classNameBytes); + if (classInfo == null) { + classInfo = + populateBytesToClassInfo( + internalTypeId, classNameBytes, packageBytes, simpleClassNameBytes); + } + return classInfo; + } + + private ClassInfo populateBytesToClassInfo( + int typeId, + ClassNameBytes classNameBytes, + MetaStringBytes packageBytes, + MetaStringBytes simpleClassNameBytes) { + String namespace = packageBytes.decode(PACKAGE_DECODER); + String typeName = simpleClassNameBytes.decode(TYPE_NAME_DECODER); + String qualifiedName = qualifiedName(namespace, typeName); + ClassInfo classInfo = qualifiedType2ClassInfo.get(qualifiedName); + if (classInfo == null) { + String msg = String.format("Class %s not registered", qualifiedName); + Class type = null; + if (config.deserializeNonexistentClass()) { + LOG.warn(msg); + switch (typeId) { + case Types.NS_ENUM: + case Types.NS_STRUCT: + case Types.NS_COMPATIBLE_STRUCT: + type = + NonexistentClass.getNonexistentClass( + qualifiedName, isEnum(typeId), 0, config.isMetaShareEnabled()); + break; + case Types.NS_EXT: + throw new SerializerUnregisteredException(qualifiedName); + } + } else { + throw new ClassUnregisteredException(qualifiedName); + } + MetaStringBytes fullClassNameBytes = + metaStringResolver.getOrCreateMetaStringBytes( + PACKAGE_ENCODER.encode(qualifiedName, MetaString.Encoding.UTF_8)); + classInfo = + new ClassInfo( + type, + fullClassNameBytes, + packageBytes, + simpleClassNameBytes, + false, + null, + NO_CLASS_ID, + NOT_SUPPORT_XLANG); + if (NonexistentClass.class.isAssignableFrom(TypeUtils.getComponentIfArray(type))) { + classInfo.serializer = NonexistentClassSerializers.getSerializer(fury, qualifiedName, type); + } + } + compositeClassNameBytes2ClassInfo.put(classNameBytes, classInfo); + return classInfo; + } + + private boolean isEnum(int internalTypeId) { + return internalTypeId == Types.ENUM || internalTypeId == Types.NS_ENUM; + } +} diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/ArraySerializers.java b/java/fury-core/src/main/java/org/apache/fury/serializer/ArraySerializers.java index c38b2bae96..53ca4d3515 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/ArraySerializers.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/ArraySerializers.java @@ -32,8 +32,8 @@ import org.apache.fury.resolver.RefResolver; import org.apache.fury.serializer.collection.CollectionFlags; import org.apache.fury.serializer.collection.FuryArrayAsListSerializer; -import org.apache.fury.type.Type; import org.apache.fury.type.TypeUtils; +import org.apache.fury.type.Types; import org.apache.fury.util.Preconditions; /** Serializers for array types. */ @@ -73,11 +73,6 @@ public ObjectArraySerializer(Fury fury, Class cls) { classInfoHolder = fury.getClassResolver().nilClassInfoHolder(); } - @Override - public short getXtypeId() { - return (short) -Type.LIST.getId(); - } - @Override public void write(MemoryBuffer buffer, T[] arr) { int len = arr.length; @@ -249,7 +244,7 @@ public abstract static class PrimitiveArraySerializer protected final int elemSize; public PrimitiveArraySerializer(Fury fury, Class cls) { - super(fury, cls, (short) primitiveInfo.get(TypeUtils.getArrayComponentInfo(cls).f0)[2]); + super(fury, cls); Class innerType = TypeUtils.getArrayComponentInfo(cls).f0; this.offset = primitiveInfo.get(innerType)[0]; this.elemSize = primitiveInfo.get(innerType)[1]; @@ -386,11 +381,6 @@ public char[] read(MemoryBuffer buffer) { } } - @Override - public short getXtypeId() { - return Fury.NOT_SUPPORT_CROSS_LANGUAGE; - } - @Override public void xwrite(MemoryBuffer buffer, char[] value) { throw new UnsupportedOperationException(); @@ -620,11 +610,6 @@ public StringArraySerializer(Fury fury) { list = new FuryArrayAsListSerializer.ArrayAsList(0); } - @Override - public short getXtypeId() { - return (short) -Type.FURY_STRING_ARRAY.getId(); - } - @Override public void write(MemoryBuffer buffer, String[] value) { int len = value.length; @@ -714,26 +699,29 @@ public String[] xread(MemoryBuffer buffer) { } public static void registerDefaultSerializers(Fury fury) { - fury.registerSerializer(Object[].class, new ObjectArraySerializer<>(fury, Object[].class)); - fury.registerSerializer(Class[].class, new ObjectArraySerializer<>(fury, Class[].class)); - fury.registerSerializer(byte[].class, new ByteArraySerializer(fury)); - fury.registerSerializer(Byte[].class, new ObjectArraySerializer<>(fury, Byte[].class)); - fury.registerSerializer(char[].class, new CharArraySerializer(fury)); - fury.registerSerializer( + ClassResolver resolver = fury.getClassResolver(); + resolver.registerSerializer(Object[].class, new ObjectArraySerializer<>(fury, Object[].class)); + resolver.registerSerializer(Class[].class, new ObjectArraySerializer<>(fury, Class[].class)); + resolver.registerSerializer(byte[].class, new ByteArraySerializer(fury)); + resolver.registerSerializer(Byte[].class, new ObjectArraySerializer<>(fury, Byte[].class)); + resolver.registerSerializer(char[].class, new CharArraySerializer(fury)); + resolver.registerSerializer( Character[].class, new ObjectArraySerializer<>(fury, Character[].class)); - fury.registerSerializer(short[].class, new ShortArraySerializer(fury)); - fury.registerSerializer(Short[].class, new ObjectArraySerializer<>(fury, Short[].class)); - fury.registerSerializer(int[].class, new IntArraySerializer(fury)); - fury.registerSerializer(Integer[].class, new ObjectArraySerializer<>(fury, Integer[].class)); - fury.registerSerializer(long[].class, new LongArraySerializer(fury)); - fury.registerSerializer(Long[].class, new ObjectArraySerializer<>(fury, Long[].class)); - fury.registerSerializer(float[].class, new FloatArraySerializer(fury)); - fury.registerSerializer(Float[].class, new ObjectArraySerializer<>(fury, Float[].class)); - fury.registerSerializer(double[].class, new DoubleArraySerializer(fury)); - fury.registerSerializer(Double[].class, new ObjectArraySerializer<>(fury, Double[].class)); - fury.registerSerializer(boolean[].class, new BooleanArraySerializer(fury)); - fury.registerSerializer(Boolean[].class, new ObjectArraySerializer<>(fury, Boolean[].class)); - fury.registerSerializer(String[].class, new StringArraySerializer(fury)); + resolver.registerSerializer(short[].class, new ShortArraySerializer(fury)); + resolver.registerSerializer(Short[].class, new ObjectArraySerializer<>(fury, Short[].class)); + resolver.registerSerializer(int[].class, new IntArraySerializer(fury)); + resolver.registerSerializer( + Integer[].class, new ObjectArraySerializer<>(fury, Integer[].class)); + resolver.registerSerializer(long[].class, new LongArraySerializer(fury)); + resolver.registerSerializer(Long[].class, new ObjectArraySerializer<>(fury, Long[].class)); + resolver.registerSerializer(float[].class, new FloatArraySerializer(fury)); + resolver.registerSerializer(Float[].class, new ObjectArraySerializer<>(fury, Float[].class)); + resolver.registerSerializer(double[].class, new DoubleArraySerializer(fury)); + resolver.registerSerializer(Double[].class, new ObjectArraySerializer<>(fury, Double[].class)); + resolver.registerSerializer(boolean[].class, new BooleanArraySerializer(fury)); + resolver.registerSerializer( + Boolean[].class, new ObjectArraySerializer<>(fury, Boolean[].class)); + resolver.registerSerializer(String[].class, new StringArraySerializer(fury)); } // ########################## utils ########################## @@ -757,25 +745,16 @@ public static PrimitiveArrayBufferObject byteArrayBufferObject(byte[] array) { static { primitiveInfo.put( - boolean.class, - new int[] {Platform.BOOLEAN_ARRAY_OFFSET, 1, Type.FURY_PRIMITIVE_BOOL_ARRAY.getId()}); - primitiveInfo.put(byte.class, new int[] {Platform.BYTE_ARRAY_OFFSET, 1, Type.BINARY.getId()}); - primitiveInfo.put( - char.class, new int[] {Platform.CHAR_ARRAY_OFFSET, 2, Fury.NOT_SUPPORT_CROSS_LANGUAGE}); - primitiveInfo.put( - short.class, - new int[] {Platform.SHORT_ARRAY_OFFSET, 2, Type.FURY_PRIMITIVE_SHORT_ARRAY.getId()}); - primitiveInfo.put( - int.class, new int[] {Platform.INT_ARRAY_OFFSET, 4, Type.FURY_PRIMITIVE_INT_ARRAY.getId()}); - primitiveInfo.put( - long.class, - new int[] {Platform.LONG_ARRAY_OFFSET, 8, Type.FURY_PRIMITIVE_LONG_ARRAY.getId()}); + boolean.class, new int[] {Platform.BOOLEAN_ARRAY_OFFSET, 1, Types.BOOL_ARRAY}); + primitiveInfo.put(byte.class, new int[] {Platform.BYTE_ARRAY_OFFSET, 1, Types.BINARY}); primitiveInfo.put( - float.class, - new int[] {Platform.FLOAT_ARRAY_OFFSET, 4, Type.FURY_PRIMITIVE_FLOAT_ARRAY.getId()}); + char.class, new int[] {Platform.CHAR_ARRAY_OFFSET, 2, Fury.NOT_SUPPORT_XLANG}); + primitiveInfo.put(short.class, new int[] {Platform.SHORT_ARRAY_OFFSET, 2, Types.INT16_ARRAY}); + primitiveInfo.put(int.class, new int[] {Platform.INT_ARRAY_OFFSET, 4, Types.INT32_ARRAY}); + primitiveInfo.put(long.class, new int[] {Platform.LONG_ARRAY_OFFSET, 8, Types.INT64_ARRAY}); + primitiveInfo.put(float.class, new int[] {Platform.FLOAT_ARRAY_OFFSET, 4, Types.FLOAT32_ARRAY}); primitiveInfo.put( - double.class, - new int[] {Platform.DOUBLE_ARRAY_OFFSET, 8, Type.FURY_PRIMITIVE_DOUBLE_ARRAY.getId()}); + double.class, new int[] {Platform.DOUBLE_ARRAY_OFFSET, 8, Types.FLOAT64_ARRAY}); } public abstract static class AbstractedNonexistentArrayClassSerializer extends Serializer { diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/BufferSerializers.java b/java/fury-core/src/main/java/org/apache/fury/serializer/BufferSerializers.java index 17ca6c3612..6d35e18a23 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/BufferSerializers.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/BufferSerializers.java @@ -23,7 +23,6 @@ import java.nio.ByteOrder; import org.apache.fury.Fury; import org.apache.fury.memory.MemoryBuffer; -import org.apache.fury.type.Type; /** Serializers for buffer related classes. */ public class BufferSerializers { @@ -36,7 +35,7 @@ public static final class ByteBufferSerializer extends Serializers.CrossLanguageCompatibleSerializer { public ByteBufferSerializer(Fury fury, Class cls) { - super(fury, cls, Type.FURY_BUFFER.getId()); + super(fury, cls); } @Override diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/FuryCopyableSerializer.java b/java/fury-core/src/main/java/org/apache/fury/serializer/FuryCopyableSerializer.java index a17c35be91..a720b44be0 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/FuryCopyableSerializer.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/FuryCopyableSerializer.java @@ -48,25 +48,6 @@ public T read(MemoryBuffer buffer) { return serializer.read(buffer); } - /** - * Returns {@link Fury#NOT_SUPPORT_CROSS_LANGUAGE} if the serializer doesn't support - * cross-language serialization. Return a number in range (0, 32767) if the serializer support - * cross-language serialization and native serialization data is the same with cross-language - * serialization. Return a negative short in range [-32768, 0) if the serializer support - * cross-language serialization and native serialization data is not the same with cross-language - * serialization. - */ - @Override - public short getXtypeId() { - return serializer.getXtypeId(); - } - - /** Returns a type tag used for setup type mapping between languages. */ - @Override - public String getCrossLanguageTypeTag() { - return serializer.getCrossLanguageTypeTag(); - } - @Override public void xwrite(MemoryBuffer buffer, T value) { serializer.xwrite(buffer, value); diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/OptionalSerializers.java b/java/fury-core/src/main/java/org/apache/fury/serializer/OptionalSerializers.java index 1fa48db8fd..30052288ef 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/OptionalSerializers.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/OptionalSerializers.java @@ -25,6 +25,7 @@ import java.util.OptionalLong; import org.apache.fury.Fury; import org.apache.fury.memory.MemoryBuffer; +import org.apache.fury.resolver.ClassResolver; /** * Serializers for {@link Optional}, {@link OptionalInt}, {@link OptionalLong} and {@link @@ -130,9 +131,10 @@ public OptionalDouble read(MemoryBuffer buffer) { } public static void registerDefaultSerializers(Fury fury) { - fury.registerSerializer(Optional.class, new OptionalSerializer(fury)); - fury.registerSerializer(OptionalInt.class, new OptionalIntSerializer(fury)); - fury.registerSerializer(OptionalLong.class, new OptionalLongSerializer(fury)); - fury.registerSerializer(OptionalDouble.class, new OptionalDoubleSerializer(fury)); + ClassResolver resolver = fury.getClassResolver(); + resolver.registerSerializer(Optional.class, new OptionalSerializer(fury)); + resolver.registerSerializer(OptionalInt.class, new OptionalIntSerializer(fury)); + resolver.registerSerializer(OptionalLong.class, new OptionalLongSerializer(fury)); + resolver.registerSerializer(OptionalDouble.class, new OptionalDoubleSerializer(fury)); } } diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/PrimitiveSerializers.java b/java/fury-core/src/main/java/org/apache/fury/serializer/PrimitiveSerializers.java index 1b386b8795..1a5c974142 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/PrimitiveSerializers.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/PrimitiveSerializers.java @@ -27,7 +27,7 @@ import org.apache.fury.config.LongEncoding; import org.apache.fury.memory.MemoryBuffer; import org.apache.fury.memory.Platform; -import org.apache.fury.type.Type; +import org.apache.fury.resolver.ClassResolver; import org.apache.fury.util.Preconditions; /** Serializers for java primitive types. */ @@ -36,12 +36,7 @@ public class PrimitiveSerializers { public static final class BooleanSerializer extends Serializers.CrossLanguageCompatibleSerializer { public BooleanSerializer(Fury fury, Class cls) { - super( - fury, - (Class) cls, - Type.BOOL.getId(), - !(cls.isPrimitive() || fury.isBasicTypesRefIgnored()), - true); + super(fury, (Class) cls, !(cls.isPrimitive() || fury.isBasicTypesRefIgnored()), true); } @Override @@ -58,12 +53,7 @@ public Boolean read(MemoryBuffer buffer) { public static final class ByteSerializer extends Serializers.CrossLanguageCompatibleSerializer { public ByteSerializer(Fury fury, Class cls) { - super( - fury, - (Class) cls, - Type.INT8.getId(), - !(cls.isPrimitive() || fury.isBasicTypesRefIgnored()), - true); + super(fury, (Class) cls, !(cls.isPrimitive() || fury.isBasicTypesRefIgnored()), true); } @Override @@ -82,11 +72,6 @@ public Uint8Serializer(Fury fury) { super(fury, Integer.class); } - @Override - public short getXtypeId() { - return Type.UINT8.getId(); - } - @Override public void xwrite(MemoryBuffer buffer, Integer value) { Preconditions.checkArgument(value >= 0 && value <= 255); @@ -105,11 +90,6 @@ public Uint16Serializer(Fury fury) { super(fury, Integer.class); } - @Override - public short getXtypeId() { - return Type.UINT16.getId(); - } - @Override public void xwrite(MemoryBuffer buffer, Integer value) { Preconditions.checkArgument(value >= 0 && value <= 65535); @@ -142,12 +122,7 @@ public Character read(MemoryBuffer buffer) { public static final class ShortSerializer extends Serializers.CrossLanguageCompatibleSerializer { public ShortSerializer(Fury fury, Class cls) { - super( - fury, - (Class) cls, - Type.INT16.getId(), - !(cls.isPrimitive() || fury.isBasicTypesRefIgnored()), - true); + super(fury, (Class) cls, !(cls.isPrimitive() || fury.isBasicTypesRefIgnored()), true); } @Override @@ -166,12 +141,7 @@ public static final class IntSerializer private final boolean compressNumber; public IntSerializer(Fury fury, Class cls) { - super( - fury, - (Class) cls, - Type.INT32.getId(), - !(cls.isPrimitive() || fury.isBasicTypesRefIgnored()), - true); + super(fury, (Class) cls, !(cls.isPrimitive() || fury.isBasicTypesRefIgnored()), true); compressNumber = fury.compressInt(); } @@ -210,12 +180,7 @@ public static final class LongSerializer private final LongEncoding longEncoding; public LongSerializer(Fury fury, Class cls) { - super( - fury, - (Class) cls, - Type.INT64.getId(), - !(cls.isPrimitive() || fury.isBasicTypesRefIgnored()), - true); + super(fury, (Class) cls, !(cls.isPrimitive() || fury.isBasicTypesRefIgnored()), true); longEncoding = fury.longEncoding(); } @@ -295,12 +260,7 @@ public Long xread(MemoryBuffer buffer) { public static final class FloatSerializer extends Serializers.CrossLanguageCompatibleSerializer { public FloatSerializer(Fury fury, Class cls) { - super( - fury, - (Class) cls, - Type.FLOAT.getId(), - !(cls.isPrimitive() || fury.isBasicTypesRefIgnored()), - true); + super(fury, (Class) cls, !(cls.isPrimitive() || fury.isBasicTypesRefIgnored()), true); } @Override @@ -317,12 +277,7 @@ public Float read(MemoryBuffer buffer) { public static final class DoubleSerializer extends Serializers.CrossLanguageCompatibleSerializer { public DoubleSerializer(Fury fury, Class cls) { - super( - fury, - (Class) cls, - Type.DOUBLE.getId(), - !(cls.isPrimitive() || fury.isBasicTypesRefIgnored()), - true); + super(fury, (Class) cls, !(cls.isPrimitive() || fury.isBasicTypesRefIgnored()), true); } @Override @@ -338,21 +293,22 @@ public Double read(MemoryBuffer buffer) { public static void registerDefaultSerializers(Fury fury) { // primitive types will be boxed. - fury.registerSerializer(boolean.class, new BooleanSerializer(fury, boolean.class)); - fury.registerSerializer(byte.class, new ByteSerializer(fury, byte.class)); - fury.registerSerializer(short.class, new ShortSerializer(fury, short.class)); - fury.registerSerializer(char.class, new CharSerializer(fury, char.class)); - fury.registerSerializer(int.class, new IntSerializer(fury, int.class)); - fury.registerSerializer(long.class, new LongSerializer(fury, long.class)); - fury.registerSerializer(float.class, new FloatSerializer(fury, float.class)); - fury.registerSerializer(double.class, new DoubleSerializer(fury, double.class)); - fury.registerSerializer(Boolean.class, new BooleanSerializer(fury, Boolean.class)); - fury.registerSerializer(Byte.class, new ByteSerializer(fury, Byte.class)); - fury.registerSerializer(Short.class, new ShortSerializer(fury, Short.class)); - fury.registerSerializer(Character.class, new CharSerializer(fury, Character.class)); - fury.registerSerializer(Integer.class, new IntSerializer(fury, Integer.class)); - fury.registerSerializer(Long.class, new LongSerializer(fury, Long.class)); - fury.registerSerializer(Float.class, new FloatSerializer(fury, Float.class)); - fury.registerSerializer(Double.class, new DoubleSerializer(fury, Double.class)); + ClassResolver resolver = fury.getClassResolver(); + resolver.registerSerializer(boolean.class, new BooleanSerializer(fury, boolean.class)); + resolver.registerSerializer(byte.class, new ByteSerializer(fury, byte.class)); + resolver.registerSerializer(short.class, new ShortSerializer(fury, short.class)); + resolver.registerSerializer(char.class, new CharSerializer(fury, char.class)); + resolver.registerSerializer(int.class, new IntSerializer(fury, int.class)); + resolver.registerSerializer(long.class, new LongSerializer(fury, long.class)); + resolver.registerSerializer(float.class, new FloatSerializer(fury, float.class)); + resolver.registerSerializer(double.class, new DoubleSerializer(fury, double.class)); + resolver.registerSerializer(Boolean.class, new BooleanSerializer(fury, Boolean.class)); + resolver.registerSerializer(Byte.class, new ByteSerializer(fury, Byte.class)); + resolver.registerSerializer(Short.class, new ShortSerializer(fury, Short.class)); + resolver.registerSerializer(Character.class, new CharSerializer(fury, Character.class)); + resolver.registerSerializer(Integer.class, new IntSerializer(fury, Integer.class)); + resolver.registerSerializer(Long.class, new LongSerializer(fury, Long.class)); + resolver.registerSerializer(Float.class, new FloatSerializer(fury, Float.class)); + resolver.registerSerializer(Double.class, new DoubleSerializer(fury, Double.class)); } } diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/Serializer.java b/java/fury-core/src/main/java/org/apache/fury/serializer/Serializer.java index 819908920b..720120acf1 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/Serializer.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/Serializer.java @@ -27,8 +27,7 @@ /** * Serialize/deserializer objects into binary. Note that this class is designed as an abstract class - * instead of interface to reduce virtual method call cost of {@link #needToWriteRef}/{@link - * #getXtypeId}. + * instead of interface to reduce virtual method call cost of {@link #needToWriteRef}. * * @param type of objects being serializing/deserializing */ @@ -63,23 +62,6 @@ public T read(MemoryBuffer buffer) { throw new UnsupportedOperationException(); } - /** - * Returns {@link Fury#NOT_SUPPORT_CROSS_LANGUAGE} if the serializer doesn't support - * cross-language serialization. Return a number in range (0, 32767) if the serializer support - * cross-language serialization and native serialization data is the same with cross-language - * serialization. Return a negative short in range [-32768, 0) if the serializer support - * cross-language serialization and native serialization data is not the same with cross-language - * serialization. - */ - public short getXtypeId() { - return Fury.NOT_SUPPORT_CROSS_LANGUAGE; - } - - /** Returns a type tag used for setup type mapping between languages. */ - public String getCrossLanguageTypeTag() { - throw new UnsupportedOperationException(); - } - public void xwrite(MemoryBuffer buffer, T value) { throw new UnsupportedOperationException(); } diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/Serializers.java b/java/fury-core/src/main/java/org/apache/fury/serializer/Serializers.java index 61e7574ef5..e5b01225c1 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/Serializers.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/Serializers.java @@ -48,7 +48,6 @@ import org.apache.fury.memory.Platform; import org.apache.fury.reflect.ReflectionUtils; import org.apache.fury.resolver.ClassResolver; -import org.apache.fury.type.Type; import org.apache.fury.util.ExceptionUtils; import org.apache.fury.util.GraalvmSupport; import org.apache.fury.util.StringUtils; @@ -178,28 +177,18 @@ public static Object readPrimitiveValue(Fury fury, MemoryBuffer buffer, short cl } public abstract static class CrossLanguageCompatibleSerializer extends Serializer { - private final short typeId; - public CrossLanguageCompatibleSerializer(Fury fury, Class cls, short typeId) { + public CrossLanguageCompatibleSerializer(Fury fury, Class cls) { super(fury, cls); - this.typeId = typeId; } public CrossLanguageCompatibleSerializer( - Fury fury, Class cls, short typeId, boolean needToWriteRef) { - super(fury, cls, needToWriteRef); - this.typeId = typeId; - } - - public CrossLanguageCompatibleSerializer( - Fury fury, Class cls, short typeId, boolean needToWriteRef, boolean immutable) { + Fury fury, Class cls, boolean needToWriteRef, boolean immutable) { super(fury, cls, needToWriteRef, immutable); - this.typeId = typeId; } - @Override - public short getXtypeId() { - return typeId; + public CrossLanguageCompatibleSerializer(Fury fury, Class cls, boolean needToWriteRef) { + super(fury, cls, needToWriteRef); } @Override @@ -242,11 +231,6 @@ public void xwrite(MemoryBuffer buffer, T value) { stringSerializer.writeUTF8String(buffer, value.toString()); } - @Override - public short getXtypeId() { - return (short) -Type.STRING.getId(); - } - @Override public void write(MemoryBuffer buffer, T value) { if (GET_CODER != null) { @@ -577,19 +561,20 @@ public Object read(MemoryBuffer buffer) { } public static void registerDefaultSerializers(Fury fury) { - fury.registerSerializer(Class.class, new ClassSerializer(fury)); - fury.registerSerializer(StringBuilder.class, new StringBuilderSerializer(fury)); - fury.registerSerializer(StringBuffer.class, new StringBufferSerializer(fury)); - fury.registerSerializer(BigInteger.class, new BigIntegerSerializer(fury)); - fury.registerSerializer(BigDecimal.class, new BigDecimalSerializer(fury)); - fury.registerSerializer(AtomicBoolean.class, new AtomicBooleanSerializer(fury)); - fury.registerSerializer(AtomicInteger.class, new AtomicIntegerSerializer(fury)); - fury.registerSerializer(AtomicLong.class, new AtomicLongSerializer(fury)); - fury.registerSerializer(AtomicReference.class, new AtomicReferenceSerializer(fury)); - fury.registerSerializer(Currency.class, new CurrencySerializer(fury)); - fury.registerSerializer(URI.class, new URISerializer(fury)); - fury.registerSerializer(Pattern.class, new RegexSerializer(fury)); - fury.registerSerializer(UUID.class, new UUIDSerializer(fury)); - fury.registerSerializer(Object.class, new EmptyObjectSerializer(fury)); + ClassResolver resolver = fury.getClassResolver(); + resolver.registerSerializer(Class.class, new ClassSerializer(fury)); + resolver.registerSerializer(StringBuilder.class, new StringBuilderSerializer(fury)); + resolver.registerSerializer(StringBuffer.class, new StringBufferSerializer(fury)); + resolver.registerSerializer(BigInteger.class, new BigIntegerSerializer(fury)); + resolver.registerSerializer(BigDecimal.class, new BigDecimalSerializer(fury)); + resolver.registerSerializer(AtomicBoolean.class, new AtomicBooleanSerializer(fury)); + resolver.registerSerializer(AtomicInteger.class, new AtomicIntegerSerializer(fury)); + resolver.registerSerializer(AtomicLong.class, new AtomicLongSerializer(fury)); + resolver.registerSerializer(AtomicReference.class, new AtomicReferenceSerializer(fury)); + resolver.registerSerializer(Currency.class, new CurrencySerializer(fury)); + resolver.registerSerializer(URI.class, new URISerializer(fury)); + resolver.registerSerializer(Pattern.class, new RegexSerializer(fury)); + resolver.registerSerializer(UUID.class, new UUIDSerializer(fury)); + resolver.registerSerializer(Object.class, new EmptyObjectSerializer(fury)); } } diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/StringSerializer.java b/java/fury-core/src/main/java/org/apache/fury/serializer/StringSerializer.java index a3379660ef..99c34c260f 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/StringSerializer.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/StringSerializer.java @@ -41,7 +41,6 @@ import org.apache.fury.memory.MemoryBuffer; import org.apache.fury.memory.Platform; import org.apache.fury.reflect.ReflectionUtils; -import org.apache.fury.type.Type; import org.apache.fury.util.MathUtils; import org.apache.fury.util.Preconditions; import org.apache.fury.util.StringEncodingUtils; @@ -115,11 +114,6 @@ public StringSerializer(Fury fury) { compressString = fury.compressString(); } - @Override - public short getXtypeId() { - return Type.STRING.getId(); - } - @Override public void write(MemoryBuffer buffer, String value) { writeJavaString(buffer, value); diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/StructSerializer.java b/java/fury-core/src/main/java/org/apache/fury/serializer/StructSerializer.java index 5e1399bce5..a0f30f3bb2 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/StructSerializer.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/StructSerializer.java @@ -36,11 +36,12 @@ import org.apache.fury.memory.Platform; import org.apache.fury.reflect.FieldAccessor; import org.apache.fury.reflect.TypeRef; +import org.apache.fury.resolver.ClassInfo; import org.apache.fury.type.Descriptor; import org.apache.fury.type.GenericType; import org.apache.fury.type.Generics; -import org.apache.fury.type.Type; import org.apache.fury.type.TypeUtils; +import org.apache.fury.type.Types; import org.apache.fury.util.ExceptionUtils; import org.apache.fury.util.Preconditions; @@ -49,10 +50,9 @@ * *

TODO(chaokunyang) support generics optimization for {@code SomeClass}. */ -@SuppressWarnings({"unchecked", "rawtypes", "UnstableApiUsage"}) +@SuppressWarnings({"unchecked", "rawtypes"}) public class StructSerializer extends Serializer { private static final Logger LOG = LoggerFactory.getLogger(StructSerializer.class); - private final String typeTag; private final Constructor constructor; private final FieldAccessor[] fieldAccessors; private GenericType[] fieldGenerics; @@ -60,9 +60,8 @@ public class StructSerializer extends Serializer { private final IdentityHashMap genericTypesCache; private int typeHash; - public StructSerializer(Fury fury, Class cls, String typeTag) { + public StructSerializer(Fury fury, Class cls) { super(fury, cls); - this.typeTag = typeTag; if (fury.getLanguage() == Language.JAVA) { LOG.warn("Type of class {} shouldn't be serialized using cross-language serializer", cls); } @@ -103,16 +102,6 @@ public T read(MemoryBuffer buffer) { return xread(buffer); } - @Override - public short getXtypeId() { - return Fury.FURY_TYPE_TAG_ID; - } - - @Override - public String getCrossLanguageTypeTag() { - return typeTag; - } - @Override public void xwrite(MemoryBuffer buffer, T value) { // TODO(chaokunyang) support fields back and forward compatible. @@ -186,7 +175,12 @@ public T xread(MemoryBuffer buffer) { if (hasGenerics) { generics.pushGenericType(fieldGeneric); } - Object fieldValue = fury.xreadRefByNullableSerializer(buffer, serializer); + Object fieldValue; + if (serializer == null) { + fieldValue = fury.xreadRef(buffer); + } else { + fieldValue = fury.xreadRef(buffer, serializer); + } fieldAccessor.set(obj, fieldValue); if (hasGenerics) { generics.popGenericType(); @@ -219,20 +213,18 @@ int computeFieldHash(int hash, GenericType fieldGeneric) { int id; if (fieldGeneric.getTypeRef().isSubtypeOf(List.class)) { // TODO(chaokunyang) add list element type into schema hash - id = Type.LIST.getId(); + id = Types.LIST; } else if (fieldGeneric.getTypeRef().isSubtypeOf(Map.class)) { // TODO(chaokunyang) add map key&value type into schema hash - id = Type.MAP.getId(); + id = Types.MAP; } else { try { - Serializer serializer = fury.getClassResolver().getSerializer(fieldGeneric.getCls()); - short xtypeId = serializer.getXtypeId(); - if (xtypeId == Fury.NOT_SUPPORT_CROSS_LANGUAGE) { - return hash; - } - id = Math.abs(xtypeId); - if (id == Type.FURY_TYPE_TAG.getId()) { - id = TypeUtils.computeStringHash(serializer.getCrossLanguageTypeTag()); + ClassInfo classInfo = fury.getClassResolver().getClassInfo(fieldGeneric.getCls()); + int xtypeId = classInfo.getXtypeId(); + if (Types.isStructType((byte) xtypeId)) { + id = TypeUtils.computeStringHash(classInfo.decodeNamespace() + classInfo.decodeTypeName()); + } else { + id = Math.abs(xtypeId); } } catch (Exception e) { return hash; diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/TimeSerializers.java b/java/fury-core/src/main/java/org/apache/fury/serializer/TimeSerializers.java index cdd4129eea..92c8f72058 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/TimeSerializers.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/TimeSerializers.java @@ -41,7 +41,7 @@ import java.util.TimeZone; import org.apache.fury.Fury; import org.apache.fury.memory.MemoryBuffer; -import org.apache.fury.type.Type; +import org.apache.fury.resolver.ClassResolver; import org.apache.fury.util.DateTimeUtils; /** Serializers for all time related types. */ @@ -126,17 +126,14 @@ protected Time newInstance(long time) { } public static final class TimestampSerializer extends TimeSerializer { - private final short typeId; public TimestampSerializer(Fury fury) { // conflict with instant super(fury, Timestamp.class); - typeId = (short) -Type.TIMESTAMP.getId(); } public TimestampSerializer(Fury fury, boolean needToWriteRef) { super(fury, Timestamp.class, needToWriteRef); - typeId = (short) -Type.TIMESTAMP.getId(); } @Override @@ -149,11 +146,6 @@ public Timestamp xread(MemoryBuffer buffer) { return DateTimeUtils.toJavaTimestamp(buffer.readInt64()); } - @Override - public short getXtypeId() { - return typeId; - } - @Override public void write(MemoryBuffer buffer, Timestamp value) { long time = value.getTime() - (value.getNanos() / 1_000_000); @@ -178,11 +170,6 @@ public LocalDateSerializer(Fury fury, boolean needToWriteRef) { super(fury, LocalDate.class, needToWriteRef); } - @Override - public short getXtypeId() { - return Type.DATE32.getId(); - } - @Override public void xwrite(MemoryBuffer buffer, LocalDate value) { // TODO use java encoding to support larger range. @@ -227,11 +214,6 @@ public InstantSerializer(Fury fury, boolean needToWriteRef) { super(fury, Instant.class, needToWriteRef); } - @Override - public short getXtypeId() { - return Type.TIMESTAMP.getId(); - } - @Override public void xwrite(MemoryBuffer buffer, Instant value) { // FIXME JDK17 may have higher precision than millisecond @@ -639,22 +621,23 @@ public OffsetDateTime read(MemoryBuffer buffer) { } public static void registerDefaultSerializers(Fury fury) { - fury.registerSerializer(Date.class, new DateSerializer(fury)); - fury.registerSerializer(java.sql.Date.class, new SqlDateSerializer(fury)); - fury.registerSerializer(Time.class, new SqlTimeSerializer(fury)); - fury.registerSerializer(Timestamp.class, new TimestampSerializer(fury)); - fury.registerSerializer(LocalDate.class, new LocalDateSerializer(fury)); - fury.registerSerializer(LocalTime.class, new LocalTimeSerializer(fury)); - fury.registerSerializer(LocalDateTime.class, new LocalDateTimeSerializer(fury)); - fury.registerSerializer(Instant.class, new InstantSerializer(fury)); - fury.registerSerializer(Duration.class, new DurationSerializer(fury)); - fury.registerSerializer(ZoneOffset.class, new ZoneOffsetSerializer(fury)); - fury.registerSerializer(ZonedDateTime.class, new ZonedDateTimeSerializer(fury)); - fury.registerSerializer(Year.class, new YearSerializer(fury)); - fury.registerSerializer(YearMonth.class, new YearMonthSerializer(fury)); - fury.registerSerializer(MonthDay.class, new MonthDaySerializer(fury)); - fury.registerSerializer(Period.class, new PeriodSerializer(fury)); - fury.registerSerializer(OffsetTime.class, new OffsetTimeSerializer(fury)); - fury.registerSerializer(OffsetDateTime.class, new OffsetDateTimeSerializer(fury)); + ClassResolver resolver = fury.getClassResolver(); + resolver.registerSerializer(Date.class, new DateSerializer(fury)); + resolver.registerSerializer(java.sql.Date.class, new SqlDateSerializer(fury)); + resolver.registerSerializer(Time.class, new SqlTimeSerializer(fury)); + resolver.registerSerializer(Timestamp.class, new TimestampSerializer(fury)); + resolver.registerSerializer(LocalDate.class, new LocalDateSerializer(fury)); + resolver.registerSerializer(LocalTime.class, new LocalTimeSerializer(fury)); + resolver.registerSerializer(LocalDateTime.class, new LocalDateTimeSerializer(fury)); + resolver.registerSerializer(Instant.class, new InstantSerializer(fury)); + resolver.registerSerializer(Duration.class, new DurationSerializer(fury)); + resolver.registerSerializer(ZoneOffset.class, new ZoneOffsetSerializer(fury)); + resolver.registerSerializer(ZonedDateTime.class, new ZonedDateTimeSerializer(fury)); + resolver.registerSerializer(Year.class, new YearSerializer(fury)); + resolver.registerSerializer(YearMonth.class, new YearMonthSerializer(fury)); + resolver.registerSerializer(MonthDay.class, new MonthDaySerializer(fury)); + resolver.registerSerializer(Period.class, new PeriodSerializer(fury)); + resolver.registerSerializer(OffsetTime.class, new OffsetTimeSerializer(fury)); + resolver.registerSerializer(OffsetDateTime.class, new OffsetDateTimeSerializer(fury)); } } diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractCollectionSerializer.java b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractCollectionSerializer.java index 04a68fc395..495c89935d 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractCollectionSerializer.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractCollectionSerializer.java @@ -744,7 +744,12 @@ public void xreadElements( if (elemGenericType.isMonomorphic()) { Serializer elemSerializer = elemGenericType.getSerializer(fury.getClassResolver()); for (int i = 0; i < numElements; i++) { - Object elem = fury.xreadRefByNullableSerializer(buffer, elemSerializer); + Object elem; + if (elemSerializer == null) { + elem = fury.xreadRef(buffer); + } else { + elem = fury.xreadRef(buffer, elemSerializer); + } collection.add(elem); } } else { diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java index 6461e8080b..0a925dfe89 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java @@ -361,38 +361,47 @@ public static void xwriteElements(Fury fury, MemoryBuffer buffer, Map value) { if (!keyGenericType.hasGenericParameters() && !valueGenericType.hasGenericParameters()) { for (Object object : value.entrySet()) { Map.Entry entry = (Map.Entry) object; - fury.xwriteRefByNullableSerializer(buffer, entry.getKey(), keySerializer); - fury.xwriteRefByNullableSerializer(buffer, entry.getValue(), valueSerializer); + xwriteRefByNullableSerializer(fury, buffer, entry.getKey(), keySerializer); + xwriteRefByNullableSerializer(fury, buffer, entry.getValue(), valueSerializer); } } else if (valueGenericType.hasGenericParameters()) { for (Object object : value.entrySet()) { Map.Entry entry = (Map.Entry) object; - fury.xwriteRefByNullableSerializer(buffer, entry.getKey(), keySerializer); + xwriteRefByNullableSerializer(fury, buffer, entry.getKey(), keySerializer); generics.pushGenericType(valueGenericType); - fury.xwriteRefByNullableSerializer(buffer, entry.getValue(), valueSerializer); + xwriteRefByNullableSerializer(fury, buffer, entry.getValue(), valueSerializer); generics.popGenericType(); } } else if (keyGenericType.hasGenericParameters()) { for (Object object : value.entrySet()) { Map.Entry entry = (Map.Entry) object; generics.pushGenericType(keyGenericType); - fury.xwriteRefByNullableSerializer(buffer, entry.getKey(), keySerializer); + xwriteRefByNullableSerializer(fury, buffer, entry.getKey(), keySerializer); generics.popGenericType(); - fury.xwriteRefByNullableSerializer(buffer, entry.getValue(), valueSerializer); + xwriteRefByNullableSerializer(fury, buffer, entry.getValue(), valueSerializer); } } else { for (Object object : value.entrySet()) { Map.Entry entry = (Map.Entry) object; generics.pushGenericType(keyGenericType); - fury.xwriteRefByNullableSerializer(buffer, entry.getKey(), keySerializer); + xwriteRefByNullableSerializer(fury, buffer, entry.getKey(), keySerializer); generics.pushGenericType(valueGenericType); - fury.xwriteRefByNullableSerializer(buffer, entry.getValue(), valueSerializer); + xwriteRefByNullableSerializer(fury, buffer, entry.getValue(), valueSerializer); } } generics.popGenericType(); } } + public static void xwriteRefByNullableSerializer( + Fury fury, MemoryBuffer buffer, T obj, Serializer serializer) { + if (serializer == null) { + fury.xwriteRef(buffer, obj); + } else { + fury.xwriteRef(buffer, obj, serializer); + } + } + private Tuple2 getKVGenericType(GenericType genericType) { Tuple2 genericTypes = partialGenericKVTypeMap.get(genericType); if (genericTypes == null) { @@ -671,33 +680,33 @@ public static void xreadElements(Fury fury, MemoryBuffer buffer, Map map, int si Serializer valueSerializer = valueGenericType.getSerializer(fury.getClassResolver()); if (!keyGenericType.hasGenericParameters() && !valueGenericType.hasGenericParameters()) { for (int i = 0; i < size; i++) { - Object key = fury.xreadRefByNullableSerializer(buffer, keySerializer); - Object value = fury.xreadRefByNullableSerializer(buffer, valueSerializer); + Object key = xreadRefByNullableSerializer(fury, buffer, keySerializer); + Object value = xreadRefByNullableSerializer(fury, buffer, valueSerializer); map.put(key, value); } } else if (valueGenericType.hasGenericParameters()) { for (int i = 0; i < size; i++) { - Object key = fury.xreadRefByNullableSerializer(buffer, keySerializer); + Object key = xreadRefByNullableSerializer(fury, buffer, keySerializer); generics.pushGenericType(valueGenericType); - Object value = fury.xreadRefByNullableSerializer(buffer, valueSerializer); + Object value = xreadRefByNullableSerializer(fury, buffer, valueSerializer); generics.popGenericType(); map.put(key, value); } } else if (keyGenericType.hasGenericParameters()) { for (int i = 0; i < size; i++) { generics.pushGenericType(keyGenericType); - Object key = fury.xreadRefByNullableSerializer(buffer, keySerializer); + Object key = xreadRefByNullableSerializer(fury, buffer, keySerializer); generics.popGenericType(); - Object value = fury.xreadRefByNullableSerializer(buffer, valueSerializer); + Object value = xreadRefByNullableSerializer(fury, buffer, valueSerializer); map.put(key, value); } } else { for (int i = 0; i < size; i++) { // FIXME(chaokunyang) nested generics may be get by mistake. generics.pushGenericType(keyGenericType); - Object key = fury.xreadRefByNullableSerializer(buffer, keySerializer); + Object key = xreadRefByNullableSerializer(fury, buffer, keySerializer); generics.pushGenericType(valueGenericType); - Object value = fury.xreadRefByNullableSerializer(buffer, valueSerializer); + Object value = xreadRefByNullableSerializer(fury, buffer, valueSerializer); map.put(key, value); } } @@ -705,6 +714,15 @@ public static void xreadElements(Fury fury, MemoryBuffer buffer, Map map, int si } } + public static Object xreadRefByNullableSerializer( + Fury fury, MemoryBuffer buffer, Serializer serializer) { + if (serializer == null) { + return fury.xreadRef(buffer); + } else { + return fury.xreadRef(buffer, serializer); + } + } + /** * Hook for java serialization codegen, read/write key/value by entrySet. * diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/CollectionSerializers.java b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/CollectionSerializers.java index a647a0ddf6..a8f78a1d69 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/CollectionSerializers.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/CollectionSerializers.java @@ -58,7 +58,6 @@ import org.apache.fury.serializer.ReplaceResolveSerializer; import org.apache.fury.serializer.Serializer; import org.apache.fury.serializer.Serializers; -import org.apache.fury.type.Type; import org.apache.fury.util.Preconditions; import org.apache.fury.util.unsafe._JDKAccess; @@ -74,11 +73,6 @@ public ArrayListSerializer(Fury fury) { super(fury, ArrayList.class, true); } - @Override - public short getXtypeId() { - return Type.LIST.getId(); - } - @Override public ArrayList newCollection(MemoryBuffer buffer) { int numElements = buffer.readVarUint32Small7(); @@ -117,11 +111,6 @@ public List copy(List originCollection) { return newCollection; } - @Override - public short getXtypeId() { - return (short) -Type.LIST.getId(); - } - @Override public void write(MemoryBuffer buffer, List value) { try { @@ -162,11 +151,6 @@ public HashSetSerializer(Fury fury) { super(fury, HashSet.class, true); } - @Override - public short getXtypeId() { - return Type.FURY_SET.getId(); - } - @Override public HashSet newCollection(MemoryBuffer buffer) { int numElements = buffer.readVarUint32Small7(); @@ -182,11 +166,6 @@ public LinkedHashSetSerializer(Fury fury) { super(fury, LinkedHashSet.class, true); } - @Override - public short getXtypeId() { - return Type.FURY_SET.getId(); - } - @Override public LinkedHashSet newCollection(MemoryBuffer buffer) { int numElements = buffer.readVarUint32Small7(); @@ -269,11 +248,6 @@ public EmptyListSerializer(Fury fury, Class> cls) { @Override public void write(MemoryBuffer buffer, List value) {} - @Override - public short getXtypeId() { - return (short) -Type.LIST.getId(); - } - @Override public void xwrite(MemoryBuffer buffer, List value) { // write length @@ -322,11 +296,6 @@ public EmptySetSerializer(Fury fury, Class> cls) { @Override public void write(MemoryBuffer buffer, Set value) {} - @Override - public short getXtypeId() { - return (short) -Type.FURY_SET.getId(); - } - @Override public void xwrite(MemoryBuffer buffer, Set value) { // write length @@ -377,11 +346,6 @@ public void write(MemoryBuffer buffer, List value) { fury.writeRef(buffer, value.get(0)); } - @Override - public short getXtypeId() { - return (short) -Type.LIST.getId(); - } - @Override public void xwrite(MemoryBuffer buffer, List value) { buffer.writeVarUint32Small7(1); @@ -416,11 +380,6 @@ public void write(MemoryBuffer buffer, Set value) { fury.writeRef(buffer, value.iterator().next()); } - @Override - public short getXtypeId() { - return (short) -Type.FURY_SET.getId(); - } - @Override public void xwrite(MemoryBuffer buffer, Set value) { buffer.writeVarUint32Small7(1); @@ -807,46 +766,48 @@ public T copy(T value) { // TODO Support ArraySubListSerializer, SubListSerializer public static void registerDefaultSerializers(Fury fury) { - fury.registerSerializer(ArrayList.class, new ArrayListSerializer(fury)); + ClassResolver resolver = fury.getClassResolver(); + resolver.registerSerializer(ArrayList.class, new ArrayListSerializer(fury)); Class arrayAsListClass = Arrays.asList(1, 2).getClass(); - fury.registerSerializer(arrayAsListClass, new ArraysAsListSerializer(fury, arrayAsListClass)); - fury.registerSerializer( + resolver.registerSerializer( + arrayAsListClass, new ArraysAsListSerializer(fury, arrayAsListClass)); + resolver.registerSerializer( LinkedList.class, new CollectionSerializer(fury, LinkedList.class, true)); - fury.registerSerializer(HashSet.class, new HashSetSerializer(fury)); - fury.registerSerializer(LinkedHashSet.class, new LinkedHashSetSerializer(fury)); - fury.registerSerializer(TreeSet.class, new SortedSetSerializer<>(fury, TreeSet.class)); - fury.registerSerializer( + resolver.registerSerializer(HashSet.class, new HashSetSerializer(fury)); + resolver.registerSerializer(LinkedHashSet.class, new LinkedHashSetSerializer(fury)); + resolver.registerSerializer(TreeSet.class, new SortedSetSerializer<>(fury, TreeSet.class)); + resolver.registerSerializer( Collections.EMPTY_LIST.getClass(), new EmptyListSerializer(fury, (Class>) Collections.EMPTY_LIST.getClass())); - fury.registerSerializer( + resolver.registerSerializer( Collections.emptySortedSet().getClass(), new EmptySortedSetSerializer( fury, (Class>) Collections.emptySortedSet().getClass())); - fury.registerSerializer( + resolver.registerSerializer( Collections.EMPTY_SET.getClass(), new EmptySetSerializer(fury, (Class>) Collections.EMPTY_SET.getClass())); - fury.registerSerializer( + resolver.registerSerializer( Collections.singletonList(null).getClass(), new CollectionsSingletonListSerializer( fury, (Class>) Collections.singletonList(null).getClass())); - fury.registerSerializer( + resolver.registerSerializer( Collections.singleton(null).getClass(), new CollectionsSingletonSetSerializer( fury, (Class>) Collections.singleton(null).getClass())); - fury.registerSerializer( + resolver.registerSerializer( ConcurrentSkipListSet.class, new ConcurrentSkipListSetSerializer(fury, ConcurrentSkipListSet.class)); - fury.registerSerializer(Vector.class, new VectorSerializer(fury, Vector.class)); - fury.registerSerializer(ArrayDeque.class, new ArrayDequeSerializer(fury, ArrayDeque.class)); - fury.registerSerializer(BitSet.class, new BitSetSerializer(fury, BitSet.class)); - fury.registerSerializer( + resolver.registerSerializer(Vector.class, new VectorSerializer(fury, Vector.class)); + resolver.registerSerializer(ArrayDeque.class, new ArrayDequeSerializer(fury, ArrayDeque.class)); + resolver.registerSerializer(BitSet.class, new BitSetSerializer(fury, BitSet.class)); + resolver.registerSerializer( PriorityQueue.class, new PriorityQueueSerializer(fury, PriorityQueue.class)); - fury.registerSerializer( + resolver.registerSerializer( CopyOnWriteArrayList.class, new CopyOnWriteArrayListSerializer(fury, CopyOnWriteArrayList.class)); final Class setFromMapClass = Collections.newSetFromMap(new HashMap<>()).getClass(); - fury.registerSerializer(setFromMapClass, new SetFromMapSerializer(fury, setFromMapClass)); - fury.registerSerializer( + resolver.registerSerializer(setFromMapClass, new SetFromMapSerializer(fury, setFromMapClass)); + resolver.registerSerializer( ConcurrentHashMap.KeySetView.class, new ConcurrentHashMapKeySetViewSerializer(fury, ConcurrentHashMap.KeySetView.class)); } diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/FuryArrayAsListSerializer.java b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/FuryArrayAsListSerializer.java index 057acb7bcc..09268c4e15 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/FuryArrayAsListSerializer.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/FuryArrayAsListSerializer.java @@ -26,7 +26,6 @@ import org.apache.fury.Fury; import org.apache.fury.annotation.Internal; import org.apache.fury.memory.MemoryBuffer; -import org.apache.fury.type.Type; /** Serializer for {@link ArrayAsList}. Helper for serialization of other classes. */ @Internal @@ -37,11 +36,6 @@ public FuryArrayAsListSerializer(Fury fury) { super(fury, ArrayAsList.class, true); } - @Override - public short getXtypeId() { - return (short) -Type.LIST.getId(); - } - public Collection newCollection(MemoryBuffer buffer) { int numElements = buffer.readVarUint32Small7(); setNumElements(numElements); diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/GuavaCollectionSerializers.java b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/GuavaCollectionSerializers.java index fe94b43556..44740cc935 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/GuavaCollectionSerializers.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/GuavaCollectionSerializers.java @@ -39,7 +39,7 @@ import org.apache.fury.Fury; import org.apache.fury.memory.MemoryBuffer; import org.apache.fury.memory.Platform; -import org.apache.fury.type.Type; +import org.apache.fury.resolver.ClassResolver; import org.apache.fury.util.unsafe._JDKAccess; /** Serializers for common guava types. */ @@ -84,11 +84,6 @@ public T onCollectionRead(Collection collection) { return (T) list; } - @Override - public short getXtypeId() { - return (short) -Type.LIST.getId(); - } - public T xnewInstance(Collection collection) { return (T) ImmutableList.copyOf(collection); } @@ -148,11 +143,6 @@ public T copy(T originCollection) { return (T) function.apply(elements); } - @Override - public short getXtypeId() { - return (short) -Type.LIST.getId(); - } - @Override protected T xnewInstance(Collection collection) { return (T) ImmutableList.copyOf(collection); @@ -179,11 +169,6 @@ public T onCollectionRead(Collection collection) { return (T) ImmutableSet.copyOf(elements); } - @Override - public short getXtypeId() { - return (short) -Type.FURY_SET.getId(); - } - @Override protected T xnewInstance(Collection collection) { return (T) ImmutableSet.copyOf(collection); @@ -271,11 +256,6 @@ public T onMapRead(Map map) { return (T) builder.build(); } - @Override - public short getXtypeId() { - return (short) -Type.MAP.getId(); - } - @Override public T xread(MemoryBuffer buffer) { int size = buffer.readVarUint32Small7(); @@ -426,65 +406,66 @@ public static void registerDefaultSerializers(Fury fury) { // inconsistent if peers load different version of guava. // For example: guava 20 return ImmutableBiMap for ImmutableMap.of(), but guava 27 return // ImmutableMap. + ClassResolver resolver = fury.getClassResolver(); Class cls = loadClass(pkg + ".RegularImmutableBiMap", ImmutableBiMap.of("k1", 1, "k2", 4).getClass()); - fury.registerSerializer(cls, new ImmutableBiMapSerializer(fury, cls)); + resolver.registerSerializer(cls, new ImmutableBiMapSerializer(fury, cls)); cls = loadClass(pkg + ".SingletonImmutableBiMap", ImmutableBiMap.of(1, 2).getClass()); - fury.registerSerializer(cls, new ImmutableBiMapSerializer(fury, cls)); + resolver.registerSerializer(cls, new ImmutableBiMapSerializer(fury, cls)); cls = loadClass(pkg + ".RegularImmutableMap", ImmutableMap.of("k1", 1, "k2", 2).getClass()); - fury.registerSerializer(cls, new ImmutableMapSerializer(fury, cls)); + resolver.registerSerializer(cls, new ImmutableMapSerializer(fury, cls)); cls = loadClass(pkg + ".RegularImmutableList", ImmutableList.of().getClass()); - fury.registerSerializer(cls, new RegularImmutableListSerializer(fury, cls)); + resolver.registerSerializer(cls, new RegularImmutableListSerializer(fury, cls)); cls = loadClass(pkg + ".SingletonImmutableList", ImmutableList.of(1).getClass()); - fury.registerSerializer(cls, new ImmutableListSerializer(fury, cls)); + resolver.registerSerializer(cls, new ImmutableListSerializer(fury, cls)); cls = loadClass(pkg + ".RegularImmutableSet", ImmutableSet.of(1, 2).getClass()); - fury.registerSerializer(cls, new ImmutableSetSerializer(fury, cls)); + resolver.registerSerializer(cls, new ImmutableSetSerializer(fury, cls)); cls = loadClass(pkg + ".SingletonImmutableSet", ImmutableSet.of(1).getClass()); - fury.registerSerializer(cls, new ImmutableSetSerializer(fury, cls)); + resolver.registerSerializer(cls, new ImmutableSetSerializer(fury, cls)); // sorted set/map doesn't support xlang. cls = loadClass(pkg + ".RegularImmutableSortedSet", ImmutableSortedSet.of(1, 2).getClass()); - fury.registerSerializer(cls, new ImmutableSortedSetSerializer<>(fury, cls)); + resolver.registerSerializer(cls, new ImmutableSortedSetSerializer<>(fury, cls)); cls = loadClass(pkg + ".ImmutableSortedMap", ImmutableSortedMap.of(1, 2).getClass()); - fury.registerSerializer(cls, new ImmutableSortedMapSerializer<>(fury, cls)); + resolver.registerSerializer(cls, new ImmutableSortedMapSerializer<>(fury, cls)); // Guava version before 19.0, of() return // EmptyImmutableSet/EmptyImmutableBiMap/EmptyImmutableSortedMap/EmptyImmutableSortedSet // we register if class exist or register empty to deserialize. if (checkClassExist(pkg + ".EmptyImmutableSet")) { cls = loadClass(pkg + ".EmptyImmutableSet", ImmutableSet.of().getClass()); - fury.registerSerializer(cls, new ImmutableSetSerializer(fury, cls)); + resolver.registerSerializer(cls, new ImmutableSetSerializer(fury, cls)); } else { class GuavaEmptySet {} cls = GuavaEmptySet.class; - fury.registerSerializer(cls, new ImmutableSetSerializer(fury, cls)); + resolver.registerSerializer(cls, new ImmutableSetSerializer(fury, cls)); } if (checkClassExist(pkg + ".EmptyImmutableBiMap")) { cls = loadClass(pkg + ".EmptyImmutableBiMap", ImmutableBiMap.of().getClass()); - fury.registerSerializer(cls, new ImmutableMapSerializer(fury, cls)); + resolver.registerSerializer(cls, new ImmutableMapSerializer(fury, cls)); } else { class GuavaEmptyBiMap {} cls = GuavaEmptyBiMap.class; - fury.registerSerializer(cls, new ImmutableMapSerializer(fury, cls)); + resolver.registerSerializer(cls, new ImmutableMapSerializer(fury, cls)); } if (checkClassExist(pkg + ".EmptyImmutableSortedSet")) { cls = loadClass(pkg + ".EmptyImmutableSortedSet", ImmutableSortedSet.of().getClass()); - fury.registerSerializer(cls, new ImmutableSortedSetSerializer(fury, cls)); + resolver.registerSerializer(cls, new ImmutableSortedSetSerializer(fury, cls)); } else { class GuavaEmptySortedSet {} cls = GuavaEmptySortedSet.class; - fury.registerSerializer(cls, new ImmutableSortedSetSerializer(fury, cls)); + resolver.registerSerializer(cls, new ImmutableSortedSetSerializer(fury, cls)); } if (checkClassExist(pkg + ".EmptyImmutableSortedMap")) { cls = loadClass(pkg + ".EmptyImmutableSortedMap", ImmutableSortedMap.of().getClass()); - fury.registerSerializer(cls, new ImmutableSortedMapSerializer(fury, cls)); + resolver.registerSerializer(cls, new ImmutableSortedMapSerializer(fury, cls)); } else { class GuavaEmptySortedMap {} cls = GuavaEmptySortedMap.class; - fury.registerSerializer(cls, new ImmutableSortedMapSerializer(fury, cls)); + resolver.registerSerializer(cls, new ImmutableSortedMapSerializer(fury, cls)); } } diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/ImmutableCollectionSerializers.java b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/ImmutableCollectionSerializers.java index ae31d90bbf..d79f52b97a 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/ImmutableCollectionSerializers.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/ImmutableCollectionSerializers.java @@ -32,6 +32,7 @@ import org.apache.fury.Fury; import org.apache.fury.memory.MemoryBuffer; import org.apache.fury.memory.Platform; +import org.apache.fury.resolver.ClassResolver; import org.apache.fury.util.unsafe._JDKAccess; /** Serializers for jdk9+ java.util.ImmutableCollections. */ @@ -259,12 +260,13 @@ public Map onMapRead(Map map) { } public static void registerSerializers(Fury fury) { - fury.registerSerializer(List12, new ImmutableListSerializer(fury, List12)); - fury.registerSerializer(ListN, new ImmutableListSerializer(fury, ListN)); - fury.registerSerializer(SubList, new ImmutableListSerializer(fury, SubList)); - fury.registerSerializer(Set12, new ImmutableSetSerializer(fury, Set12)); - fury.registerSerializer(SetN, new ImmutableSetSerializer(fury, SetN)); - fury.registerSerializer(Map1, new ImmutableMapSerializer(fury, Map1)); - fury.registerSerializer(MapN, new ImmutableMapSerializer(fury, MapN)); + ClassResolver resolver = fury.getClassResolver(); + resolver.registerSerializer(List12, new ImmutableListSerializer(fury, List12)); + resolver.registerSerializer(ListN, new ImmutableListSerializer(fury, ListN)); + resolver.registerSerializer(SubList, new ImmutableListSerializer(fury, SubList)); + resolver.registerSerializer(Set12, new ImmutableSetSerializer(fury, Set12)); + resolver.registerSerializer(SetN, new ImmutableSetSerializer(fury, SetN)); + resolver.registerSerializer(Map1, new ImmutableMapSerializer(fury, Map1)); + resolver.registerSerializer(MapN, new ImmutableMapSerializer(fury, MapN)); } } diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapSerializers.java b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapSerializers.java index a9f6a6263a..2ca878705f 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapSerializers.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapSerializers.java @@ -43,7 +43,6 @@ import org.apache.fury.serializer.Serializer; import org.apache.fury.serializer.Serializers; import org.apache.fury.serializer.StringSerializer; -import org.apache.fury.type.Type; import org.apache.fury.util.Preconditions; /** @@ -58,11 +57,6 @@ public HashMapSerializer(Fury fury) { super(fury, HashMap.class, true); } - @Override - public short getXtypeId() { - return Type.MAP.getId(); - } - @Override public HashMap newMap(MemoryBuffer buffer) { int numElements = buffer.readVarUint32Small7(); @@ -83,11 +77,6 @@ public LinkedHashMapSerializer(Fury fury) { super(fury, LinkedHashMap.class, true); } - @Override - public short getXtypeId() { - return Type.MAP.getId(); - } - @Override public LinkedHashMap newMap(MemoryBuffer buffer) { int numElements = buffer.readVarUint32Small7(); @@ -108,11 +97,6 @@ public LazyMapSerializer(Fury fury) { super(fury, LazyMap.class, true); } - @Override - public short getXtypeId() { - return Type.MAP.getId(); - } - @Override public LazyMap newMap(MemoryBuffer buffer) { int numElements = buffer.readVarUint32Small7(); @@ -189,11 +173,6 @@ public EmptyMapSerializer(Fury fury, Class> cls) { @Override public void write(MemoryBuffer buffer, Map value) {} - @Override - public short getXtypeId() { - return (short) -Type.MAP.getId(); - } - @Override public void xwrite(MemoryBuffer buffer, Map value) { // write length @@ -246,11 +225,6 @@ public void write(MemoryBuffer buffer, Map value) { fury.writeRef(buffer, entry.getValue()); } - @Override - public short getXtypeId() { - return (short) -Type.MAP.getId(); - } - @Override public void xwrite(MemoryBuffer buffer, Map value) { buffer.writeVarUint32Small7(1); @@ -294,11 +268,6 @@ public ConcurrentHashMap newMap(MemoryBuffer buffer) { public Map newMap(Map map) { return new ConcurrentHashMap(map.size()); } - - @Override - public short getXtypeId() { - return Fury.NOT_SUPPORT_CROSS_LANGUAGE; - } } public static final class ConcurrentSkipListMapSerializer @@ -323,11 +292,6 @@ public Map newMap(Map originMap) { Comparator comparator = fury.copyObject(((ConcurrentSkipListMap) originMap).comparator()); return new ConcurrentSkipListMap(comparator); } - - @Override - public short getXtypeId() { - return Fury.NOT_SUPPORT_CROSS_LANGUAGE; - } } public static class EnumMapSerializer extends MapSerializer { @@ -517,27 +481,28 @@ public T copy(T value) { // TODO(chaokunyang) support ConcurrentSkipListMap.SubMap mo efficiently. public static void registerDefaultSerializers(Fury fury) { - fury.registerSerializer(HashMap.class, new HashMapSerializer(fury)); + ClassResolver resolver = fury.getClassResolver(); + resolver.registerSerializer(HashMap.class, new HashMapSerializer(fury)); fury.getClassResolver() .registerSerializer(LinkedHashMap.class, new LinkedHashMapSerializer(fury)); - fury.registerSerializer(TreeMap.class, new SortedMapSerializer<>(fury, TreeMap.class)); - fury.registerSerializer( + resolver.registerSerializer(TreeMap.class, new SortedMapSerializer<>(fury, TreeMap.class)); + resolver.registerSerializer( Collections.EMPTY_MAP.getClass(), new EmptyMapSerializer(fury, (Class>) Collections.EMPTY_MAP.getClass())); - fury.registerSerializer( + resolver.registerSerializer( Collections.emptySortedMap().getClass(), new EmptySortedMapSerializer( fury, (Class>) Collections.emptySortedMap().getClass())); - fury.registerSerializer( + resolver.registerSerializer( Collections.singletonMap(null, null).getClass(), new SingletonMapSerializer( fury, (Class>) Collections.singletonMap(null, null).getClass())); - fury.registerSerializer( + resolver.registerSerializer( ConcurrentHashMap.class, new ConcurrentHashMapSerializer(fury, ConcurrentHashMap.class)); - fury.registerSerializer( + resolver.registerSerializer( ConcurrentSkipListMap.class, new ConcurrentSkipListMapSerializer(fury, ConcurrentSkipListMap.class)); - fury.registerSerializer(EnumMap.class, new EnumMapSerializer(fury)); - fury.registerSerializer(LazyMap.class, new LazyMapSerializer(fury)); + resolver.registerSerializer(EnumMap.class, new EnumMapSerializer(fury)); + resolver.registerSerializer(LazyMap.class, new LazyMapSerializer(fury)); } } diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/SubListSerializers.java b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/SubListSerializers.java index 60353e7866..0f3fa73da2 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/SubListSerializers.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/SubListSerializers.java @@ -28,6 +28,7 @@ import org.apache.fury.logging.LoggerFactory; import org.apache.fury.memory.MemoryBuffer; import org.apache.fury.reflect.ReflectionUtils; +import org.apache.fury.resolver.ClassResolver; import org.apache.fury.serializer.ObjectSerializer; @SuppressWarnings({"rawtypes", "unchecked"}) @@ -74,14 +75,15 @@ class ImmutableSubListStub implements Stub {} } public static void registerSerializers(Fury fury, boolean preserveView) { + ClassResolver classResolver = fury.getClassResolver(); for (Class cls : new Class[] { SubListClass, RandomAccessSubListClass, ArrayListSubListClass, ImmutableSubListClass }) { if (fury.trackingRef() && preserveView && fury.getConfig().getLanguage() == Language.JAVA) { - fury.registerSerializer(cls, new SubListViewSerializer(fury, cls)); + classResolver.registerSerializer(cls, new SubListViewSerializer(fury, cls)); } else { - fury.registerSerializer(cls, new SubListSerializer(fury, (Class) cls)); + classResolver.registerSerializer(cls, new SubListSerializer(fury, (Class) cls)); } } } diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/SynchronizedSerializers.java b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/SynchronizedSerializers.java index c7e7c87357..9af57399f0 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/SynchronizedSerializers.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/SynchronizedSerializers.java @@ -40,6 +40,7 @@ import org.apache.fury.logging.LoggerFactory; import org.apache.fury.memory.MemoryBuffer; import org.apache.fury.memory.Platform; +import org.apache.fury.resolver.ClassResolver; import org.apache.fury.serializer.Serializer; import org.apache.fury.util.ExceptionUtils; @@ -211,8 +212,9 @@ static Tuple2, Function>[] synchronizedFactories() { */ public static void registerSerializers(Fury fury) { try { + ClassResolver resolver = fury.getClassResolver(); for (Tuple2, Function> factory : synchronizedFactories()) { - fury.registerSerializer(factory.f0, createSerializer(fury, factory)); + resolver.registerSerializer(factory.f0, createSerializer(fury, factory)); } } catch (Throwable e) { ExceptionUtils.ignore(e); diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/UnmodifiableSerializers.java b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/UnmodifiableSerializers.java index 84c71ae4c9..15abf6740b 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/UnmodifiableSerializers.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/UnmodifiableSerializers.java @@ -39,6 +39,7 @@ import org.apache.fury.logging.LoggerFactory; import org.apache.fury.memory.MemoryBuffer; import org.apache.fury.memory.Platform; +import org.apache.fury.resolver.ClassResolver; import org.apache.fury.serializer.Serializer; import org.apache.fury.util.ExceptionUtils; import org.apache.fury.util.Preconditions; @@ -209,8 +210,9 @@ static Tuple2, Function>[] unmodifiableFactories() { */ public static void registerSerializers(Fury fury) { try { + ClassResolver resolver = fury.getClassResolver(); for (Tuple2, Function> factory : unmodifiableFactories()) { - fury.registerSerializer(factory.f0, createSerializer(fury, factory)); + resolver.registerSerializer(factory.f0, createSerializer(fury, factory)); } } catch (Throwable e) { ExceptionUtils.ignore(e); diff --git a/java/fury-core/src/main/java/org/apache/fury/type/ScalaTypes.java b/java/fury-core/src/main/java/org/apache/fury/type/ScalaTypes.java index 31d439067f..3615f0b75b 100644 --- a/java/fury-core/src/main/java/org/apache/fury/type/ScalaTypes.java +++ b/java/fury-core/src/main/java/org/apache/fury/type/ScalaTypes.java @@ -30,6 +30,7 @@ public class ScalaTypes { private static volatile Class SCALA_MAP_TYPE; private static volatile Class SCALA_SEQ_TYPE; + private static volatile Class SCALA_SET_TYPE; private static volatile Class SCALA_ITERABLE_TYPE; private static volatile java.lang.reflect.Type SCALA_ITERATOR_RETURN_TYPE; private static volatile java.lang.reflect.Type SCALA_NEXT_RETURN_TYPE; @@ -50,6 +51,13 @@ public static Class getScalaSeqType() { return SCALA_SEQ_TYPE; } + public static Class getScalaSetType() { + if (SCALA_SET_TYPE == null) { + SCALA_SET_TYPE = ReflectionUtils.loadClass("scala.collection.Set"); + } + return SCALA_SET_TYPE; + } + public static Class getScalaIterableType() { if (SCALA_ITERABLE_TYPE == null) { SCALA_ITERABLE_TYPE = ReflectionUtils.loadClass("scala.collection.Iterable"); diff --git a/java/fury-core/src/main/java/org/apache/fury/type/TypeUtils.java b/java/fury-core/src/main/java/org/apache/fury/type/TypeUtils.java index 73861e5c24..b7462554b5 100644 --- a/java/fury-core/src/main/java/org/apache/fury/type/TypeUtils.java +++ b/java/fury-core/src/main/java/org/apache/fury/type/TypeUtils.java @@ -51,6 +51,7 @@ import org.apache.fury.reflect.TypeParameter; import org.apache.fury.reflect.TypeRef; import org.apache.fury.util.Preconditions; +import org.apache.fury.util.StringUtils; /** Type utils for common type inference and extraction. */ @SuppressWarnings({"UnstableApiUsage", "unchecked"}) @@ -734,4 +735,12 @@ public static boolean isEnumArray(Class clz) { } return getArrayComponent(clz).isEnum(); } + + public static String qualifiedName(String pkg, String className) { + if (StringUtils.isBlank(pkg)) { + return className; + } else { + return pkg + "." + className; + } + } } diff --git a/java/fury-core/src/main/java/org/apache/fury/type/Types.java b/java/fury-core/src/main/java/org/apache/fury/type/Types.java new file mode 100644 index 0000000000..59062391e3 --- /dev/null +++ b/java/fury-core/src/main/java/org/apache/fury/type/Types.java @@ -0,0 +1,60 @@ +package org.apache.fury.type; + +public class Types { + public static final int BOOL = 1; + public static final int INT8 = 2; + public static final int INT16 = 3; + public static final int INT32 = 4; + public static final int VAR_INT32 = 5; + public static final int INT64 = 6; + public static final int VAR_INT64 = 7; + public static final int SLI_INT64 = 8; + public static final int FLOAT16 = 9; + public static final int FLOAT32 = 10; + public static final int FLOAT64 = 11; + public static final int STRING = 12; + public static final int ENUM = 13; + public static final int NS_ENUM = 14; + public static final int STRUCT = 15; + public static final int POLYMORPHIC_STRUCT = 16; + public static final int COMPATIBLE_STRUCT = 17; + public static final int POLYMORPHIC_COMPATIBLE_STRUCT = 18; + public static final int NS_STRUCT = 19; + public static final int NS_POLYMORPHIC_STRUCT = 20; + public static final int NS_COMPATIBLE_STRUCT = 21; + public static final int NS_POLYMORPHIC_COMPATIBLE_STRUCT = 22; + public static final int EXT = 23; + public static final int POLYMORPHIC_EXT = 24; + public static final int NS_EXT = 25; + public static final int NS_POLYMORPHIC_EXT = 26; + public static final int LIST = 27; + public static final int SET = 28; + public static final int MAP = 29; + public static final int DURATION = 30; + public static final int TIMESTAMP = 31; + public static final int LOCAL_DATE = 32; + public static final int DECIMAL = 33; + public static final int BINARY = 34; + public static final int ARRAY = 35; + public static final int BOOL_ARRAY = 36; + public static final int INT8_ARRAY = 37; + public static final int INT16_ARRAY = 38; + public static final int INT32_ARRAY = 39; + public static final int INT64_ARRAY = 40; + public static final int FLOAT16_ARRAY = 41; + public static final int FLOAT32_ARRAY = 42; + public static final int FLOAT64_ARRAY = 43; + public static final int ARROW_RECORD_BATCH = 44; + public static final int ARROW_TABLE = 45; + + public static boolean isStructType(int value) { + return value == STRUCT || + value == POLYMORPHIC_STRUCT || + value == COMPATIBLE_STRUCT || + value == POLYMORPHIC_COMPATIBLE_STRUCT || + value == NS_STRUCT || + value == NS_POLYMORPHIC_STRUCT || + value == NS_COMPATIBLE_STRUCT || + value == NS_POLYMORPHIC_COMPATIBLE_STRUCT; + } +} diff --git a/java/fury-core/src/test/java/org/apache/fury/CrossLanguageTest.java b/java/fury-core/src/test/java/org/apache/fury/CrossLanguageTest.java index 8a85c678df..55c8ee39ba 100644 --- a/java/fury-core/src/test/java/org/apache/fury/CrossLanguageTest.java +++ b/java/fury-core/src/test/java/org/apache/fury/CrossLanguageTest.java @@ -588,16 +588,6 @@ public ComplexObject1 read(MemoryBuffer buffer) { return xread(buffer); } - @Override - public short getXtypeId() { - return Fury.FURY_TYPE_TAG_ID; - } - - @Override - public String getCrossLanguageTypeTag() { - return "test.ComplexObject1"; - } - @Override public void xwrite(MemoryBuffer buffer, ComplexObject1 value) { fury.xwriteRef(buffer, value.f1); @@ -625,6 +615,7 @@ public void testRegisterSerializer() throws Exception { .withRefTracking(true) .requireClassRegistration(false) .build(); + fury.register(ComplexObject1.class, "test.ComplexObject1"); fury.registerSerializer(ComplexObject1.class, ComplexObject1Serializer.class); ComplexObject1 obj = new ComplexObject1(); obj.f1 = true; diff --git a/java/fury-core/src/main/java/org/apache/fury/type/Type.java b/java/fury-format/src/main/java/org/apache/fury/format/type/ArrowType.java similarity index 97% rename from java/fury-core/src/main/java/org/apache/fury/type/Type.java rename to java/fury-format/src/main/java/org/apache/fury/format/type/ArrowType.java index 24e17579e7..cc41567277 100644 --- a/java/fury-core/src/main/java/org/apache/fury/type/Type.java +++ b/java/fury-format/src/main/java/org/apache/fury/format/type/ArrowType.java @@ -17,12 +17,12 @@ * under the License. */ -package org.apache.fury.type; +package org.apache.fury.format.type; import org.apache.fury.util.Preconditions; /** Keep in sync with Type::type in arrow/type_fwd.h */ -public enum Type { +public enum ArrowType { /// A NULL type having no physical storage NA, // NA = 0 @@ -164,12 +164,12 @@ public enum Type { private short id; - Type() { + ArrowType() { Preconditions.checkArgument(ordinal() < Short.MAX_VALUE); this.id = (short) ordinal(); } - Type(int id) { + ArrowType(int id) { Preconditions.checkArgument(id < Short.MAX_VALUE && id >= 0); this.id = (short) id; } diff --git a/java/fury-format/src/main/java/org/apache/fury/format/type/DataTypes.java b/java/fury-format/src/main/java/org/apache/fury/format/type/DataTypes.java index 3bb83c4085..450c83718f 100644 --- a/java/fury-format/src/main/java/org/apache/fury/format/type/DataTypes.java +++ b/java/fury-format/src/main/java/org/apache/fury/format/type/DataTypes.java @@ -36,259 +36,268 @@ import org.apache.arrow.vector.types.DateUnit; import org.apache.arrow.vector.types.FloatingPointPrecision; import org.apache.arrow.vector.types.TimeUnit; -import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; import org.apache.fury.exception.FuryException; import org.apache.fury.io.MemoryBufferOutputStream; import org.apache.fury.memory.MemoryBuffer; -import org.apache.fury.type.Type; import org.apache.fury.util.DecimalUtils; import org.apache.fury.util.Preconditions; /** Arrow data type utils. */ public class DataTypes { - public static Field PRIMITIVE_BOOLEAN_ARRAY_FIELD = primitiveArrayField(ArrowType.Bool.INSTANCE); + public static Field PRIMITIVE_BOOLEAN_ARRAY_FIELD = + primitiveArrayField(org.apache.arrow.vector.types.pojo.ArrowType.Bool.INSTANCE); public static Field PRIMITIVE_BYTE_ARRAY_FIELD = primitiveArrayField(intType(8)); public static Field PRIMITIVE_SHORT_ARRAY_FIELD = primitiveArrayField(intType(16)); public static Field PRIMITIVE_INT_ARRAY_FIELD = primitiveArrayField(intType(32)); public static Field PRIMITIVE_LONG_ARRAY_FIELD = primitiveArrayField(intType(64)); public static Field PRIMITIVE_FLOAT_ARRAY_FIELD = - primitiveArrayField(new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)); + primitiveArrayField( + new org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint( + FloatingPointPrecision.SINGLE)); public static Field PRIMITIVE_DOUBLE_ARRAY_FIELD = - primitiveArrayField(new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)); + primitiveArrayField( + new org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint( + FloatingPointPrecision.DOUBLE)); // Array item field default name public static final String ARRAY_ITEM_NAME = "item"; - private static final ArrowType.ArrowTypeVisitor typeWidthVisitor = - new DefaultTypeVisitor() { + private static final org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeVisitor + typeWidthVisitor = + new DefaultTypeVisitor() { - @Override - public Integer visit(ArrowType.Struct type) { - return -1; - } + @Override + public Integer visit(org.apache.arrow.vector.types.pojo.ArrowType.Struct type) { + return -1; + } - @Override - public Integer visit(ArrowType.List type) { - return -1; - } + @Override + public Integer visit(org.apache.arrow.vector.types.pojo.ArrowType.List type) { + return -1; + } - @Override - public Integer visit(ArrowType.Map type) { - return -1; - } + @Override + public Integer visit(org.apache.arrow.vector.types.pojo.ArrowType.Map type) { + return -1; + } - @Override - public Integer visit(ArrowType.Bool type) { - return 1; - } + @Override + public Integer visit(org.apache.arrow.vector.types.pojo.ArrowType.Bool type) { + return 1; + } - @Override - public Integer visit(ArrowType.Int type) { - return type.getBitWidth() / 8; - } + @Override + public Integer visit(org.apache.arrow.vector.types.pojo.ArrowType.Int type) { + return type.getBitWidth() / 8; + } - @Override - public Integer visit(ArrowType.FloatingPoint type) { - switch (type.getPrecision()) { - case SINGLE: - return 4; - case DOUBLE: - return 8; - default: - return unsupported(type); - } - } + @Override + public Integer visit(org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint type) { + switch (type.getPrecision()) { + case SINGLE: + return 4; + case DOUBLE: + return 8; + default: + return unsupported(type); + } + } - @Override - public Integer visit(ArrowType.Date type) { - return 4; - } + @Override + public Integer visit(org.apache.arrow.vector.types.pojo.ArrowType.Date type) { + return 4; + } - @Override - public Integer visit(ArrowType.Timestamp type) { - return 8; - } + @Override + public Integer visit(org.apache.arrow.vector.types.pojo.ArrowType.Timestamp type) { + return 8; + } - @Override - public Integer visit(ArrowType.Binary type) { - return -1; - } + @Override + public Integer visit(org.apache.arrow.vector.types.pojo.ArrowType.Binary type) { + return -1; + } - @Override - public Integer visit(ArrowType.Decimal type) { - return -1; - } + @Override + public Integer visit(org.apache.arrow.vector.types.pojo.ArrowType.Decimal type) { + return -1; + } - @Override - public Integer visit(ArrowType.Utf8 type) { - return -1; - } - }; + @Override + public Integer visit(org.apache.arrow.vector.types.pojo.ArrowType.Utf8 type) { + return -1; + } + }; - private static final ArrowType.ArrowTypeVisitor typeIdVisitor = - new DefaultTypeVisitor() { + private static final org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeVisitor + typeIdVisitor = + new DefaultTypeVisitor() { - @Override - public Type visit(ArrowType.Bool type) { - return Type.BOOL; - } - - @Override - public Type visit(ArrowType.Int type) { - if (type.getIsSigned()) { - int byteWidth = type.getBitWidth() / 8; - switch (byteWidth) { - case 1: - return Type.INT8; - case 2: - return Type.INT16; - case 4: - return Type.INT32; - case 8: - return Type.INT64; - default: - return unsupported(type); + @Override + public ArrowType visit(org.apache.arrow.vector.types.pojo.ArrowType.Bool type) { + return ArrowType.BOOL; } - } - return unsupported(type); - } - @Override - public Type visit(ArrowType.FloatingPoint type) { - switch (type.getPrecision()) { - case SINGLE: - return Type.FLOAT; - case DOUBLE: - return Type.DOUBLE; - default: + @Override + public ArrowType visit(org.apache.arrow.vector.types.pojo.ArrowType.Int type) { + if (type.getIsSigned()) { + int byteWidth = type.getBitWidth() / 8; + switch (byteWidth) { + case 1: + return ArrowType.INT8; + case 2: + return ArrowType.INT16; + case 4: + return ArrowType.INT32; + case 8: + return ArrowType.INT64; + default: + return unsupported(type); + } + } return unsupported(type); - } - } + } - @Override - public Type visit(ArrowType.Date type) { - switch (type.getUnit()) { - case DAY: - return Type.DATE32; - case MILLISECOND: - return Type.DATE64; - default: - return unsupported(type); - } - } + @Override + public ArrowType visit( + org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint type) { + switch (type.getPrecision()) { + case SINGLE: + return ArrowType.FLOAT; + case DOUBLE: + return ArrowType.DOUBLE; + default: + return unsupported(type); + } + } - @Override - public Type visit(ArrowType.Timestamp type) { - return Type.TIMESTAMP; - } + @Override + public ArrowType visit(org.apache.arrow.vector.types.pojo.ArrowType.Date type) { + switch (type.getUnit()) { + case DAY: + return ArrowType.DATE32; + case MILLISECOND: + return ArrowType.DATE64; + default: + return unsupported(type); + } + } - @Override - public Type visit(ArrowType.Binary type) { - return Type.BINARY; - } + @Override + public ArrowType visit(org.apache.arrow.vector.types.pojo.ArrowType.Timestamp type) { + return ArrowType.TIMESTAMP; + } - @Override - public Type visit(ArrowType.Decimal type) { - return Type.DECIMAL; - } + @Override + public ArrowType visit(org.apache.arrow.vector.types.pojo.ArrowType.Binary type) { + return ArrowType.BINARY; + } - @Override - public Type visit(ArrowType.Utf8 type) { - return Type.STRING; - } + @Override + public ArrowType visit(org.apache.arrow.vector.types.pojo.ArrowType.Decimal type) { + return ArrowType.DECIMAL; + } - @Override - public Type visit(ArrowType.Struct type) { - return Type.STRUCT; - } + @Override + public ArrowType visit(org.apache.arrow.vector.types.pojo.ArrowType.Utf8 type) { + return ArrowType.STRING; + } - @Override - public Type visit(ArrowType.List type) { - return Type.LIST; - } + @Override + public ArrowType visit(org.apache.arrow.vector.types.pojo.ArrowType.Struct type) { + return ArrowType.STRUCT; + } - @Override - public Type visit(ArrowType.Map type) { - return Type.MAP; - } - }; + @Override + public ArrowType visit(org.apache.arrow.vector.types.pojo.ArrowType.List type) { + return ArrowType.LIST; + } - public static int getTypeWidth(ArrowType type) { + @Override + public ArrowType visit(org.apache.arrow.vector.types.pojo.ArrowType.Map type) { + return ArrowType.MAP; + } + }; + + public static int getTypeWidth(org.apache.arrow.vector.types.pojo.ArrowType type) { return type.accept(typeWidthVisitor); } - public static Type getTypeId(ArrowType type) { + public static ArrowType getTypeId(org.apache.arrow.vector.types.pojo.ArrowType type) { return type.accept(typeIdVisitor); } - public static short getTypeIdValue(ArrowType type) { + public static short getTypeIdValue(org.apache.arrow.vector.types.pojo.ArrowType type) { return type.accept(typeIdVisitor).getId(); } - public static ArrowType.Bool bool() { - return ArrowType.Bool.INSTANCE; + public static org.apache.arrow.vector.types.pojo.ArrowType.Bool bool() { + return org.apache.arrow.vector.types.pojo.ArrowType.Bool.INSTANCE; } - public static ArrowType.Int intType(int bitWidth) { - return new ArrowType.Int(bitWidth, true); + public static org.apache.arrow.vector.types.pojo.ArrowType.Int intType(int bitWidth) { + return new org.apache.arrow.vector.types.pojo.ArrowType.Int(bitWidth, true); } - public static ArrowType.Int int8() { + public static org.apache.arrow.vector.types.pojo.ArrowType.Int int8() { return intType(8); } - public static ArrowType.Int int16() { + public static org.apache.arrow.vector.types.pojo.ArrowType.Int int16() { return intType(16); } - public static ArrowType.Int int32() { + public static org.apache.arrow.vector.types.pojo.ArrowType.Int int32() { return intType(32); } - public static ArrowType.Int int64() { + public static org.apache.arrow.vector.types.pojo.ArrowType.Int int64() { return intType(64); } - public static ArrowType.FloatingPoint float32() { - return new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE); + public static org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint float32() { + return new org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint( + FloatingPointPrecision.SINGLE); } - public static ArrowType.FloatingPoint float64() { - return new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE); + public static org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint float64() { + return new org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint( + FloatingPointPrecision.DOUBLE); } - public static ArrowType.Date date32() { - return new ArrowType.Date(DateUnit.DAY); + public static org.apache.arrow.vector.types.pojo.ArrowType.Date date32() { + return new org.apache.arrow.vector.types.pojo.ArrowType.Date(DateUnit.DAY); } - public static ArrowType.Date date64() { - return new ArrowType.Date(DateUnit.MILLISECOND); + public static org.apache.arrow.vector.types.pojo.ArrowType.Date date64() { + return new org.apache.arrow.vector.types.pojo.ArrowType.Date(DateUnit.MILLISECOND); } - public static ArrowType.Timestamp timestamp() { - return new ArrowType.Timestamp(TimeUnit.MICROSECOND, null); + public static org.apache.arrow.vector.types.pojo.ArrowType.Timestamp timestamp() { + return new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(TimeUnit.MICROSECOND, null); } - public static ArrowType.Binary binary() { - return ArrowType.Binary.INSTANCE; + public static org.apache.arrow.vector.types.pojo.ArrowType.Binary binary() { + return org.apache.arrow.vector.types.pojo.ArrowType.Binary.INSTANCE; } - public static ArrowType.Utf8 utf8() { - return ArrowType.Utf8.INSTANCE; + public static org.apache.arrow.vector.types.pojo.ArrowType.Utf8 utf8() { + return org.apache.arrow.vector.types.pojo.ArrowType.Utf8.INSTANCE; } - public static ArrowType.Decimal decimal() { + public static org.apache.arrow.vector.types.pojo.ArrowType.Decimal decimal() { return decimal(DecimalUtils.MAX_PRECISION, DecimalUtils.MAX_SCALE); } - public static ArrowType.Decimal decimal(int precision, int scale) { - return new ArrowType.Decimal(precision, scale); + public static org.apache.arrow.vector.types.pojo.ArrowType.Decimal decimal( + int precision, int scale) { + return new org.apache.arrow.vector.types.pojo.ArrowType.Decimal(precision, scale); } - public static ArrowType.Decimal bigintDecimal() { + public static org.apache.arrow.vector.types.pojo.ArrowType.Decimal bigintDecimal() { return decimal(DecimalUtils.MAX_PRECISION, 0); } @@ -297,7 +306,11 @@ public static Field field(String name, FieldType fieldType) { return field(name, fieldType, Collections.emptyList()); } - public static Field field(String name, boolean nullable, ArrowType type, Field... children) { + public static Field field( + String name, + boolean nullable, + org.apache.arrow.vector.types.pojo.ArrowType type, + Field... children) { return field(name, new FieldType(nullable, type, null), children); } @@ -305,11 +318,16 @@ public static Field field(String name, FieldType fieldType, Field... children) { return field(name, fieldType, Arrays.asList(children)); } - public static Field field(String name, boolean nullable, ArrowType type, List children) { + public static Field field( + String name, + boolean nullable, + org.apache.arrow.vector.types.pojo.ArrowType type, + List children) { return field(name, new FieldType(nullable, type, null), children); } - public static Field field(String name, ArrowType type, Field... children) { + public static Field field( + String name, org.apache.arrow.vector.types.pojo.ArrowType type, Field... children) { return field(name, true, type, children); } @@ -317,34 +335,36 @@ public static Field field(String name, FieldType fieldType, List children return new Field(name, fieldType, children); } - public static Field notNullField(String name, ArrowType type, Field... children) { + public static Field notNullField( + String name, org.apache.arrow.vector.types.pojo.ArrowType type, Field... children) { return field(name, false, type, children); } - public static FieldType notNullFieldType(ArrowType type) { + public static FieldType notNullFieldType(org.apache.arrow.vector.types.pojo.ArrowType type) { return new FieldType(false, type, null); } /* ========================= array field utils ========================= */ - public static Field primitiveArrayField(ArrowType type) { + public static Field primitiveArrayField(org.apache.arrow.vector.types.pojo.ArrowType type) { return primitiveArrayField("", type); } - public static Field primitiveArrayField(String name, ArrowType type) { + public static Field primitiveArrayField( + String name, org.apache.arrow.vector.types.pojo.ArrowType type) { return field( name, - FieldType.nullable(ArrowType.List.INSTANCE), + FieldType.nullable(org.apache.arrow.vector.types.pojo.ArrowType.List.INSTANCE), Collections.singletonList(field(ARRAY_ITEM_NAME, false, type))); } - public static Field arrayField(ArrowType type) { + public static Field arrayField(org.apache.arrow.vector.types.pojo.ArrowType type) { return arrayField("", type); } - public static Field arrayField(String name, ArrowType type) { + public static Field arrayField(String name, org.apache.arrow.vector.types.pojo.ArrowType type) { return field( name, - FieldType.nullable(ArrowType.List.INSTANCE), + FieldType.nullable(org.apache.arrow.vector.types.pojo.ArrowType.List.INSTANCE), Collections.singletonList(field(ARRAY_ITEM_NAME, true, type))); } @@ -355,7 +375,7 @@ public static Field arrayField(FieldType valueType) { public static Field arrayField(String name, FieldType valueType) { return field( name, - FieldType.nullable(ArrowType.List.INSTANCE), + FieldType.nullable(org.apache.arrow.vector.types.pojo.ArrowType.List.INSTANCE), Collections.singletonList(field(ARRAY_ITEM_NAME, valueType))); } @@ -365,7 +385,9 @@ public static Field arrayField(Field valueField) { public static Field arrayField(String name, Field valueField) { return field( - name, FieldType.nullable(ArrowType.List.INSTANCE), Collections.singletonList(valueField)); + name, + FieldType.nullable(org.apache.arrow.vector.types.pojo.ArrowType.List.INSTANCE), + Collections.singletonList(valueField)); } public static Field arrayElementField(Field field) { @@ -373,11 +395,16 @@ public static Field arrayElementField(Field field) { } /* ========================= map field utils start ========================= */ - public static Field mapField(ArrowType keyType, ArrowType itemType) { + public static Field mapField( + org.apache.arrow.vector.types.pojo.ArrowType keyType, + org.apache.arrow.vector.types.pojo.ArrowType itemType) { return mapField("", keyType, itemType); } - public static Field mapField(String name, ArrowType keyType, ArrowType itemType) { + public static Field mapField( + String name, + org.apache.arrow.vector.types.pojo.ArrowType keyType, + org.apache.arrow.vector.types.pojo.ArrowType itemType) { return mapField( name, field(MapVector.KEY_NAME, false, keyType), @@ -392,7 +419,8 @@ public static Field mapField(String name, Field keyField, Field itemField) { Preconditions.checkArgument(!keyField.isNullable(), "Map's keys must be non-nullable"); // Map's key-item pairs must be non-nullable structs Field valueField = structField(false, keyField, itemField); - return field(name, true, new ArrowType.Map(false), valueField); + return field( + name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Map(false), valueField); } public static Field keyFieldForMap(Field mapField) { @@ -425,11 +453,13 @@ public static Field structField(boolean nullable, Field... fields) { } public static Field structField(String name, boolean nullable, Field... fields) { - return field(name, nullable, ArrowType.Struct.INSTANCE, fields); + return field( + name, nullable, org.apache.arrow.vector.types.pojo.ArrowType.Struct.INSTANCE, fields); } public static Field structField(String name, boolean nullable, List fields) { - return field(name, nullable, ArrowType.Struct.INSTANCE, fields); + return field( + name, nullable, org.apache.arrow.vector.types.pojo.ArrowType.Struct.INSTANCE, fields); } /* ========================= struct field utils end ========================= */ @@ -475,7 +505,7 @@ public static long computeSchemaHash(Schema schema) { } private static long computeHash(long hash, Field field) { - Type typeID = getTypeId(field.getType()); + ArrowType typeID = getTypeId(field.getType()); while (true) { try { hash = Math.addExact(Math.multiplyExact(hash, 31), (long) typeID.getId()); diff --git a/java/fury-format/src/main/java/org/apache/fury/format/vectorized/ArrowSerializers.java b/java/fury-format/src/main/java/org/apache/fury/format/vectorized/ArrowSerializers.java index efc117a695..c6f3c83f9d 100644 --- a/java/fury-format/src/main/java/org/apache/fury/format/vectorized/ArrowSerializers.java +++ b/java/fury-format/src/main/java/org/apache/fury/format/vectorized/ArrowSerializers.java @@ -39,7 +39,7 @@ import org.apache.fury.memory.Platform; import org.apache.fury.serializer.BufferObject; import org.apache.fury.serializer.Serializers.CrossLanguageCompatibleSerializer; -import org.apache.fury.type.Type; +import org.apache.fury.type.Types; /** Serializers for apache arrow. */ public class ArrowSerializers { @@ -57,7 +57,7 @@ public VectorSchemaRootSerializer(Fury fury) { } public VectorSchemaRootSerializer(Fury fury, BufferAllocator allocator) { - super(fury, VectorSchemaRoot.class, Type.FURY_ARROW_RECORD_BATCH.getId()); + super(fury, VectorSchemaRoot.class); this.allocator = allocator; } @@ -166,6 +166,8 @@ public MemoryBuffer toBuffer() { } public static void registerSerializers(Fury fury) { + fury.register(ArrowTable.class, Types.ARROW_TABLE); + fury.register(VectorSchemaRoot.class, Types.ARROW_RECORD_BATCH); fury.registerSerializer(ArrowTable.class, new ArrowTableSerializer(fury)); fury.registerSerializer(VectorSchemaRoot.class, new VectorSchemaRootSerializer(fury)); } diff --git a/java/fury-format/src/main/java/org/apache/fury/format/vectorized/ArrowTableSerializer.java b/java/fury-format/src/main/java/org/apache/fury/format/vectorized/ArrowTableSerializer.java index 451e5e07cd..25bda0e1c8 100644 --- a/java/fury-format/src/main/java/org/apache/fury/format/vectorized/ArrowTableSerializer.java +++ b/java/fury-format/src/main/java/org/apache/fury/format/vectorized/ArrowTableSerializer.java @@ -32,7 +32,6 @@ import org.apache.fury.memory.MemoryBuffer; import org.apache.fury.memory.Platform; import org.apache.fury.serializer.Serializers; -import org.apache.fury.type.Type; /** Serializers for {@link ArrowTable}. */ public class ArrowTableSerializer @@ -46,7 +45,7 @@ public ArrowTableSerializer(Fury fury) { } public ArrowTableSerializer(Fury fury, BufferAllocator allocator) { - super(fury, ArrowTable.class, Type.FURY_ARROW_TABLE.getId()); + super(fury, ArrowTable.class); this.allocator = allocator; }