Add a slightly more comprehensive (and common) implementation of the

pdvrieze · pdvrieze · commit 870f23ab857d · 2025-04-10T21:24:37.000+01:00
default EfficientBinaryFormat, and add it to the parametrizedTest. It
contains some workarounds to handle json format specifics. It
implements stringformat by recording the binary serialization as shadow
value.
diff --git a/formats/json-tests/commonTest/src/kotlinx/serialization/efficientBinaryFormat/ByteReadingBuffer.kt b/formats/json-tests/commonTest/src/kotlinx/serialization/efficientBinaryFormat/ByteReadingBuffer.kt
@@ -0,0 +1,68 @@
+/*
+ * Copyright 2017-2025 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license.
+ */
+
+package kotlinx.serialization.efficientBinaryFormat
+
+class ByteReadingBuffer(val buffer: ByteArray) {
+    private var next = 0
+
+    private fun nextByte(): Int {
+        return buffer[next++].toInt() and 0xff
+    }
+
+    private fun nextByteL(): Long {
+        return buffer[next++].toLong() and 0xffL
+    }
+
+    operator fun get(pos: Int): Byte {
+        if(pos !in 0..<buffer.size) { throw IndexOutOfBoundsException("Position $pos out of range") }
+        return buffer[pos]
+    }
+
+    fun readByte(): Byte {
+        return buffer[next++]
+    }
+
+    fun readShort(): Short {
+        return (nextByte() or (nextByte() shl 8)).toShort()
+    }
+
+    fun readInt(): Int {
+        return nextByte() or
+            (nextByte() shl 8) or
+            (nextByte() shl 16) or
+            (nextByte() shl 24)
+    }
+
+    fun readLong(): Long {
+        return nextByteL() or
+            (nextByteL() shl 8) or
+            (nextByteL() shl 16) or
+            (nextByteL() shl 24) or
+            (nextByteL() shl 32) or
+            (nextByteL() shl 40) or
+            (nextByteL() shl 48) or
+            (nextByteL() shl 56)
+    }
+
+    fun readFloat(): Float {
+        return Float.fromBits(readInt())
+    }
+
+    fun readDouble(): Double {
+        val l = readLong()
+        return Double.fromBits(l)
+    }
+
+    fun readChar(): Char {
+        return (nextByte() or (nextByte() shl 8)).toChar()
+    }
+
+    fun readString(): String {
+        val len = readInt()
+        val chars = CharArray(len) { readChar() }
+        return chars.concatToString()
+    }
+
+}
diff --git a/formats/json-tests/commonTest/src/kotlinx/serialization/efficientBinaryFormat/ByteWritingBuffer.kt b/formats/json-tests/commonTest/src/kotlinx/serialization/efficientBinaryFormat/ByteWritingBuffer.kt
@@ -0,0 +1,89 @@
+/*
+ * Copyright 2017-2025 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license.
+ */
+
+package kotlinx.serialization.efficientBinaryFormat
+
+import kotlin.experimental.and
+
+class ByteWritingBuffer() {
+    private var buffer = ByteArray(8192)
+    private var next = 0
+    val size
+        get() = next
+
+    operator fun get(pos: Int): Byte {
+        if(pos !in 0..<size) { throw IndexOutOfBoundsException("Position $pos out of range") }
+        return buffer[pos]
+    }
+
+    private fun growIfNeeded(additionalNeeded: Int = 1) {
+        val minNew = size + additionalNeeded
+        if (minNew < buffer.size) return
+
+        var newSize = buffer.size shl 1
+        while (newSize < minNew) { newSize = newSize shl 1}
+
+        buffer = buffer.copyOf(newSize)
+    }
+
+    fun toByteArray(): ByteArray {
+        return buffer.copyOf(size)
+    }
+
+    fun writeByte(b: Byte) {
+        growIfNeeded(1)
+        buffer[next++] = b
+    }
+
+    fun writeShort(s: Short) {
+        growIfNeeded(2)
+        buffer[next++] = (s and 0xff).toByte()
+        buffer[next++] = ((s.toInt() shr 8) and 0xff).toByte()
+    }
+
+    fun writeInt(i: Int) {
+        growIfNeeded(4)
+        buffer[next++] = (i and 0xff).toByte()
+        buffer[next++] = ((i shr 8) and 0xff).toByte()
+        buffer[next++] = ((i shr 16) and 0xff).toByte()
+        buffer[next++] = ((i shr 24) and 0xff).toByte()
+    }
+
+    fun writeLong(l: Long) {
+        growIfNeeded(4)
+        buffer[next++] = (l and 0xff).toByte()
+        buffer[next++] = ((l shr 8) and 0xff).toByte()
+        buffer[next++] = ((l shr 16) and 0xff).toByte()
+        buffer[next++] = ((l shr 24) and 0xff).toByte()
+        buffer[next++] = ((l shr 32) and 0xff).toByte()
+        buffer[next++] = ((l shr 40) and 0xff).toByte()
+        buffer[next++] = ((l shr 48) and 0xff).toByte()
+        buffer[next++] = ((l shr 56) and 0xff).toByte()
+    }
+
+    fun writeFloat(f: Float) {
+        writeInt(f.toBits())
+    }
+
+    fun writeDouble(d: Double) {
+        writeLong(d.toBits())
+    }
+
+    fun writeChar(c: Char) {
+        growIfNeeded(2)
+        buffer[next++] = (c.code and 0xff).toByte()
+        buffer[next++] = ((c.code shr 8) and 0xff).toByte()
+    }
+
+    fun writeString(s: String) {
+        growIfNeeded(s.length * 2+4)
+        writeInt(s.length)
+        for (c in s) {
+            buffer[next++] = (c.code and 0xff).toByte()
+            buffer[next++] = ((c.code shr 8) and 0xff).toByte()
+        }
+    }
+
+}
+
diff --git a/formats/json-tests/commonTest/src/kotlinx/serialization/efficientBinaryFormat/EfficientBinaryFormat.kt b/formats/json-tests/commonTest/src/kotlinx/serialization/efficientBinaryFormat/EfficientBinaryFormat.kt
@@ -0,0 +1,118 @@
+/*
+ * Copyright 2017-2025 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license.
+ */
+
+package kotlinx.serialization.efficientBinaryFormat
+
+import kotlinx.serialization.BinaryFormat
+import kotlinx.serialization.DeserializationStrategy
+import kotlinx.serialization.ExperimentalSerializationApi
+import kotlinx.serialization.SerializationStrategy
+import kotlinx.serialization.descriptors.SerialDescriptor
+import kotlinx.serialization.encoding.AbstractDecoder
+import kotlinx.serialization.encoding.AbstractEncoder
+import kotlinx.serialization.encoding.CompositeDecoder
+import kotlinx.serialization.encoding.CompositeEncoder
+import kotlinx.serialization.modules.EmptySerializersModule
+import kotlinx.serialization.modules.SerializersModule
+
+class EfficientBinaryFormat(
+    override val serializersModule: SerializersModule = EmptySerializersModule(),
+): BinaryFormat {
+
+    override fun <T> encodeToByteArray(
+        serializer: SerializationStrategy<T>,
+        value: T
+    ): ByteArray {
+        val encoder = Encoder(serializersModule)
+        serializer.serialize(encoder, value)
+        return encoder.byteBuffer.toByteArray()
+    }
+
+    override fun <T> decodeFromByteArray(
+        deserializer: DeserializationStrategy<T>,
+        bytes: ByteArray
+    ): T {
+        val decoder = Decoder(serializersModule, bytes)
+        return deserializer.deserialize(decoder)
+    }
+
+    class Encoder(override val serializersModule: SerializersModule): AbstractEncoder() {
+        val byteBuffer = ByteWritingBuffer()
+        override fun encodeBoolean(value: Boolean) = byteBuffer.writeByte(if (value) 1 else 0)
+        override fun encodeByte(value: Byte) = byteBuffer.writeByte(value)
+        override fun encodeShort(value: Short) = byteBuffer.writeShort(value)
+        override fun encodeInt(value: Int) = byteBuffer.writeInt(value)
+        override fun encodeLong(value: Long) = byteBuffer.writeLong(value)
+        override fun encodeFloat(value: Float) = byteBuffer.writeFloat(value)
+        override fun encodeDouble(value: Double) = byteBuffer.writeDouble(value)
+        override fun encodeChar(value: Char) = byteBuffer.writeChar(value)
+        override fun encodeString(value: String) = byteBuffer.writeString(value)
+        override fun encodeEnum(enumDescriptor: SerialDescriptor, index: Int) = byteBuffer.writeInt(index)
+
+        @ExperimentalSerializationApi
+        override fun shouldEncodeElementDefault(descriptor: SerialDescriptor, index: Int): Boolean = true
+
+        override fun beginCollection(descriptor: SerialDescriptor, collectionSize: Int): CompositeEncoder {
+            encodeInt(collectionSize)
+            return this
+        }
+
+        override fun encodeNull() = encodeBoolean(false)
+        override fun encodeNotNullMark() = encodeBoolean(true)
+
+    }
+
+    class Decoder(override val serializersModule: SerializersModule, private val reader: ByteReadingBuffer) : AbstractDecoder() {
+
+        constructor(serializersModule: SerializersModule, bytes: ByteArray) : this(
+            serializersModule,
+            ByteReadingBuffer(bytes)
+        )
+
+        private var nextElementIndex = 0
+//        private var currentDesc: SerialDescriptor? = null
+
+        override fun decodeBoolean(): Boolean = reader.readByte().toInt() != 0
+
+        override fun decodeByte(): Byte = reader.readByte()
+
+        override fun decodeShort(): Short = reader.readShort()
+
+        override fun decodeInt(): Int = reader.readInt()
+
+        override fun decodeLong(): Long = reader.readLong()
+
+        override fun decodeFloat(): Float = reader.readFloat()
+
+        override fun decodeDouble(): Double = reader.readDouble()
+
+        override fun decodeChar(): Char = reader.readChar()
+
+        override fun decodeString(): String = reader.readString()
+
+        override fun decodeEnum(enumDescriptor: SerialDescriptor): Int = reader.readInt()
+
+        override fun decodeNotNullMark(): Boolean = decodeBoolean()
+
+        @ExperimentalSerializationApi
+        override fun decodeSequentially(): Boolean = true
+
+        override fun decodeCollectionSize(descriptor: SerialDescriptor): Int = reader.readInt()
+
+        override fun beginStructure(descriptor: SerialDescriptor): CompositeDecoder {
+            return Decoder(serializersModule, reader)
+        }
+
+        override fun endStructure(descriptor: SerialDescriptor) {
+            check(nextElementIndex ==0 || descriptor.elementsCount == nextElementIndex) { "Type: ${descriptor.serialName} not fully read: ${descriptor.elementsCount} != $nextElementIndex" }
+        }
+
+        override fun decodeElementIndex(descriptor: SerialDescriptor): Int {
+            return when (nextElementIndex) {
+                descriptor.elementsCount -> CompositeDecoder.DECODE_DONE
+                else -> nextElementIndex++
+            }
+        }
+    }
+}
diff --git a/formats/json-tests/commonTest/src/kotlinx/serialization/json/JsonTestBase.kt b/formats/json-tests/commonTest/src/kotlinx/serialization/json/JsonTestBase.kt
@@ -6,6 +6,7 @@ package kotlinx.serialization.json
 
 import kotlinx.io.*
 import kotlinx.serialization.*
+import kotlinx.serialization.efficientBinaryFormat.EfficientBinaryFormat
 import kotlinx.serialization.json.internal.*
 import kotlinx.serialization.json.io.*
 import kotlinx.serialization.json.okio.decodeFromBufferedSource
@@ -129,12 +130,67 @@ abstract class JsonTestBase {
         }
     }
 
+    /** A test runner that effectively handles the json tests to also test serialization to
+     * "efficient" binary. This mainly checks serializer implementations.
+     */
+    private inner class EfficientBinary(
+        val json: Json,
+        val ebf: EfficientBinaryFormat = EfficientBinaryFormat(),
+    ) : StringFormat {
+        override val serializersModule: SerializersModule = ebf.serializersModule
+
+        private var bytes: ByteArray? = null
+        private var jsonStr: String? = null
+
+        @OptIn(ExperimentalStdlibApi::class)
+        override fun <T> encodeToString(serializer: SerializationStrategy<T>, value: T): String {
+            bytes = runCatching { ebf.encodeToByteArray(serializer, value) }
+                .onFailure { if ("Json format" !in it.message!!) throw it }
+                .getOrNull()
+            return json.encodeToString(serializer, value).also {
+                if (bytes != null) jsonStr = it
+            }
+        }
+
+        @OptIn(ExperimentalStdlibApi::class)
+        override fun <T> decodeFromString(deserializer: DeserializationStrategy<T>, string: String): T {
+            /*
+             * to retain compatibility with json we support different cases. If
+             * the string has been encoded already use that. Instead, if the
+             * deserializer is also a serializer (the default) then use that to
+             * get the value from json and encode that to bytes which are then
+             * decoded. In this case capture and ignore cases that require a
+             * json encoder.
+             *
+             * Finally fall back to json decoding (nothing can be done)
+             */
+
+            var bytes = this@EfficientBinary.bytes
+            if (string == jsonStr && bytes != null) {
+                return ebf.decodeFromByteArray(deserializer, bytes)
+            } else if (deserializer is SerializationStrategy<*>) {
+                val value = json.decodeFromString(deserializer, string)
+                //
+                @Suppress("UNCHECKED_CAST")
+                runCatching { ebf.encodeToByteArray(deserializer as SerializationStrategy<T>, value) }.onSuccess { r ->
+                    bytes = r
+                    jsonStr = string
+                    return ebf.decodeFromByteArray(deserializer, bytes)
+                }.onFailure { e ->
+                    if ("Json format" !in e.message!!) throw e
+                }
+            }
+            return json.decodeFromString(deserializer, string)
+        }
+    }
+
     protected fun parametrizedTest(json: Json, test: StringFormat.() -> Unit) {
         val streamingResult = runCatching { SwitchableJson(json, JsonTestingMode.STREAMING).test() }
         val treeResult = runCatching { SwitchableJson(json, JsonTestingMode.TREE).test() }
         val okioResult = runCatching { SwitchableJson(json, JsonTestingMode.OKIO_STREAMS).test() }
         val kxioResult = runCatching { SwitchableJson(json, JsonTestingMode.KXIO_STREAMS).test() }
-        processResults(listOf(streamingResult, treeResult, okioResult, kxioResult))
+        val efficientBinaryResult = runCatching { EfficientBinary(json).test() }
+        processResults(listOf(streamingResult, treeResult, okioResult, kxioResult, efficientBinaryResult))
     }
 
     protected fun processResults(results: List<Result<*>>) {
