Skip to content

Add batch bulk insert, refactor insert #14

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 4 commits into
base: dev-0.5.0
Choose a base branch
from
Draft

Add batch bulk insert, refactor insert #14

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
81e55c4
Fix List Query 0.5 branch
Sylvyrfysh Oct 11, 2019
fc4b633
Adds batch and bulk insert operations
Sylvyrfysh Oct 11, 2019
3a0e113
Acutally add items on batch insert
Sylvyrfysh Oct 12, 2019
a14a9b8
Merge branch 'dev-0.5.0' into add-batch-bulk-insert
Sylvyrfysh Sep 19, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions src/main/kotlin/com/vladsch/kotlin/jdbc/Model.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,10 @@ abstract class Model<M : Model<M, D>, D>(session: Session?, sqlTable: String, db
return _db.appendSelectSql(this)
}

fun bulkInsert(items: List<M>) = _db.bulkInsert(items)
fun batchInsert(items: List<M>) = _db.batchInsert(items)
fun bulkInsertIgnoreKeys(items: List<M>) = _db.bulkInsertIgnoreKeys(items)
fun batchInsertIgnoreKeys(items: List<M>) = _db.batchInsertIgnoreKeys(items)
fun insert() = _db.insert()
fun insertIgnoreKeys() = _db.insertIgnoreKeys()
fun select() = _db.select()
Expand Down
161 changes: 125 additions & 36 deletions src/main/kotlin/com/vladsch/kotlin/jdbc/ModelProperties.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import kotlin.reflect.KMutableProperty
import kotlin.reflect.KProperty
import kotlin.reflect.KVisibility

class ModelProperties<M>(val session: Session, val tableName: String, val dbCase: Boolean, val allowSetAuto: Boolean = false, quote: String? = null) : InternalModelPropertyProvider<M> {
class ModelProperties<M : Model<*, *>>(val session: Session, val tableName: String, val dbCase: Boolean, val allowSetAuto: Boolean = false, quote: String? = null) : InternalModelPropertyProvider<M> {
private val properties = HashMap<String, Any?>()
private val kProperties = ArrayList<KProperty<*>>()
private val propertyTypes = HashMap<String, PropertyType>()
Expand Down Expand Up @@ -325,43 +325,12 @@ class ModelProperties<M>(val session: Session, val tableName: String, val dbCase

fun sqlInsertQuery(): SqlQuery {
val sb = StringBuilder()
val sbValues = StringBuilder()
val params = ArrayList<Any?>()
var sep = ""

sb.append("INSERT INTO ")
appendQuoted(sb, tableName).append(" (")

for (prop in kProperties) {
val propType = propertyTypes[prop.name] ?: PropertyType.PROPERTY

if (!propType.isAuto) {
if (properties.containsKey(prop.name) || propertyDefaults.containsKey(prop.name)) {
val propValue = properties[prop.name]
// skip null properties which have defaults (null means use default)
if (propValue != null || !propType.isDefault) {
val columnName = columnNames[prop.name] ?: prop.name
sb.append(sep)
appendQuoted(sb, columnName)
sbValues.append(sep).append("?")
sep = ", "
params.add(propValue)
} else if (propertyDefaults.containsKey(prop.name)) {
val columnName = columnNames[prop.name] ?: prop.name
sb.append(sep)
appendQuoted(sb, columnName)
sbValues.append(sep).append("?")
sep = ", "
params.add(propertyDefaults[prop.name])
}
} else if (!prop.returnType.isMarkedNullable && !propType.isDefault) {
throw IllegalStateException("$tableName.${prop.name} property is not nullable nor default and not defined in ${this}")
}
}
}
sb.append(getBaseInsertString())
sb.append(" VALUES ")
sb.append(getInsertQuestionMarks())

sb.append(") VALUES (").append(sbValues).append(")")
return sqlQuery(sb.toString(), params)
return sqlQuery(sb.toString(), generateInsertString())
}

@Suppress("MemberVisibilityCanBePrivate")
Expand Down Expand Up @@ -525,6 +494,126 @@ class ModelProperties<M>(val session: Session, val tableName: String, val dbCase
session.update(sqlInsertQuery())
}

fun bulkInsert(items: List<M>) {
bulkInsertIgnoreKeys(items)
items.forEach { it._db.select() }
}

fun bulkInsertIgnoreKeys(items: List<M>) {
val sb = StringBuilder()

sb.append(getBaseInsertString())
sb.append(" VALUES ")

val s = items.map { it._db.generateInsertString() }

val appStr = ArrayList<String>().apply { repeat(s.size) { add(getInsertQuestionMarks()) } }.joinToString(separator = ",")
val allItems = s.reduce { acc, arrayList -> ArrayList(acc + arrayList) }

session.execute(sqlQuery(sb.append(appStr).toString(), allItems))
}

fun batchInsert(items: List<M>) {
batchInsertIgnoreKeys(items)
items.forEach { it._db.select() }
}

fun batchInsertIgnoreKeys(items: List<M>) {
val sb = StringBuilder()

sb.append(getBaseInsertString())
sb.append(" VALUES ")
sb.append(getInsertQuestionMarks())

val stmt = session.prepare(sqlQuery(sb.toString()))

items.map { it._db.generateInsertString() }.forEach {
it.forEachIndexed { index, value ->
stmt.setTypedParam(index + 1, value.param())
}
stmt.addBatch()
}

stmt.executeBatch()
}

internal fun generateInsertString(): ArrayList<Any?> {
val params = ArrayList<Any?>()

for (prop in kProperties) {
val propType = propertyTypes[prop.name] ?: PropertyType.PROPERTY

if (!propType.isAuto) {
if (properties.containsKey(prop.name) || propertyDefaults.containsKey(prop.name)) {
val propValue = properties[prop.name]
// skip null properties which have defaults (null means use default)
if (propValue != null || !propType.isDefault) {
params.add(propValue)
} else if (propertyDefaults.containsKey(prop.name)) {
params.add(propertyDefaults[prop.name])
}
} else if (!prop.returnType.isMarkedNullable && !propType.isDefault) {
throw IllegalStateException("$tableName.${prop.name} property is not nullable nor default and not defined in ${this}")
}
}
}
return params
}

internal fun getInsertQuestionMarks(): String {
val sbValues = StringBuilder().append('(')
var sep = ""

for (prop in kProperties) {
val propType = propertyTypes[prop.name] ?: PropertyType.PROPERTY

if (!propType.isAuto) {
if (properties.containsKey(prop.name) || propertyDefaults.containsKey(prop.name)) {
sbValues.append(sep).append("?")
sep = ", "
} else if (!prop.returnType.isMarkedNullable && !propType.isDefault) {
throw IllegalStateException("$tableName.${prop.name} property is not nullable nor default and not defined in ${this}")
}
}
}

return sbValues.append(')').toString()
}

internal fun getBaseInsertString(): String {
val sb = StringBuilder()
var sep = ""

sb.append("INSERT INTO ")
appendQuoted(sb, tableName).append(" (")

for (prop in kProperties) {
val propType = propertyTypes[prop.name] ?: PropertyType.PROPERTY

if (!propType.isAuto) {
if (properties.containsKey(prop.name) || propertyDefaults.containsKey(prop.name)) {
val propValue = properties[prop.name]
// skip null properties which have defaults (null means use default)
if (propValue != null || !propType.isDefault) {
val columnName = columnNames[prop.name] ?: prop.name
sb.append(sep)
appendQuoted(sb, columnName)
sep = ", "
} else if (propertyDefaults.containsKey(prop.name)) {
val columnName = columnNames[prop.name] ?: prop.name
sb.append(sep)
appendQuoted(sb, columnName)
sep = ", "
}
} else if (!prop.returnType.isMarkedNullable && !propType.isDefault) {
throw IllegalStateException("$tableName.${prop.name} property is not nullable nor default and not defined in ${this}")
}
}
}

return sb.append(')').toString()
}

fun select() {
session.first(sqlSelectQuery()) {
load(it)
Expand Down
10 changes: 5 additions & 5 deletions src/main/kotlin/com/vladsch/kotlin/jdbc/ModelPropertyProviderBase.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ package com.vladsch.kotlin.jdbc

import kotlin.reflect.KProperty

open class ModelPropertyProviderBase<T>(val provider: ModelProperties<T>, val propType: PropertyType, override val columnName: String?, override val defaultValue:Any? = Unit) : InternalModelPropertyProvider<T> {
open class ModelPropertyProviderBase<T: Model<*, *>>(val provider: ModelProperties<T>, val propType: PropertyType, override val columnName: String?, override val defaultValue:Any? = Unit) : InternalModelPropertyProvider<T> {
final override fun provideDelegate(thisRef: T, prop: KProperty<*>): ModelProperties<T> {
return provider.registerProp(prop, propType, columnName, defaultValue)
}
Expand Down Expand Up @@ -44,7 +44,7 @@ open class ModelPropertyProviderBase<T>(val provider: ModelProperties<T>, val pr
}
}

class ModelPropertyProviderAutoKey<T>(provider: ModelProperties<T>, columnName: String?) : ModelPropertyProviderBase<T>(provider, PropertyType.AUTO_KEY, columnName) {
class ModelPropertyProviderAutoKey<T: Model<*, *>>(provider: ModelProperties<T>, columnName: String?) : ModelPropertyProviderBase<T>(provider, PropertyType.AUTO_KEY, columnName) {
override val key: ModelPropertyProvider<T>
get() = this

Expand Down Expand Up @@ -72,7 +72,7 @@ class ModelPropertyProviderAutoKey<T>(provider: ModelProperties<T>, columnName:
}
}

class ModelPropertyProviderKey<T>(provider: ModelProperties<T>, columnName: String?) : ModelPropertyProviderBase<T>(provider, PropertyType.KEY, columnName) {
class ModelPropertyProviderKey<T: Model<*, *>>(provider: ModelProperties<T>, columnName: String?) : ModelPropertyProviderBase<T>(provider, PropertyType.KEY, columnName) {
override val key: ModelPropertyProvider<T>
get() = this

Expand Down Expand Up @@ -100,7 +100,7 @@ class ModelPropertyProviderKey<T>(provider: ModelProperties<T>, columnName: Stri
}
}

class ModelPropertyProviderAuto<T>(provider: ModelProperties<T>, columnName: String?) : ModelPropertyProviderBase<T>(provider, PropertyType.AUTO, columnName) {
class ModelPropertyProviderAuto<T: Model<*, *>>(provider: ModelProperties<T>, columnName: String?) : ModelPropertyProviderBase<T>(provider, PropertyType.AUTO, columnName) {
override val key: ModelPropertyProvider<T>
get() = provider.autoKey

Expand Down Expand Up @@ -128,7 +128,7 @@ class ModelPropertyProviderAuto<T>(provider: ModelProperties<T>, columnName: Str
}
}

class ModelPropertyProviderDefault<T>(provider: ModelProperties<T>, columnName: String?, value: Any?) : ModelPropertyProviderBase<T>(provider, PropertyType.DEFAULT, columnName, value) {
class ModelPropertyProviderDefault<T: Model<*, *>>(provider: ModelProperties<T>, columnName: String?, value: Any?) : ModelPropertyProviderBase<T>(provider, PropertyType.DEFAULT, columnName, value) {
override val key: ModelPropertyProvider<T>
get() = provider.autoKey

Expand Down