Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[wip/draft] Allow remote units loader #118

Draft
wants to merge 3 commits into
base: main
Choose a base branch
from
Draft

[wip/draft] Allow remote units loader #118

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7c48b3d
draft, introduce remote loaders
polomani Nov 1, 2024
2daa94e
try to fix tests
polomani Nov 1, 2024
65d906a
move validation out of the file
polomani Nov 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
220 changes: 125 additions & 95 deletions src/main/kotlin/com/cognite/units/UnitService.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,19 +18,25 @@ package com.cognite.units
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
import com.fasterxml.jackson.module.kotlin.readValue
import java.net.URI
import java.net.URL
import java.util.concurrent.CompletableFuture
import kotlin.math.abs
import kotlin.math.ceil
import kotlin.math.log10
import kotlin.math.pow
import kotlin.math.roundToLong

class UnitService(units: String, systems: String) {

constructor(unitsPath: URL, systemPath: URL) : this(unitsPath.readText(), systemPath.readText())
class UnitService(
units: String? = null,
systems: String? = null,
private val getUnitSystemsCallback: ((String) -> CompletableFuture<String>)? = null,
private val getUnitsCallback: ((String) -> CompletableFuture<String>)? = null,
) {
constructor(unitsPath: URL, systemPath: URL) : this(units = unitsPath.readText(), systems = systemPath.readText())

companion object {
const val GLOBAL_PROJECT = "_global_"

val service: UnitService by lazy {
UnitService(
UnitService::class.java.getResource("/units.json")!!,
Expand All @@ -39,47 +45,18 @@ class UnitService(units: String, systems: String) {
}
}

private val unitsByAlias = mutableMapOf<String, ArrayList<TypedUnit>>()
private val unitsByExternalId = mutableMapOf<String, TypedUnit>()
private val unitsByQuantity = mutableMapOf<String, ArrayList<TypedUnit>>()
private val unitsByQuantityAndAlias = mutableMapOf<String, LinkedHashMap<String, TypedUnit>>()
private val defaultUnitByQuantityAndSystem = mutableMapOf<String, MutableMap<String, TypedUnit>>()
private val unitsByAlias = mutableMapOf<String, MutableMap<String, ArrayList<TypedUnit>>>()
private val unitsByExternalId = mutableMapOf<String, MutableMap<String, TypedUnit>>()
private val unitsByQuantity = mutableMapOf<String, MutableMap<String, ArrayList<TypedUnit>>>()
private val unitsByQuantityAndAlias = mutableMapOf<String, MutableMap<String, LinkedHashMap<String, TypedUnit>>>()
private val defaultUnitByQuantityAndSystem =
mutableMapOf<String, MutableMap<String, MutableMap<String, TypedUnit>>>()

init {
loadUnits(units)
loadSystem(systems)
}

private fun sanitizeIdentifier(identifier: String): String {
// remove all special characters except - and _
return identifier.lowercase().replace(Regex("[^a-z0-9_-]"), "_")
}

private fun generateExpectedExternalId(unit: TypedUnit): String {
val sanitizedQuantity = sanitizeIdentifier(unit.quantity)
val sanitizedName = sanitizeIdentifier(unit.name)
return "$sanitizedQuantity:$sanitizedName"
}

private fun generatedExpectedSourceReference(unit: TypedUnit): String? {
if (unit.source == "qudt.org") {
return "https://qudt.org/vocab/unit/${unit.name}"
if (units != null && systems != null) {
loadUnits(units, GLOBAL_PROJECT)
loadSystem(systems, GLOBAL_PROJECT)
}

val errorMessage = "Invalid sourceReference ${unit.sourceReference} for unit ${unit.name} (${unit.quantity})"

// check reference is a valid http(s) url if present
if (unit.sourceReference != null) {
try {
val url = URI.create(unit.sourceReference).toURL()
if (url.protocol != "http" && url.protocol != "https") {
throw IllegalArgumentException(errorMessage)
}
} catch (e: Exception) {
throw IllegalArgumentException(errorMessage, e)
}
}
return unit.sourceReference
}

// For a given quantity, there should not be duplicate units
Expand All @@ -91,102 +68,149 @@ class UnitService(units: String, systems: String) {
}.filter { it.value.isNotEmpty() }
}

private fun loadUnits(units: String) {
val mapper: ObjectMapper = jacksonObjectMapper()
private fun ensureProjectLoaded(projectName: String): CompletableFuture<Void> {
if (unitsByExternalId.containsKey(projectName)) {
return CompletableFuture.completedFuture(null)
}
return getUnitsCallback?.invoke(projectName)?.thenAccept { loadUnits(it, projectName) }
?: CompletableFuture.completedFuture(null)
}

private fun loadUnits(units: String, projectName: String = GLOBAL_PROJECT) {
val mapper: ObjectMapper = jacksonObjectMapper()
// 1. Syntax Check: Every unit item in `units.json` must have the specified keys
val listOfUnits: List<TypedUnit> = mapper.readValue<List<TypedUnit>>(units)
val listOfUnits: List<TypedUnit> = mapper.readValue(units)

listOfUnits.forEach {
// 2. Unique IDs: All unit `externalIds` in `units.json` must be unique
assert(unitsByExternalId[it.externalId] == null) { "Duplicate externalId ${it.externalId}" }
unitsByExternalId[it.externalId] = it

// 7. ExternalId Format: All unit `externalIds` must follow the pattern `{quantity}:{unit}`, where both
// `quantity` and `unit` are in snake_case.
assert(it.externalId == generateExpectedExternalId(it)) {
"Invalid externalId ${it.externalId} for unit ${it.name} (${it.quantity})"
assert(unitsByExternalId[projectName]?.get(it.externalId) == null) {
"Duplicate externalId ${it.externalId}"
}
unitsByExternalId.getOrPut(projectName) { mutableMapOf() }[it.externalId] = it

// if source is qudt.org, reference should be in the format https://qudt.org/vocab/unit/{unit.name}
if (it.source == "qudt.org") {
assert(it.sourceReference == generatedExpectedSourceReference(it)) {
"Invalid sourceReference ${it.sourceReference} for unit ${it.name} (${it.quantity})"
}
}
UnitUtils.validateUnit(it)

unitsByQuantity.computeIfAbsent(it.quantity) { ArrayList() }.add(it)
unitsByQuantityAndAlias.computeIfAbsent(it.quantity) { LinkedHashMap() }
unitsByQuantity.getOrPut(projectName) { mutableMapOf() }
.getOrPut(it.quantity) { ArrayList() }.add(it)
unitsByQuantityAndAlias.getOrPut(projectName) { mutableMapOf() }
.getOrPut(it.quantity) { LinkedHashMap() }
// convert to set first, to remove duplicate aliases due to encoding (e.g. "\u00b0C" vs "°C")
it.aliasNames.toSet().forEach { alias ->
unitsByAlias.computeIfAbsent(alias) { ArrayList() }.add(it)
unitsByAlias.getOrPut(projectName) { mutableMapOf() }
.getOrPut(alias) { ArrayList() }.add(it)
// 6. Unique aliases: All pairs of (alias and quantity) must be unique, for all aliases in `aliasNames`
assert(unitsByQuantityAndAlias[it.quantity]!![alias] == null) {
assert(unitsByQuantityAndAlias[projectName]!![it.quantity]!![alias] == null) {
"Duplicate alias $alias for quantity ${it.quantity}"
}
unitsByQuantityAndAlias[it.quantity]!![alias] = it
unitsByQuantityAndAlias[projectName]!![it.quantity]!![alias] = it
}
}
}

private fun loadSystem(systems: String) {
private fun loadSystem(systems: String, projectName: String = GLOBAL_PROJECT) {
val mapper: ObjectMapper = jacksonObjectMapper()
val listOfSystems: List<UnitSystem> = mapper.readValue<List<UnitSystem>>(systems)
val listOfSystems: List<UnitSystem> = mapper.readValue(systems)

listOfSystems.forEach {
val system = it.name
// check for duplicate systems
assert(defaultUnitByQuantityAndSystem[system] == null) { "Duplicate system $system" }
defaultUnitByQuantityAndSystem[system] = it.quantities.associate { sq ->
assert(defaultUnitByQuantityAndSystem.getOrPut(projectName) { mutableMapOf() }[system] == null) {
"Duplicate system $system"
}
defaultUnitByQuantityAndSystem[projectName]!![system] = it.quantities.associate { sq ->
// 3. Reference Validation: There should be no references to non-existent unit `externalIds` in
// `unitSystems.json`
val unit = getUnitByExternalId(sq.unitExternalId)
val unit = getUnitByExternalId(sq.unitExternalId, projectName).join()
// 5. Consistent References: All quantity references in `unitSystems.json` must exist in `units.json`
assert(unitsByQuantity.containsKey(sq.name)) { "Unknown quantity ${sq.name}" }
assert(unitsByQuantity[projectName]!!.containsKey(sq.name)) { "Unknown quantity ${sq.name}" }
sq.name to unit
}.toMutableMap()
}
// check if a Default system is defined
assert(defaultUnitByQuantityAndSystem.containsKey("Default")) { "Missing Default system" }
assert(defaultUnitByQuantityAndSystem[projectName]!!.containsKey("Default")) { "Missing Default system" }
// 4. Default Quantities: All quantities must be present in the `unitSystems.json` for the Default quantity
assert(defaultUnitByQuantityAndSystem["Default"]!!.size == unitsByQuantity.size) {
assert(defaultUnitByQuantityAndSystem[projectName]!!["Default"]!!.size == unitsByQuantity[projectName]!!.size) {
"Missing units in Default system"
}
}

fun getUnits(): List<TypedUnit> = unitsByExternalId.values.toList()
fun getUnits(projectName: String = GLOBAL_PROJECT): CompletableFuture<List<TypedUnit>> {
return getUnitsCallback?.invoke(projectName)?.thenApply {
loadUnits(it, projectName)
unitsByExternalId[projectName]!!.values.toList()
} ?: CompletableFuture.completedFuture(unitsByExternalId[GLOBAL_PROJECT]!!.values.toList())
}

fun getUnitSystems(): Set<String> = defaultUnitByQuantityAndSystem.keys
fun getUnitSystems(projectName: String = GLOBAL_PROJECT): CompletableFuture<Set<String>> {
return getUnitSystemsCallback?.invoke(projectName)?.thenApply {
loadSystem(it, projectName)
defaultUnitByQuantityAndSystem[projectName]!!.keys
} ?: CompletableFuture.completedFuture(defaultUnitByQuantityAndSystem[GLOBAL_PROJECT]!!.keys)
}

fun getUnitByExternalId(externalId: String): TypedUnit {
return unitsByExternalId[externalId] ?: throw IllegalArgumentException("Unknown unit '$externalId'")
fun getUnitByExternalId(externalId: String, projectName: String = GLOBAL_PROJECT): CompletableFuture<TypedUnit> {
val defaultUnit = unitsByExternalId[GLOBAL_PROJECT]?.get(externalId)
if (defaultUnit != null) {
return CompletableFuture.completedFuture(defaultUnit)
}
return ensureProjectLoaded(projectName).thenApply {
unitsByExternalId[projectName]?.get(externalId)
?: throw IllegalArgumentException("Unknown unit '$externalId'")
}
}

fun getUnitsByQuantity(quantity: String): List<TypedUnit> {
return unitsByQuantity[quantity] ?: throw IllegalArgumentException("Unknown unit quantity '$quantity'")
fun getUnitsByQuantity(quantity: String, projectName: String = GLOBAL_PROJECT): CompletableFuture<List<TypedUnit>> {
val defaultUnits = unitsByQuantity[GLOBAL_PROJECT]?.get(quantity)
if (defaultUnits != null) {
return CompletableFuture.completedFuture(defaultUnits)
}
return ensureProjectLoaded(projectName).thenApply {
unitsByQuantity[projectName]?.get(quantity)
?: throw IllegalArgumentException("Unknown unit quantity '$quantity'")
}
}

fun getUnitByQuantityAndAlias(quantity: String, alias: String): TypedUnit {
val quantityTable = unitsByQuantityAndAlias[quantity] ?: throw IllegalArgumentException(
"Unknown quantity '$quantity'",
)
return quantityTable[alias] ?: throw IllegalArgumentException(
"Unknown unit alias '$alias' for quantity '$quantity'",
)
fun getUnitByQuantityAndAlias(
quantity: String,
alias: String,
projectName: String = GLOBAL_PROJECT,
): CompletableFuture<TypedUnit> {
val defaultQuantityTable = unitsByQuantityAndAlias[GLOBAL_PROJECT]?.get(quantity)
if (defaultQuantityTable != null && defaultQuantityTable.containsKey(alias)) {
return CompletableFuture.completedFuture(defaultQuantityTable[alias]!!)
}
return ensureProjectLoaded(projectName).thenApply {
val quantityTable = unitsByQuantityAndAlias[projectName]?.get(quantity)
?: throw IllegalArgumentException("Unknown quantity '$quantity'")
quantityTable[alias]
?: throw IllegalArgumentException("Unknown unit alias '$alias' for quantity '$quantity'")
}
}

fun getUnitBySystem(sourceUnit: TypedUnit, targetSystem: String): TypedUnit {
if (!defaultUnitByQuantityAndSystem.containsKey(targetSystem)) {
throw IllegalArgumentException("Unknown system $targetSystem")
fun getUnitBySystem(
sourceUnit: TypedUnit,
targetSystem: String,
projectName: String = GLOBAL_PROJECT,
): CompletableFuture<TypedUnit> {
val defaultSystem = defaultUnitByQuantityAndSystem[GLOBAL_PROJECT]?.get(targetSystem)?.get(sourceUnit.quantity)
if (defaultSystem != null) {
return CompletableFuture.completedFuture(defaultSystem)
}
return ensureProjectLoaded(projectName).thenApply {
defaultUnitByQuantityAndSystem[projectName]?.get(targetSystem)?.get(sourceUnit.quantity)
?: throw IllegalArgumentException("Cannot convert from ${sourceUnit.quantity}")
}
return defaultUnitByQuantityAndSystem[targetSystem]!![sourceUnit.quantity]
?: defaultUnitByQuantityAndSystem["Default"]!![sourceUnit.quantity] ?: throw IllegalArgumentException(
"Cannot convert from ${sourceUnit.quantity}",
)
}

fun getUnitsByAlias(alias: String): ArrayList<TypedUnit> {
return unitsByAlias[alias] ?: throw IllegalArgumentException("Unknown alias '$alias'")
fun getUnitsByAlias(alias: String, projectName: String = GLOBAL_PROJECT): CompletableFuture<ArrayList<TypedUnit>> {
val defaultUnits = unitsByAlias[GLOBAL_PROJECT]?.get(alias)
if (defaultUnits != null) {
return CompletableFuture.completedFuture(defaultUnits)
}
return ensureProjectLoaded(projectName).thenApply {
unitsByAlias[projectName]?.get(alias)
?: throw IllegalArgumentException("Unknown alias '$alias'")
}
}

fun verifyIsConvertible(unitFrom: TypedUnit, unitTo: TypedUnit) {
Expand Down Expand Up @@ -246,8 +270,14 @@ class UnitService(units: String, systems: String) {
return shifted / magnitude
}

fun isValidUnit(unitExternalId: String): Boolean {
return unitsByExternalId.containsKey(unitExternalId)
fun isValidUnit(unitExternalId: String, projectName: String = GLOBAL_PROJECT): CompletableFuture<Boolean> {
val defaultValid = unitsByExternalId[GLOBAL_PROJECT]?.containsKey(unitExternalId) ?: false
if (defaultValid) {
return CompletableFuture.completedFuture(true)
}
return ensureProjectLoaded(projectName).thenApply {
unitsByExternalId[projectName]?.containsKey(unitExternalId) ?: false
}
}
}

Expand Down
54 changes: 54 additions & 0 deletions src/main/kotlin/com/cognite/units/UnitUtils.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
package com.cognite.units

import java.net.URI

object UnitUtils {
private fun sanitizeIdentifier(identifier: String): String {
// remove all special characters except - and _
return identifier.lowercase().replace(Regex("[^a-z0-9_-]"), "_")
}

private fun generateExpectedExternalId(unit: TypedUnit): String {
val sanitizedQuantity = sanitizeIdentifier(unit.quantity)
val sanitizedName = sanitizeIdentifier(unit.name)
return "$sanitizedQuantity:$sanitizedName"
}

fun generatedExpectedSourceReference(unit: TypedUnit): String? {
if (unit.source == "qudt.org") {
return "https://qudt.org/vocab/unit/${unit.name}"
}

val errorMessage = "Invalid sourceReference ${unit.sourceReference} for unit ${unit.name} (${unit.quantity})"

// check reference is a valid http(s) url if present
if (unit.sourceReference != null) {
try {
val url = URI.create(unit.sourceReference).toURL()
if (url.protocol != "http" && url.protocol != "https") {
throw IllegalArgumentException(errorMessage)
}
} catch (e: Exception) {
throw IllegalArgumentException(errorMessage, e)
}
}
return unit.sourceReference
}

fun validateUnit(unit: TypedUnit) {
// ExternalId Format: All unit `externalIds` must follow the pattern `{quantity}:{unit}`, where both
// `quantity` and `unit` are in snake_case.
assert(unit.externalId == generateExpectedExternalId(unit)) {
"Invalid externalId ${unit.externalId} for unit ${unit.name} (${unit.quantity})"
}

// if source is qudt.org, reference should be in the format https://qudt.org/vocab/unit/{unit.name}
if (unit.source == "qudt.org") {
assert(unit.sourceReference == generatedExpectedSourceReference(unit)) {
"Invalid sourceReference ${unit.sourceReference} for unit ${unit.name} (${unit.quantity})"
}
}

return
}
}
2 changes: 1 addition & 1 deletion src/test/kotlin/DuplicateUnitsTest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ class DuplicatedUnitsTest {

private fun getDuplicateConversions(failOnError: Boolean) {
val unitService = UnitService.service
val duplicates = unitService.getDuplicateConversions(unitService.getUnits())
val duplicates = unitService.getDuplicateConversions(unitService.getUnits().get())
// We want to filter out all units that are marked as equivalent
val newDuplicates = duplicates.mapValues {
(_, duplicatesByConversion) ->
Expand Down
Loading