From 0a1154c9c9d78ba03d62f02695aab8619eebc92d Mon Sep 17 00:00:00 2001 From: lizk886 Date: Fri, 26 Jul 2024 14:38:28 -0600 Subject: [PATCH 001/431] configs+initiate s3 key provider --- conf/default-config.json | 1 + conf/docker-config.json | 1 + conf/integ-config.json | 3 +- conf/local-config.json | 1 + conf/local-e2e-docker-private-config.json | 1 + conf/local-e2e-docker-public-config.json | 1 + conf/local-e2e-private-config.json | 1 + conf/local-e2e-public-config.json | 1 + ...dator-latest-e2e-docker-public-config.json | 1 + pom.xml | 2 +- src/main/java/com/uid2/operator/Main.java | 4 + .../test/s3encryption_keys/metadata.json | 7 ++ .../s3encryption_keys/s3encryption_keys.json | 73 +++++++++++++++++++ 13 files changed, 95 insertions(+), 2 deletions(-) create mode 100644 src/main/resources/com.uid2.core/test/s3encryption_keys/metadata.json create mode 100644 src/main/resources/com.uid2.core/test/s3encryption_keys/s3encryption_keys.json diff --git a/conf/default-config.json b/conf/default-config.json index f8f1a2bcc..79d72dc51 100644 --- a/conf/default-config.json +++ b/conf/default-config.json @@ -30,6 +30,7 @@ "salts_metadata_path": "salts/metadata.json", "services_metadata_path": "services/metadata.json", "service_links_metadata_path": "service_links/metadata.json", + "s3_keys_metadata_path": "s3encryption_keys/metadata.json", "optout_metadata_path": null, "optout_inmem_cache": false, "enclave_platform": null, diff --git a/conf/docker-config.json b/conf/docker-config.json index e97d64187..64e54a8f1 100644 --- a/conf/docker-config.json +++ b/conf/docker-config.json @@ -32,6 +32,7 @@ "salts_metadata_path": "/com.uid2.core/test/salts/metadata.json", "services_metadata_path": "/com.uid2.core/test/services/metadata.json", "service_links_metadata_path": "/com.uid2.core/test/service_links/metadata.json", + "s3_keys_metadata_path": "/com.uid2.core/test/s3encryption_keys/metadata.json", "identity_token_expires_after_seconds": 3600, "optout_metadata_path": null, "optout_inmem_cache": false, diff --git a/conf/integ-config.json b/conf/integ-config.json index c16a16d9a..0b6a83b92 100644 --- a/conf/integ-config.json +++ b/conf/integ-config.json @@ -13,5 +13,6 @@ "core_api_token": "trusted-partner-key", "optout_api_token": "test-operator-key", "optout_api_uri": "http://localhost:8081/optout/replicate", - "salts_expired_shutdown_hours": 12 + "salts_expired_shutdown_hours": 12, + "s3_keys_metadata_path": "http://localhost:8088/s3encryption_keys/retrieve" } \ No newline at end of file diff --git a/conf/local-config.json b/conf/local-config.json index 309bd00b3..6c46e0130 100644 --- a/conf/local-config.json +++ b/conf/local-config.json @@ -9,6 +9,7 @@ "salts_metadata_path": "/com.uid2.core/test/salts/metadata.json", "services_metadata_path": "/com.uid2.core/test/services/metadata.json", "service_links_metadata_path": "/com.uid2.core/test/service_links/metadata.json", + "s3_keys_metadata_path":"/com.uid2.core/test/s3encryption_keys/metadata.json", "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, diff --git a/conf/local-e2e-docker-private-config.json b/conf/local-e2e-docker-private-config.json index b777533fe..ff3eb87f8 100644 --- a/conf/local-e2e-docker-private-config.json +++ b/conf/local-e2e-docker-private-config.json @@ -11,6 +11,7 @@ "keysets_metadata_path": "http://core:8088/key/keyset/refresh", "keyset_keys_metadata_path": "http://core:8088/key/keyset-keys/refresh", "salts_metadata_path": "http://core:8088/salt/refresh", + "s3_keys_metadata_path": "https://core:8088/s3encryption_keys/retrieve", "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, diff --git a/conf/local-e2e-docker-public-config.json b/conf/local-e2e-docker-public-config.json index 5b44ea981..4c7e9ff1c 100644 --- a/conf/local-e2e-docker-public-config.json +++ b/conf/local-e2e-docker-public-config.json @@ -13,6 +13,7 @@ "salts_metadata_path": "http://core:8088/salt/refresh", "services_metadata_path": "http://core:8088/services/refresh", "service_links_metadata_path": "http://core:8088/service_links/refresh", + "s3_keys_metadata_path": "https://core:8088/s3encryption_keys/retrieve", "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, diff --git a/conf/local-e2e-private-config.json b/conf/local-e2e-private-config.json index 920a159e7..af7af641d 100644 --- a/conf/local-e2e-private-config.json +++ b/conf/local-e2e-private-config.json @@ -13,6 +13,7 @@ "salts_metadata_path": "http://localhost:8088/salt/refresh", "services_metadata_path": "http://localhost:8088/services/refresh", "service_links_metadata_path": "http://localhost:8088/service_links/refresh", + "s3_keys_metadata_path": "https://core:8088/s3encryption_keys/retrieve", "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, diff --git a/conf/local-e2e-public-config.json b/conf/local-e2e-public-config.json index d36dc9139..e53522ec8 100644 --- a/conf/local-e2e-public-config.json +++ b/conf/local-e2e-public-config.json @@ -13,6 +13,7 @@ "salts_metadata_path": "http://localhost:8088/salt/refresh", "services_metadata_path": "http://localhost:8088/services/refresh", "service_links_metadata_path": "http://localhost:8088/service_links/refresh", + "s3_keys_metadata_path": "https://core:8088/s3encryption_keys/retrieve", "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, diff --git a/conf/validator-latest-e2e-docker-public-config.json b/conf/validator-latest-e2e-docker-public-config.json index d6789bd00..1901d4a86 100644 --- a/conf/validator-latest-e2e-docker-public-config.json +++ b/conf/validator-latest-e2e-docker-public-config.json @@ -14,6 +14,7 @@ "salts_metadata_path": "http://core:8088/salt/refresh", "services_metadata_path": "http://core:8088/services/refresh", "service_links_metadata_path": "http://core:8088/service_links/refresh", + "s3_keys_metadata_path": "https://core:8088/s3encryption_keys/retrieve", "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, diff --git a/pom.xml b/pom.xml index 69fb85ac2..07119f6bf 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ 2.0.0-f7c174410e 2.0.4-ef52553c57 2.0.0-21f950573a - 7.9.0 + 7.11.4 ${project.version} 21 21 diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index 37e06fbc0..d865a8d0a 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -56,6 +56,7 @@ import java.util.*; import java.util.function.Supplier; + import static io.micrometer.core.instrument.Metrics.globalRegistry; public class Main { @@ -80,6 +81,7 @@ public class Main { private IStatsCollectorQueue _statsCollectorQueue; private RotatingServiceStore serviceProvider; private RotatingServiceLinkStore serviceLinkProvider; + private RotatingS3KeyProvider s3KeyProvider; public Main(Vertx vertx, JsonObject config) throws Exception { this.vertx = vertx; @@ -143,6 +145,8 @@ public Main(Vertx vertx, JsonObject config) throws Exception { String saltsMdPath = this.config.getString(Const.Config.SaltsMetadataPathProp); this.saltProvider = new RotatingSaltProvider(fsStores, saltsMdPath); this.optOutStore = new CloudSyncOptOutStore(vertx, fsLocal, this.config, operatorKey, Clock.systemUTC()); + String s3KeyMdPath = this.config.getString(Const.Config.S3keysMetadataPathProp); + this.s3KeyProvider = new RotatingS3KeyProvider(fsStores, new GlobalScope(new CloudPath(s3KeyMdPath))); if (this.validateServiceLinks) { String serviceMdPath = this.config.getString(Const.Config.ServiceMetadataPathProp); diff --git a/src/main/resources/com.uid2.core/test/s3encryption_keys/metadata.json b/src/main/resources/com.uid2.core/test/s3encryption_keys/metadata.json new file mode 100644 index 000000000..4a667ec4c --- /dev/null +++ b/src/main/resources/com.uid2.core/test/s3encryption_keys/metadata.json @@ -0,0 +1,7 @@ +{ + "version": 1, + "generated": 1620253519, + "s3encryption_keys": { + "location": "/com.uid2.core/test/s3encryption_keys/s3encryption_keys.json" + } +} \ No newline at end of file diff --git a/src/main/resources/com.uid2.core/test/s3encryption_keys/s3encryption_keys.json b/src/main/resources/com.uid2.core/test/s3encryption_keys/s3encryption_keys.json new file mode 100644 index 000000000..b875d6797 --- /dev/null +++ b/src/main/resources/com.uid2.core/test/s3encryption_keys/s3encryption_keys.json @@ -0,0 +1,73 @@ +[ { + "id" : 1, + "siteId" : 999, + "activates" : 1720641670, + "created" : 1720641670, + "secret" : "mydrCudb2PZOm01Qn0SpthltmexHUAA11Hy1m+uxjVw=" +}, { + "id" : 2, + "siteId" : 999, + "activates" : 1720728070, + "created" : 1720641670, + "secret" : "FtdslrFSsvVXOuhOWGwEI+0QTkCvM8SGZAP3k2u3PgY=" +}, { + "id" : 3, + "siteId" : 999, + "activates" : 1720814470, + "created" : 1720641670, + "secret" : "/7zO6QbKrhZKIV36G+cU9UR4hZUVg5bD+KjbczICjHw=" +}, { + "id" : 4, + "siteId" : 123, + "activates" : 1720641671, + "created" : 1720641671, + "secret" : "XjiqRlWQQJGLr7xfV1qbueKwyzt881GVohuUkQt/ht4=" +}, { + "id" : 5, + "siteId" : 123, + "activates" : 1720728071, + "created" : 1720641671, + "secret" : "QmpIf5NzO+UROjl5XjB/BmF6paefM8n6ub9B2plC9aI=" +}, { + "id" : 6, + "siteId" : 123, + "activates" : 1720814471, + "created" : 1720641671, + "secret" : "40w9UMSYxGm+KldOWOXhBGI8QgjvUUQjivtkP4VpKV8=" +}, { + "id" : 7, + "siteId" : 124, + "activates" : 1720641671, + "created" : 1720641671, + "secret" : "QdwD0kQV1BwmLRD0PH1YpqgaOrgpVTfu08o98mSZ6uE=" +}, { + "id" : 8, + "siteId" : 124, + "activates" : 1720728071, + "created" : 1720641671, + "secret" : "yCVCM/HLf9/6k+aUNrx7w17VbyfSzI8JykLQLSR+CW0=" +}, { + "id" : 9, + "siteId" : 124, + "activates" : 1720814471, + "created" : 1720641671, + "secret" : "JqHl8BrTyx9XpR2lYj/5xvUpzgnibGeomETTwF4rn1U=" +}, { + "id" : 10, + "siteId" : 127, + "activates" : 1720641671, + "created" : 1720641671, + "secret" : "JqiG1b34AvrdO3Aj6cCcjOBJMijrDzTmrR+p9ZtP2es=" +}, { + "id" : 11, + "siteId" : 127, + "activates" : 1720728072, + "created" : 1720641672, + "secret" : "lp1CyHdfc7K0aO5JGpA+Ve5Z/V5LImtGEQwCg/YB0kY=" +}, { + "id" : 12, + "siteId" : 127, + "activates" : 1720814472, + "created" : 1720641672, + "secret" : "G99rFYJF+dnSlk/xG6fuC3WNqQxTLJbDIdVyPMbGQ6s=" +} ] \ No newline at end of file From 582ea54db5833223ff2aac1ccd2b2f7962b50845 Mon Sep 17 00:00:00 2001 From: lizk886 Date: Tue, 30 Jul 2024 10:28:28 -0600 Subject: [PATCH 002/431] small changes --- src/main/java/com/uid2/operator/Main.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index d865a8d0a..2cc20f761 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -166,6 +166,7 @@ public Main(Vertx vertx, JsonObject config) throws Exception { this.saltProvider.loadContent(); this.keysetProvider.loadContent(); this.keysetKeyStore.loadContent(); + this.s3KeyProvider.loadContent(); if (this.validateServiceLinks) { this.serviceProvider.loadContent(); @@ -333,6 +334,7 @@ private Future createStoreVerticles() throws Exception { fs.add(createAndDeployRotatingStoreVerticle("auth", clientKeyProvider, "auth_refresh_ms")); fs.add(createAndDeployRotatingStoreVerticle("keyset", keysetProvider, "keyset_refresh_ms")); fs.add(createAndDeployRotatingStoreVerticle("keysetkey", keysetKeyStore, "keysetkey_refresh_ms")); + fs.add(createAndDeployRotatingStoreVerticle("s3encryption_keys", s3KeyProvider, "s3keys_refresh_ms")); fs.add(createAndDeployRotatingStoreVerticle("salt", saltProvider, "salt_refresh_ms")); fs.add(createAndDeployCloudSyncStoreVerticle("optout", fsOptOut, optOutCloudSync)); CompositeFuture.all(fs).onComplete(ar -> { From 590b67e7a3ad4e5cde619481dcaba749ac095d2c Mon Sep 17 00:00:00 2001 From: lizk886 Date: Tue, 30 Jul 2024 10:47:56 -0600 Subject: [PATCH 003/431] update client --- src/main/java/com/uid2/operator/Main.java | 2 +- .../java/com/uid2/operator/vertx/UIDOperatorVerticle.java | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index 2cc20f761..2670d6e12 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -270,7 +270,7 @@ private ICloudStorage wrapCloudStorageForOptOut(ICloudStorage cloudStorage) { private void run() throws Exception { Supplier operatorVerticleSupplier = () -> { - UIDOperatorVerticle verticle = new UIDOperatorVerticle(config, this.clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, getKeyManager(), saltProvider, optOutStore, Clock.systemUTC(), _statsCollectorQueue, new SecureLinkValidatorService(this.serviceLinkProvider, this.serviceProvider), this.shutdownHandler::handleSaltRetrievalResponse); + UIDOperatorVerticle verticle = new UIDOperatorVerticle(config, this.clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, getKeyManager(), saltProvider, optOutStore, Clock.systemUTC(), _statsCollectorQueue, new SecureLinkValidatorService(this.serviceLinkProvider, this.serviceProvider), this.shutdownHandler::handleSaltRetrievalResponse, s3KeyProvider); return verticle; }; diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index 4a7a66e5f..9d979bc0c 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -28,6 +28,7 @@ import com.uid2.shared.store.IClientKeyProvider; import com.uid2.shared.store.IClientSideKeypairStore; import com.uid2.shared.store.ISaltProvider; +import com.uid2.shared.store.reader.RotatingS3KeyProvider; import com.uid2.shared.vertx.RequestCapturingHandler; import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.DistributionSummary; @@ -126,6 +127,8 @@ public class UIDOperatorVerticle extends AbstractVerticle { private final int optOutStatusMaxRequestSize; private final boolean optOutStatusApiEnabled; + private final RotatingS3KeyProvider s3KeyProvider; + //"Android" is from https://github.com/IABTechLab/uid2-android-sdk/blob/ff93ebf597f5de7d440a84f7015a334ba4138ede/sdk/src/main/java/com/uid2/UID2Client.kt#L46 //"ios"/"tvos" is from https://github.com/IABTechLab/uid2-ios-sdk/blob/91c290d29a7093cfc209eca493d1fee80c17e16a/Sources/UID2/UID2Client.swift#L36-L38 private final static List SUPPORTED_IN_APP = Arrays.asList("Android", "ios", "tvos"); @@ -142,7 +145,8 @@ public UIDOperatorVerticle(JsonObject config, Clock clock, IStatsCollectorQueue statsCollectorQueue, SecureLinkValidatorService secureLinkValidatorService, - Handler saltRetrievalResponseHandler) { + Handler saltRetrievalResponseHandler, + RotatingS3KeyProvider s3KeyProvider) { this.keyManager = keyManager; this.secureLinkValidatorService = secureLinkValidatorService; try { @@ -180,6 +184,7 @@ public UIDOperatorVerticle(JsonObject config, this.saltRetrievalResponseHandler = saltRetrievalResponseHandler; this.optOutStatusApiEnabled = config.getBoolean(Const.Config.OptOutStatusApiEnabled, true); this.optOutStatusMaxRequestSize = config.getInteger(Const.Config.OptOutStatusMaxRequestSize, 5000); + this.s3KeyProvider = s3KeyProvider; } @Override From e06509adb1416794769225628b9a941dfcd831d1 Mon Sep 17 00:00:00 2001 From: lizk886 Date: Tue, 30 Jul 2024 11:00:24 -0600 Subject: [PATCH 004/431] update shared repo --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 07119f6bf..97a178acb 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ 2.0.0-f7c174410e 2.0.4-ef52553c57 2.0.0-21f950573a - 7.11.4 + 7.16.8-alpha-127-SNAPSHOT ${project.version} 21 21 From 275e5cb8c7042abfa1ff93b216c19d19d9e5ab30 Mon Sep 17 00:00:00 2001 From: lizk886 Date: Tue, 30 Jul 2024 11:05:55 -0600 Subject: [PATCH 005/431] keep origional constuctor --- .../uid2/operator/vertx/UIDOperatorVerticle.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index 9d979bc0c..b7eedde27 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -134,6 +134,21 @@ public class UIDOperatorVerticle extends AbstractVerticle { private final static List SUPPORTED_IN_APP = Arrays.asList("Android", "ios", "tvos"); public final static String ORIGIN_HEADER = "Origin"; + public UIDOperatorVerticle(JsonObject config, + boolean clientSideTokenGenerate, + ISiteStore siteProvider, + IClientKeyProvider clientKeyProvider, + IClientSideKeypairStore clientSideKeypairProvider, + KeyManager keyManager, + ISaltProvider saltProvider, + IOptOutStore optOutStore, + Clock clock, + IStatsCollectorQueue statsCollectorQueue, + SecureLinkValidatorService secureLinkValidatorService, + Handler saltRetrievalResponseHandler) { + this(config, clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, keyManager, saltProvider, optOutStore, clock, statsCollectorQueue, secureLinkValidatorService, saltRetrievalResponseHandler, null); + + } public UIDOperatorVerticle(JsonObject config, boolean clientSideTokenGenerate, ISiteStore siteProvider, From 4ef6e87ff55a035cc3f5cd79bc2cca0f6f2e88e5 Mon Sep 17 00:00:00 2001 From: lizk886 Date: Tue, 30 Jul 2024 14:22:33 -0600 Subject: [PATCH 006/431] fix https --- conf/local-e2e-docker-private-config.json | 2 +- conf/local-e2e-docker-public-config.json | 2 +- conf/local-e2e-private-config.json | 2 +- conf/local-e2e-public-config.json | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/conf/local-e2e-docker-private-config.json b/conf/local-e2e-docker-private-config.json index ff3eb87f8..47f51e2d2 100644 --- a/conf/local-e2e-docker-private-config.json +++ b/conf/local-e2e-docker-private-config.json @@ -11,7 +11,7 @@ "keysets_metadata_path": "http://core:8088/key/keyset/refresh", "keyset_keys_metadata_path": "http://core:8088/key/keyset-keys/refresh", "salts_metadata_path": "http://core:8088/salt/refresh", - "s3_keys_metadata_path": "https://core:8088/s3encryption_keys/retrieve", + "s3_keys_metadata_path": "http://core:8088/s3encryption_keys/retrieve", "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, diff --git a/conf/local-e2e-docker-public-config.json b/conf/local-e2e-docker-public-config.json index 4c7e9ff1c..174766096 100644 --- a/conf/local-e2e-docker-public-config.json +++ b/conf/local-e2e-docker-public-config.json @@ -13,7 +13,7 @@ "salts_metadata_path": "http://core:8088/salt/refresh", "services_metadata_path": "http://core:8088/services/refresh", "service_links_metadata_path": "http://core:8088/service_links/refresh", - "s3_keys_metadata_path": "https://core:8088/s3encryption_keys/retrieve", + "s3_keys_metadata_path": "http://core:8088/s3encryption_keys/retrieve", "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, diff --git a/conf/local-e2e-private-config.json b/conf/local-e2e-private-config.json index af7af641d..ca5f7384f 100644 --- a/conf/local-e2e-private-config.json +++ b/conf/local-e2e-private-config.json @@ -13,7 +13,7 @@ "salts_metadata_path": "http://localhost:8088/salt/refresh", "services_metadata_path": "http://localhost:8088/services/refresh", "service_links_metadata_path": "http://localhost:8088/service_links/refresh", - "s3_keys_metadata_path": "https://core:8088/s3encryption_keys/retrieve", + "s3_keys_metadata_path": "http://core:8088/s3encryption_keys/retrieve", "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, diff --git a/conf/local-e2e-public-config.json b/conf/local-e2e-public-config.json index e53522ec8..71ae71fb1 100644 --- a/conf/local-e2e-public-config.json +++ b/conf/local-e2e-public-config.json @@ -13,7 +13,7 @@ "salts_metadata_path": "http://localhost:8088/salt/refresh", "services_metadata_path": "http://localhost:8088/services/refresh", "service_links_metadata_path": "http://localhost:8088/service_links/refresh", - "s3_keys_metadata_path": "https://core:8088/s3encryption_keys/retrieve", + "s3_keys_metadata_path": "http://core:8088/s3encryption_keys/retrieve", "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, From 7c25f1c44c05c55c670f4341ea4f802d0e8d11f8 Mon Sep 17 00:00:00 2001 From: lizk886 Date: Tue, 30 Jul 2024 18:09:03 -0600 Subject: [PATCH 007/431] working --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 97a178acb..f1e2b3cee 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ 2.0.0-f7c174410e 2.0.4-ef52553c57 2.0.0-21f950573a - 7.16.8-alpha-127-SNAPSHOT + 7.16.10-alpha-129-SNAPSHOT ${project.version} 21 21 From 12131f95e5d7fad089bf888b75f6caea162ed795 Mon Sep 17 00:00:00 2001 From: lizk886 Date: Tue, 30 Jul 2024 18:15:02 -0600 Subject: [PATCH 008/431] clean up --- src/main/java/com/uid2/operator/Main.java | 3 +-- .../operator/vertx/UIDOperatorVerticle.java | 19 ------------------- 2 files changed, 1 insertion(+), 21 deletions(-) diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index 2670d6e12..73ef1d3ba 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -270,8 +270,7 @@ private ICloudStorage wrapCloudStorageForOptOut(ICloudStorage cloudStorage) { private void run() throws Exception { Supplier operatorVerticleSupplier = () -> { - UIDOperatorVerticle verticle = new UIDOperatorVerticle(config, this.clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, getKeyManager(), saltProvider, optOutStore, Clock.systemUTC(), _statsCollectorQueue, new SecureLinkValidatorService(this.serviceLinkProvider, this.serviceProvider), this.shutdownHandler::handleSaltRetrievalResponse, s3KeyProvider); - return verticle; + UIDOperatorVerticle verticle = new UIDOperatorVerticle(config, this.clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, getKeyManager(), saltProvider, optOutStore, Clock.systemUTC(), _statsCollectorQueue, new SecureLinkValidatorService(this.serviceLinkProvider, this.serviceProvider), this.shutdownHandler::handleSaltRetrievalResponse); return verticle; }; DeploymentOptions options = new DeploymentOptions(); diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index b7eedde27..c635ca725 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -56,7 +56,6 @@ import org.apache.http.HttpStatus; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - import javax.crypto.*; import javax.crypto.spec.GCMParameterSpec; import javax.crypto.spec.SecretKeySpec; @@ -127,7 +126,6 @@ public class UIDOperatorVerticle extends AbstractVerticle { private final int optOutStatusMaxRequestSize; private final boolean optOutStatusApiEnabled; - private final RotatingS3KeyProvider s3KeyProvider; //"Android" is from https://github.com/IABTechLab/uid2-android-sdk/blob/ff93ebf597f5de7d440a84f7015a334ba4138ede/sdk/src/main/java/com/uid2/UID2Client.kt#L46 //"ios"/"tvos" is from https://github.com/IABTechLab/uid2-ios-sdk/blob/91c290d29a7093cfc209eca493d1fee80c17e16a/Sources/UID2/UID2Client.swift#L36-L38 @@ -146,22 +144,6 @@ public UIDOperatorVerticle(JsonObject config, IStatsCollectorQueue statsCollectorQueue, SecureLinkValidatorService secureLinkValidatorService, Handler saltRetrievalResponseHandler) { - this(config, clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, keyManager, saltProvider, optOutStore, clock, statsCollectorQueue, secureLinkValidatorService, saltRetrievalResponseHandler, null); - - } - public UIDOperatorVerticle(JsonObject config, - boolean clientSideTokenGenerate, - ISiteStore siteProvider, - IClientKeyProvider clientKeyProvider, - IClientSideKeypairStore clientSideKeypairProvider, - KeyManager keyManager, - ISaltProvider saltProvider, - IOptOutStore optOutStore, - Clock clock, - IStatsCollectorQueue statsCollectorQueue, - SecureLinkValidatorService secureLinkValidatorService, - Handler saltRetrievalResponseHandler, - RotatingS3KeyProvider s3KeyProvider) { this.keyManager = keyManager; this.secureLinkValidatorService = secureLinkValidatorService; try { @@ -199,7 +181,6 @@ public UIDOperatorVerticle(JsonObject config, this.saltRetrievalResponseHandler = saltRetrievalResponseHandler; this.optOutStatusApiEnabled = config.getBoolean(Const.Config.OptOutStatusApiEnabled, true); this.optOutStatusMaxRequestSize = config.getInteger(Const.Config.OptOutStatusMaxRequestSize, 5000); - this.s3KeyProvider = s3KeyProvider; } @Override From 1d858928249eb5f7689664410901499ddf1904fb Mon Sep 17 00:00:00 2001 From: lizk886 Date: Tue, 30 Jul 2024 18:15:43 -0600 Subject: [PATCH 009/431] clean up --- src/main/java/com/uid2/operator/Main.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index 73ef1d3ba..2cc20f761 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -270,7 +270,8 @@ private ICloudStorage wrapCloudStorageForOptOut(ICloudStorage cloudStorage) { private void run() throws Exception { Supplier operatorVerticleSupplier = () -> { - UIDOperatorVerticle verticle = new UIDOperatorVerticle(config, this.clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, getKeyManager(), saltProvider, optOutStore, Clock.systemUTC(), _statsCollectorQueue, new SecureLinkValidatorService(this.serviceLinkProvider, this.serviceProvider), this.shutdownHandler::handleSaltRetrievalResponse); return verticle; + UIDOperatorVerticle verticle = new UIDOperatorVerticle(config, this.clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, getKeyManager(), saltProvider, optOutStore, Clock.systemUTC(), _statsCollectorQueue, new SecureLinkValidatorService(this.serviceLinkProvider, this.serviceProvider), this.shutdownHandler::handleSaltRetrievalResponse); + return verticle; }; DeploymentOptions options = new DeploymentOptions(); From 08bf71a4b0a39bb23fb71d10e5394f180e9b60e1 Mon Sep 17 00:00:00 2001 From: lizk886 Date: Tue, 30 Jul 2024 18:16:21 -0600 Subject: [PATCH 010/431] clean up --- src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index c635ca725..4c2271d2a 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -28,7 +28,6 @@ import com.uid2.shared.store.IClientKeyProvider; import com.uid2.shared.store.IClientSideKeypairStore; import com.uid2.shared.store.ISaltProvider; -import com.uid2.shared.store.reader.RotatingS3KeyProvider; import com.uid2.shared.vertx.RequestCapturingHandler; import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.DistributionSummary; @@ -56,6 +55,7 @@ import org.apache.http.HttpStatus; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import javax.crypto.*; import javax.crypto.spec.GCMParameterSpec; import javax.crypto.spec.SecretKeySpec; From 4c10d2d642a02a9b496792b06b2f43938b6e0bbb Mon Sep 17 00:00:00 2001 From: lizk886 Date: Tue, 30 Jul 2024 18:16:51 -0600 Subject: [PATCH 011/431] clean up --- src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index 4c2271d2a..4a7a66e5f 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -126,7 +126,6 @@ public class UIDOperatorVerticle extends AbstractVerticle { private final int optOutStatusMaxRequestSize; private final boolean optOutStatusApiEnabled; - //"Android" is from https://github.com/IABTechLab/uid2-android-sdk/blob/ff93ebf597f5de7d440a84f7015a334ba4138ede/sdk/src/main/java/com/uid2/UID2Client.kt#L46 //"ios"/"tvos" is from https://github.com/IABTechLab/uid2-ios-sdk/blob/91c290d29a7093cfc209eca493d1fee80c17e16a/Sources/UID2/UID2Client.swift#L36-L38 private final static List SUPPORTED_IN_APP = Arrays.asList("Android", "ios", "tvos"); From d1061809c094f3461ddf188997d84550c03da9e2 Mon Sep 17 00:00:00 2001 From: lizk886 Date: Tue, 30 Jul 2024 18:17:26 -0600 Subject: [PATCH 012/431] clean up --- src/main/java/com/uid2/operator/Main.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index 2cc20f761..8ff20358e 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -56,7 +56,6 @@ import java.util.*; import java.util.function.Supplier; - import static io.micrometer.core.instrument.Metrics.globalRegistry; public class Main { From 195ec0dbf04a1a9d961cebc931f6691a3f69dd5f Mon Sep 17 00:00:00 2001 From: lizk886 Date: Fri, 2 Aug 2024 15:00:40 -0600 Subject: [PATCH 013/431] push pom --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f1e2b3cee..a24faccbc 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ 2.0.0-f7c174410e 2.0.4-ef52553c57 2.0.0-21f950573a - 7.16.10-alpha-129-SNAPSHOT + 7.16.9-alpha-130-SNAPSHOT ${project.version} 21 21 From 8700b1ff770f24cebecf9d7905ba27b40fbf1cfc Mon Sep 17 00:00:00 2001 From: lizk886 Date: Fri, 2 Aug 2024 16:06:28 -0600 Subject: [PATCH 014/431] update configs to extreme big number --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a24faccbc..0de119d55 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ 2.0.0-f7c174410e 2.0.4-ef52553c57 2.0.0-21f950573a - 7.16.9-alpha-130-SNAPSHOT + 7.17.1-alpha-133-SNAPSHOT ${project.version} 21 21 From 20481ef19122852adfbd91d5de03f27bca0c3e89 Mon Sep 17 00:00:00 2001 From: lizk886 Date: Fri, 2 Aug 2024 16:57:13 -0600 Subject: [PATCH 015/431] refactor --- .../uid2/operator/reader/ApiStoreReader.java | 51 ++++++++++++++++++ .../reader/RotatingS3KeyOperatorProvider.java | 53 +++++++++++++++++++ 2 files changed, 104 insertions(+) create mode 100644 src/main/java/com/uid2/operator/reader/ApiStoreReader.java create mode 100644 src/main/java/com/uid2/operator/reader/RotatingS3KeyOperatorProvider.java diff --git a/src/main/java/com/uid2/operator/reader/ApiStoreReader.java b/src/main/java/com/uid2/operator/reader/ApiStoreReader.java new file mode 100644 index 000000000..f34da3513 --- /dev/null +++ b/src/main/java/com/uid2/operator/reader/ApiStoreReader.java @@ -0,0 +1,51 @@ +package com.uid2.operator.reader; + +import com.uid2.shared.cloud.DownloadCloudStorage; +import com.uid2.shared.store.ScopedStoreReader; +import com.uid2.shared.store.parser.Parser; +import com.uid2.shared.store.parser.ParsingResult; +import com.uid2.shared.store.scope.StoreScope; +import io.vertx.core.json.JsonArray; +import io.vertx.core.json.JsonObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; + +public class ApiStoreReader extends ScopedStoreReader { + private static final Logger LOGGER = LoggerFactory.getLogger(ApiStoreReader.class); + + public ApiStoreReader(DownloadCloudStorage fileStreamProvider, StoreScope scope, Parser parser, String dataTypeName) { + super(fileStreamProvider, scope, parser, dataTypeName); + } + + @Override + public long loadContent(JsonObject contents, String dataType) throws Exception { + if (contents == null) { + throw new IllegalArgumentException(String.format("No contents provided for loading data type %s, cannot load content", dataType)); + } + + try { + JsonArray s3KeysArray = contents.getJsonArray(dataType); + if (s3KeysArray == null) { + throw new IllegalArgumentException("No array found in the contents"); + } + + String jsonString = s3KeysArray.toString(); + InputStream inputStream = new ByteArrayInputStream(jsonString.getBytes(StandardCharsets.UTF_8)); + + ParsingResult parsed = parser.deserialize(inputStream); + latestSnapshot.set(parsed.getData()); + + final int count = parsed.getCount(); + latestEntryCount.set(count); + LOGGER.info(String.format("Loaded %d %s", count, dataTypeName)); + return count; + } catch (Exception e) { + LOGGER.error(String.format("Unable to load %s", dataTypeName)); + throw e; + } + } +} diff --git a/src/main/java/com/uid2/operator/reader/RotatingS3KeyOperatorProvider.java b/src/main/java/com/uid2/operator/reader/RotatingS3KeyOperatorProvider.java new file mode 100644 index 000000000..4e5453358 --- /dev/null +++ b/src/main/java/com/uid2/operator/reader/RotatingS3KeyOperatorProvider.java @@ -0,0 +1,53 @@ +package com.uid2.operator.reader; + +import com.uid2.operator.reader.ApiStoreReader; +import com.uid2.shared.cloud.DownloadCloudStorage; +import com.uid2.shared.model.S3Key; +import com.uid2.shared.store.CloudPath; +import com.uid2.shared.store.parser.S3KeyParser; +import com.uid2.shared.store.reader.RotatingS3KeyProvider; +import com.uid2.shared.store.scope.StoreScope; +import io.vertx.core.json.JsonObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Instant; +import java.util.*; +import java.util.stream.Collectors; + +public class RotatingS3KeyOperatorProvider extends RotatingS3KeyProvider { + private static final Logger LOGGER = LoggerFactory.getLogger(RotatingS3KeyOperatorProvider.class); + + private final ApiStoreReader> apiStoreReader; + + public RotatingS3KeyOperatorProvider(DownloadCloudStorage fileStreamProvider, StoreScope scope) { + super(fileStreamProvider, scope); + this.apiStoreReader = new ApiStoreReader<>(fileStreamProvider, scope, new S3KeyParser(), "s3encryption_keys"); + } + + @Override + public JsonObject getMetadata() throws Exception { + return apiStoreReader.getMetadata(); + } + + @Override + public CloudPath getMetadataPath() { + return apiStoreReader.getMetadataPath(); + } + + @Override + public long loadContent(JsonObject metadata) throws Exception { + return apiStoreReader.loadContent(metadata, "s3Keys"); + } + + @Override + public Map getAll() { + Map keys = apiStoreReader.getSnapshot(); + return keys != null ? keys : new HashMap<>(); + } + + @Override + public void loadContent() throws Exception { + this.loadContent(this.getMetadata()); + } +} From 88f2716e26c76e9d09335d7f430410fb06d0fcd4 Mon Sep 17 00:00:00 2001 From: lizk886 Date: Fri, 2 Aug 2024 16:59:59 -0600 Subject: [PATCH 016/431] refactor' --- src/main/java/com/uid2/operator/Main.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index 8ff20358e..8af109428 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -7,6 +7,7 @@ import com.uid2.operator.monitoring.IStatsCollectorQueue; import com.uid2.operator.monitoring.OperatorMetrics; import com.uid2.operator.monitoring.StatsCollectorVerticle; +import com.uid2.operator.reader.RotatingS3KeyOperatorProvider; import com.uid2.operator.service.SecureLinkValidatorService; import com.uid2.operator.service.ShutdownService; import com.uid2.operator.vertx.Endpoints; @@ -80,7 +81,7 @@ public class Main { private IStatsCollectorQueue _statsCollectorQueue; private RotatingServiceStore serviceProvider; private RotatingServiceLinkStore serviceLinkProvider; - private RotatingS3KeyProvider s3KeyProvider; + private RotatingS3KeyOperatorProvider s3KeyProvider; public Main(Vertx vertx, JsonObject config) throws Exception { this.vertx = vertx; @@ -145,7 +146,7 @@ public Main(Vertx vertx, JsonObject config) throws Exception { this.saltProvider = new RotatingSaltProvider(fsStores, saltsMdPath); this.optOutStore = new CloudSyncOptOutStore(vertx, fsLocal, this.config, operatorKey, Clock.systemUTC()); String s3KeyMdPath = this.config.getString(Const.Config.S3keysMetadataPathProp); - this.s3KeyProvider = new RotatingS3KeyProvider(fsStores, new GlobalScope(new CloudPath(s3KeyMdPath))); + this.s3KeyProvider = new RotatingS3KeyOperatorProvider(fsStores, new GlobalScope(new CloudPath(s3KeyMdPath))); if (this.validateServiceLinks) { String serviceMdPath = this.config.getString(Const.Config.ServiceMetadataPathProp); From 8975886b42443ee14ad196d593c95a5de5875b5b Mon Sep 17 00:00:00 2001 From: lizk886 Date: Fri, 2 Aug 2024 17:02:37 -0600 Subject: [PATCH 017/431] update shared --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0de119d55..e9e1ced54 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ 2.0.0-f7c174410e 2.0.4-ef52553c57 2.0.0-21f950573a - 7.17.1-alpha-133-SNAPSHOT + 7.17.2-alpha-134-SNAPSHOT ${project.version} 21 21 From fc960129eb7564f24e8c6ca368e06dda297c3a3d Mon Sep 17 00:00:00 2001 From: lizk886 Date: Fri, 2 Aug 2024 18:44:37 -0600 Subject: [PATCH 018/431] only be able to decrypt keyset and keyset_keys, needs to figure out what's going on with auth --- pom.xml | 2 +- src/main/java/com/uid2/operator/Main.java | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pom.xml b/pom.xml index e9e1ced54..18461b8e1 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ 2.0.0-f7c174410e 2.0.4-ef52553c57 2.0.0-21f950573a - 7.17.2-alpha-134-SNAPSHOT + 7.17.4-alpha-136-SNAPSHOT ${project.version} 21 21 diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index 8af109428..898600856 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -133,20 +133,20 @@ public Main(Vertx vertx, JsonObject config) throws Exception { this.fsOptOut = configureCloudOptOutStore(); } + String s3KeyMdPath = this.config.getString(Const.Config.S3keysMetadataPathProp); + this.s3KeyProvider = new RotatingS3KeyOperatorProvider(fsStores, new GlobalScope(new CloudPath(s3KeyMdPath))); String sitesMdPath = this.config.getString(Const.Config.SitesMetadataPathProp); String keypairMdPath = this.config.getString(Const.Config.ClientSideKeypairsMetadataPathProp); this.clientSideKeypairProvider = new RotatingClientSideKeypairStore(fsStores, new GlobalScope(new CloudPath(keypairMdPath))); String clientsMdPath = this.config.getString(Const.Config.ClientsMetadataPathProp); this.clientKeyProvider = new RotatingClientKeyProvider(fsStores, new GlobalScope(new CloudPath(clientsMdPath))); String keysetKeysMdPath = this.config.getString(Const.Config.KeysetKeysMetadataPathProp); - this.keysetKeyStore = new RotatingKeysetKeyStore(fsStores, new GlobalScope(new CloudPath(keysetKeysMdPath))); + this.keysetKeyStore = new RotatingKeysetKeyStore(fsStores, new GlobalScope(new CloudPath(keysetKeysMdPath)),s3KeyProvider); String keysetMdPath = this.config.getString(Const.Config.KeysetsMetadataPathProp); - this.keysetProvider = new RotatingKeysetProvider(fsStores, new GlobalScope(new CloudPath(keysetMdPath))); + this.keysetProvider = new RotatingKeysetProvider(fsStores, new GlobalScope(new CloudPath(keysetMdPath)),s3KeyProvider); String saltsMdPath = this.config.getString(Const.Config.SaltsMetadataPathProp); this.saltProvider = new RotatingSaltProvider(fsStores, saltsMdPath); this.optOutStore = new CloudSyncOptOutStore(vertx, fsLocal, this.config, operatorKey, Clock.systemUTC()); - String s3KeyMdPath = this.config.getString(Const.Config.S3keysMetadataPathProp); - this.s3KeyProvider = new RotatingS3KeyOperatorProvider(fsStores, new GlobalScope(new CloudPath(s3KeyMdPath))); if (this.validateServiceLinks) { String serviceMdPath = this.config.getString(Const.Config.ServiceMetadataPathProp); From 1083e59550979eccb68c9b2f15d475b425c44aef Mon Sep 17 00:00:00 2001 From: lizk886 Date: Tue, 6 Aug 2024 16:01:46 -0600 Subject: [PATCH 019/431] update --- pom.xml | 2 +- src/main/java/com/uid2/operator/Main.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 18461b8e1..a5fc50efd 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ 2.0.0-f7c174410e 2.0.4-ef52553c57 2.0.0-21f950573a - 7.17.4-alpha-136-SNAPSHOT + 7.17.6-alpha-138-SNAPSHOT ${project.version} 21 21 diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index 898600856..593bb1368 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -139,7 +139,7 @@ public Main(Vertx vertx, JsonObject config) throws Exception { String keypairMdPath = this.config.getString(Const.Config.ClientSideKeypairsMetadataPathProp); this.clientSideKeypairProvider = new RotatingClientSideKeypairStore(fsStores, new GlobalScope(new CloudPath(keypairMdPath))); String clientsMdPath = this.config.getString(Const.Config.ClientsMetadataPathProp); - this.clientKeyProvider = new RotatingClientKeyProvider(fsStores, new GlobalScope(new CloudPath(clientsMdPath))); + this.clientKeyProvider = new RotatingClientKeyProvider(fsStores, new GlobalScope(new CloudPath(clientsMdPath)),s3KeyProvider); String keysetKeysMdPath = this.config.getString(Const.Config.KeysetKeysMetadataPathProp); this.keysetKeyStore = new RotatingKeysetKeyStore(fsStores, new GlobalScope(new CloudPath(keysetKeysMdPath)),s3KeyProvider); String keysetMdPath = this.config.getString(Const.Config.KeysetsMetadataPathProp); From 18ccab1467208e07599e231109363bb6e257211c Mon Sep 17 00:00:00 2001 From: lizk886 Date: Tue, 6 Aug 2024 19:09:42 -0600 Subject: [PATCH 020/431] update with unit tests --- .../reader/RotatingS3KeyOperatorProvider.java | 2 +- .../com/uid2/operator/ApiStoreReaderTest.java | 103 ++++++++++++++++++ .../RotatingS3KeyOperatorProviderTest.java | 103 ++++++++++++++++++ 3 files changed, 207 insertions(+), 1 deletion(-) create mode 100644 src/test/java/com/uid2/operator/ApiStoreReaderTest.java create mode 100644 src/test/java/com/uid2/operator/RotatingS3KeyOperatorProviderTest.java diff --git a/src/main/java/com/uid2/operator/reader/RotatingS3KeyOperatorProvider.java b/src/main/java/com/uid2/operator/reader/RotatingS3KeyOperatorProvider.java index 4e5453358..df73b790a 100644 --- a/src/main/java/com/uid2/operator/reader/RotatingS3KeyOperatorProvider.java +++ b/src/main/java/com/uid2/operator/reader/RotatingS3KeyOperatorProvider.java @@ -18,7 +18,7 @@ public class RotatingS3KeyOperatorProvider extends RotatingS3KeyProvider { private static final Logger LOGGER = LoggerFactory.getLogger(RotatingS3KeyOperatorProvider.class); - private final ApiStoreReader> apiStoreReader; + public ApiStoreReader> apiStoreReader; public RotatingS3KeyOperatorProvider(DownloadCloudStorage fileStreamProvider, StoreScope scope) { super(fileStreamProvider, scope); diff --git a/src/test/java/com/uid2/operator/ApiStoreReaderTest.java b/src/test/java/com/uid2/operator/ApiStoreReaderTest.java new file mode 100644 index 000000000..460d57b8d --- /dev/null +++ b/src/test/java/com/uid2/operator/ApiStoreReaderTest.java @@ -0,0 +1,103 @@ +package com.uid2.operator; + +import com.uid2.operator.reader.ApiStoreReader; +import com.uid2.shared.cloud.DownloadCloudStorage; +import com.uid2.shared.store.CloudPath; +import com.uid2.shared.store.parser.Parser; +import com.uid2.shared.store.parser.ParsingResult; +import com.uid2.shared.store.scope.GlobalScope; +import io.vertx.core.json.JsonArray; +import io.vertx.core.json.JsonObject; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +import java.io.InputStream; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + + class ApiStoreReaderTest { + + @Mock + private DownloadCloudStorage mockStorage; + + @Mock + private Parser> mockParser; + + private final CloudPath metadataPath = new CloudPath("test/test-metadata.json"); + private final String dataType = "test-data-type"; + private final GlobalScope scope = new GlobalScope(metadataPath); + + private ApiStoreReader> reader; + + @BeforeEach + void setUp() { + MockitoAnnotations.openMocks(this); + reader = new ApiStoreReader<>(mockStorage, scope, mockParser, dataType); + } + + @Test + void getMetadataPathReturnsPathFromScope() { + CloudPath actual = reader.getMetadataPath(); + assertThat(actual).isEqualTo(metadataPath); + } + + @Test + void loadContentThrowsExceptionWhenContentsAreNull() { + assertThatThrownBy(() -> reader.loadContent(null, dataType)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("No contents provided for loading data type"); + } + + @Test + void loadContentThrowsExceptionWhenArrayNotFound() { + JsonObject contents = new JsonObject(); + assertThatThrownBy(() -> reader.loadContent(contents, dataType)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("No array found in the contents"); + } + + @Test + void loadContentSuccessfullyLoadsData() throws Exception { + JsonObject contents = new JsonObject() + .put(dataType, new JsonArray().add("value1").add("value2")); + + List expectedData = Arrays.asList(new TestData("value1"), new TestData("value2")); + when(mockParser.deserialize(any(InputStream.class))) + .thenReturn(new ParsingResult<>(expectedData, expectedData.size())); + + long count = reader.loadContent(contents, dataType); + + assertThat(count).isEqualTo(2); + assertThat(reader.getSnapshot()).isEqualTo(expectedData); + } + + private static class TestData { + private final String value; + + TestData(String value) { + this.value = value; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + TestData testData = (TestData) o; + return value.equals(testData.value); + } + + @Override + public int hashCode() { + return value.hashCode(); + } + } + } + diff --git a/src/test/java/com/uid2/operator/RotatingS3KeyOperatorProviderTest.java b/src/test/java/com/uid2/operator/RotatingS3KeyOperatorProviderTest.java new file mode 100644 index 000000000..ee53d3ab5 --- /dev/null +++ b/src/test/java/com/uid2/operator/RotatingS3KeyOperatorProviderTest.java @@ -0,0 +1,103 @@ +package com.uid2.operator; + +import com.uid2.operator.reader.ApiStoreReader; +import com.uid2.operator.reader.RotatingS3KeyOperatorProvider; +import com.uid2.shared.cloud.DownloadCloudStorage; +import com.uid2.shared.model.S3Key; +import com.uid2.shared.store.CloudPath; +import com.uid2.shared.store.scope.StoreScope; +import io.vertx.core.json.JsonObject; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +import java.util.HashMap; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; + +class RotatingS3KeyOperatorProviderTest { + + @Mock + private DownloadCloudStorage mockFileStreamProvider; + + @Mock + private StoreScope mockScope; + + @Mock + private ApiStoreReader> mockApiStoreReader; + + private RotatingS3KeyOperatorProvider rotatingS3KeyOperatorProvider; + + @BeforeEach + void setUp() { + MockitoAnnotations.openMocks(this); + rotatingS3KeyOperatorProvider = new RotatingS3KeyOperatorProvider(mockFileStreamProvider, mockScope); + rotatingS3KeyOperatorProvider.apiStoreReader = mockApiStoreReader; + } + + @Test + void testGetMetadata() throws Exception { + JsonObject expectedMetadata = new JsonObject().put("version", 1L); + when(mockApiStoreReader.getMetadata()).thenReturn(expectedMetadata); + + JsonObject metadata = rotatingS3KeyOperatorProvider.getMetadata(); + assertEquals(expectedMetadata, metadata); + verify(mockApiStoreReader).getMetadata(); + } + + @Test + void testGetMetadataPath() { + CloudPath expectedPath = new CloudPath("test/path"); + when(mockApiStoreReader.getMetadataPath()).thenReturn(expectedPath); + + CloudPath path = rotatingS3KeyOperatorProvider.getMetadataPath(); + assertEquals(expectedPath, path); + verify(mockApiStoreReader).getMetadataPath(); + } + + @Test + void testLoadContentWithMetadata() throws Exception { + JsonObject metadata = new JsonObject(); + when(mockApiStoreReader.loadContent(metadata, "s3Keys")).thenReturn(1L); + + long version = rotatingS3KeyOperatorProvider.loadContent(metadata); + assertEquals(1L, version); + verify(mockApiStoreReader).loadContent(metadata, "s3Keys"); + } + + @Test + void testGetAll() { + Map expectedKeys = new HashMap<>(); + S3Key key = new S3Key(1, 123, 1687635529, 1687808329, "secret"); + expectedKeys.put(1, key); + when(mockApiStoreReader.getSnapshot()).thenReturn(expectedKeys); + + Map keys = rotatingS3KeyOperatorProvider.getAll(); + assertEquals(expectedKeys, keys); + verify(mockApiStoreReader).getSnapshot(); + } + + @Test + void testGetAllWithNullSnapshot() { + when(mockApiStoreReader.getSnapshot()).thenReturn(null); + + Map keys = rotatingS3KeyOperatorProvider.getAll(); + assertNotNull(keys); + assertTrue(keys.isEmpty()); + verify(mockApiStoreReader).getSnapshot(); + } + + @Test + void testLoadContent() throws Exception { + JsonObject metadata = new JsonObject().put("version", 1L); + when(mockApiStoreReader.getMetadata()).thenReturn(metadata); + when(mockApiStoreReader.loadContent(metadata, "s3Keys")).thenReturn(1L); + + rotatingS3KeyOperatorProvider.loadContent(); + verify(mockApiStoreReader).getMetadata(); + verify(mockApiStoreReader).loadContent(metadata, "s3Keys"); + } +} \ No newline at end of file From 578992cdfe27c6c625c378498a0fa167bb558c86 Mon Sep 17 00:00:00 2001 From: lizk886 Date: Wed, 7 Aug 2024 12:01:03 -0600 Subject: [PATCH 021/431] update site --- pom.xml | 2 +- src/main/java/com/uid2/operator/Main.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index a5fc50efd..78933009d 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ 2.0.0-f7c174410e 2.0.4-ef52553c57 2.0.0-21f950573a - 7.17.6-alpha-138-SNAPSHOT + 7.17.9-alpha-142-SNAPSHOT ${project.version} 21 21 diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index 593bb1368..f4c2f6bd7 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -155,7 +155,7 @@ public Main(Vertx vertx, JsonObject config) throws Exception { this.serviceLinkProvider = new RotatingServiceLinkStore(fsStores, new GlobalScope(new CloudPath(serviceLinkMdPath))); } - this.siteProvider = clientSideTokenGenerate ? new RotatingSiteStore(fsStores, new GlobalScope(new CloudPath(sitesMdPath))) : null; + this.siteProvider = clientSideTokenGenerate ? new RotatingSiteStore(fsStores, new GlobalScope(new CloudPath(sitesMdPath)), s3KeyProvider) : null; if (useStorageMock && coreAttestUrl == null) { if (clientSideTokenGenerate) { From e057a8480498f6a409d3013654f437c286f3d87c Mon Sep 17 00:00:00 2001 From: "ian.nara" Date: Fri, 4 Oct 2024 12:58:16 -0600 Subject: [PATCH 022/431] testing debug config value --- scripts/aws/eks-pod/entrypoint.sh | 13 +++++++++++-- scripts/aws/entrypoint.sh | 14 +++++++++----- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/scripts/aws/eks-pod/entrypoint.sh b/scripts/aws/eks-pod/entrypoint.sh index c506d6cbf..9ec4ea009 100644 --- a/scripts/aws/eks-pod/entrypoint.sh +++ b/scripts/aws/eks-pod/entrypoint.sh @@ -3,6 +3,7 @@ CID=42 EIF_PATH=/home/uid2operator.eif MEMORY_MB=24576 CPU_COUNT=6 +DEBUG_MODE="false" set -x @@ -87,12 +88,20 @@ function update_config() { { set +x; } 2>/dev/null; { CPU_COUNT=$(echo $IDENTITY_SERVICE_CONFIG | jq -r '.enclave_cpu_count'); set -x; } { set +x; } 2>/dev/null; { MEMORY_MB=$(echo $IDENTITY_SERVICE_CONFIG | jq -r '.enclave_memory_mb'); set -x; } fi + + { set +x; } 2>/dev/null; { DEBUG_MODE=$(echo $IDENTITY_SERVICE_CONFIG | jq -r '.debug_mode'); set -x; } + shopt -u nocasematch } function run_enclave() { - echo "starting enclave... --cpu-count $CPU_COUNT --memory $MEMORY_MB --eif-path $EIF_PATH --enclave-cid $CID" - nitro-cli run-enclave --cpu-count $CPU_COUNT --memory $MEMORY_MB --eif-path $EIF_PATH --enclave-cid $CID --enclave-name uid2-operator + if [ "$DEBUG_MODE" = "true" ]; then + echo "starting enclave... --cpu-count $CPU_COUNT --memory $MEMORY_MB --eif-path $EIF_PATH --enclave-cid $CID --debug-mode --attach-console" + nitro-cli run-enclave --cpu-count $CPU_COUNT --memory $MEMORY_MB --eif-path $EIF_PATH --enclave-cid $CID --enclave-name uid2-operator --debug-mode --attach-console + else + echo "starting enclave... --cpu-count $CPU_COUNT --memory $MEMORY_MB --eif-path $EIF_PATH --enclave-cid $CID" + nitro-cli run-enclave --cpu-count $CPU_COUNT --memory $MEMORY_MB --eif-path $EIF_PATH --enclave-cid $CID --enclave-name uid2-operator + fi } echo "starting ..." diff --git a/scripts/aws/entrypoint.sh b/scripts/aws/entrypoint.sh index 32db563fa..f51041b8b 100755 --- a/scripts/aws/entrypoint.sh +++ b/scripts/aws/entrypoint.sh @@ -5,7 +5,7 @@ LOG_FILE="/home/start.txt" set -x -exec > $LOG_FILE +exec &> >(tee -a "$LOG_FILE") exec 2>&1 set -o pipefail @@ -19,10 +19,6 @@ ifconfig lo 127.0.0.1 echo "Starting vsock proxy..." /app/vsockpx --config /app/proxies.nitro.yaml --daemon --workers $(( $(nproc) * 2 )) --log-level 3 -# -- setup syslog-ng -echo "Starting syslog-ng..." -/usr/sbin/syslog-ng --verbose - # -- load config from identity service echo "Loading config from identity service via proxy..." @@ -42,6 +38,14 @@ do sleep 2 done +DEBUG_MODE=$(jq -r ".debug_mode" < "${OVERRIDES_CONFIG}") + +if [[ ! "$DEBUG_MODE" = "true" ]]; then + # -- setup syslog-ng + echo "Starting syslog-ng..." + /usr/sbin/syslog-ng --verbose +fi + # check the config is valid. Querying for a known missing element (empty) makes jq parse the file, but does not echo the results if jq empty "${OVERRIDES_CONFIG}"; then echo "Identity service returned valid config" From a70141b39191c4d51d3e56181d31d284735a9ff0 Mon Sep 17 00:00:00 2001 From: "ian.nara" Date: Fri, 4 Oct 2024 13:01:00 -0600 Subject: [PATCH 023/431] testing debug config value --- scripts/aws/entrypoint.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/aws/entrypoint.sh b/scripts/aws/entrypoint.sh index f51041b8b..b687ae845 100755 --- a/scripts/aws/entrypoint.sh +++ b/scripts/aws/entrypoint.sh @@ -6,7 +6,6 @@ LOG_FILE="/home/start.txt" set -x exec &> >(tee -a "$LOG_FILE") -exec 2>&1 set -o pipefail ulimit -n 65536 From ec27533ce3e8d238f1c55280341802de62d0e218 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Fri, 4 Oct 2024 19:05:20 +0000 Subject: [PATCH 024/431] [CI Pipeline] Released Snapshot version: 5.40.49-alpha-24-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 8a1c418f7..1621ffdd1 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.40.48 + 5.40.49-alpha-24-SNAPSHOT UTF-8 From b24fbf8f84ed69dee3d3c9c4128e3fd8d6ab003b Mon Sep 17 00:00:00 2001 From: "ian.nara" Date: Fri, 4 Oct 2024 15:38:56 -0600 Subject: [PATCH 025/431] don't use logback conf in debug mode --- scripts/aws/entrypoint.sh | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/scripts/aws/entrypoint.sh b/scripts/aws/entrypoint.sh index b687ae845..38de68709 100755 --- a/scripts/aws/entrypoint.sh +++ b/scripts/aws/entrypoint.sh @@ -98,12 +98,23 @@ cd /app # -- start operator echo "Starting Java application..." -java \ - -XX:MaxRAMPercentage=95 -XX:-UseCompressedOops -XX:+PrintFlagsFinal \ - -Djava.security.egd=file:/dev/./urandom \ - -Djava.library.path=/app/lib \ - -Dvertx-config-path="${FINAL_CONFIG}" \ - -Dvertx.logger-delegate-factory-class-name=io.vertx.core.logging.SLF4JLogDelegateFactory \ - -Dlogback.configurationFile=./conf/logback.xml \ - -Dhttp_proxy=socks5://127.0.0.1:3305 \ - -jar /app/"${JAR_NAME}"-"${JAR_VERSION}".jar +if [[ ! "$DEBUG_MODE" = "true" ]]; then + java \ + -XX:MaxRAMPercentage=95 -XX:-UseCompressedOops -XX:+PrintFlagsFinal \ + -Djava.security.egd=file:/dev/./urandom \ + -Djava.library.path=/app/lib \ + -Dvertx-config-path="${FINAL_CONFIG}" \ + -Dvertx.logger-delegate-factory-class-name=io.vertx.core.logging.SLF4JLogDelegateFactory \ + -Dhttp_proxy=socks5://127.0.0.1:3305 \ + -jar /app/"${JAR_NAME}"-"${JAR_VERSION}".jar +else + java \ + -XX:MaxRAMPercentage=95 -XX:-UseCompressedOops -XX:+PrintFlagsFinal \ + -Djava.security.egd=file:/dev/./urandom \ + -Djava.library.path=/app/lib \ + -Dvertx-config-path="${FINAL_CONFIG}" \ + -Dvertx.logger-delegate-factory-class-name=io.vertx.core.logging.SLF4JLogDelegateFactory \ + -Dlogback.configurationFile=./conf/logback.xml \ + -Dhttp_proxy=socks5://127.0.0.1:3305 \ + -jar /app/"${JAR_NAME}"-"${JAR_VERSION}".jar +fi \ No newline at end of file From 59d9dd7847b3736dcb7cecdd6b3f76256011b8fc Mon Sep 17 00:00:00 2001 From: "ian.nara" Date: Fri, 4 Oct 2024 15:41:04 -0600 Subject: [PATCH 026/431] add more log information --- scripts/aws/eks-pod/entrypoint.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/aws/eks-pod/entrypoint.sh b/scripts/aws/eks-pod/entrypoint.sh index 9ec4ea009..3bdce6a25 100644 --- a/scripts/aws/eks-pod/entrypoint.sh +++ b/scripts/aws/eks-pod/entrypoint.sh @@ -14,7 +14,9 @@ function terminate_old_enclave() { nitro-cli terminate-enclave --enclave-id ${ENCLAVE_ID} echo "Terminated enclave with ID ${ENCLAVE_ID}" else + nitro-cli describe-enclaves echo "No running enclaves to terminate." + sleep 30 fi } @@ -120,6 +122,7 @@ wait_for_config update_config run_enclave +nitro-cli describe-enclaves sleep 60s set +x ENCLAVE_ID=$(nitro-cli describe-enclaves | jq -r ".[0].EnclaveID") From b4aa9c48a962b1e33adbe0a2f092bbedd2f2763d Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Fri, 4 Oct 2024 21:42:07 +0000 Subject: [PATCH 027/431] [CI Pipeline] Released Snapshot version: 5.40.50-alpha-25-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1621ffdd1..5fe89e8dc 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.40.49-alpha-24-SNAPSHOT + 5.40.50-alpha-25-SNAPSHOT UTF-8 From da0e0583e201e5e8937b426c18ca416b26ba501d Mon Sep 17 00:00:00 2001 From: "ian.nara" Date: Fri, 4 Oct 2024 16:31:42 -0600 Subject: [PATCH 028/431] fix if statement --- scripts/aws/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/aws/entrypoint.sh b/scripts/aws/entrypoint.sh index 38de68709..9fd5e31ca 100755 --- a/scripts/aws/entrypoint.sh +++ b/scripts/aws/entrypoint.sh @@ -98,7 +98,7 @@ cd /app # -- start operator echo "Starting Java application..." -if [[ ! "$DEBUG_MODE" = "true" ]]; then +if [[ "$DEBUG_MODE" = "true" ]]; then java \ -XX:MaxRAMPercentage=95 -XX:-UseCompressedOops -XX:+PrintFlagsFinal \ -Djava.security.egd=file:/dev/./urandom \ From 7ec499f977df9100a1e07b00cb3421fbed49cb15 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Fri, 4 Oct 2024 22:35:02 +0000 Subject: [PATCH 029/431] [CI Pipeline] Released Snapshot version: 5.40.51-alpha-27-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5fe89e8dc..33cc957bf 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.40.50-alpha-25-SNAPSHOT + 5.40.51-alpha-27-SNAPSHOT UTF-8 From 3a377d5616f6c21bd47b1fe78d1cd0f3fde07cb8 Mon Sep 17 00:00:00 2001 From: "ian.nara" Date: Mon, 7 Oct 2024 09:27:09 -0600 Subject: [PATCH 030/431] add debug logback conf --- Makefile.eif | 5 ++++- scripts/aws/conf/logback-debug.xml | 15 +++++++++++++++ scripts/aws/entrypoint.sh | 17 +++++++++-------- 3 files changed, 28 insertions(+), 9 deletions(-) create mode 100644 scripts/aws/conf/logback-debug.xml diff --git a/Makefile.eif b/Makefile.eif index 395685024..9b5b391ac 100644 --- a/Makefile.eif +++ b/Makefile.eif @@ -37,7 +37,7 @@ build/make_config.py: ./scripts/aws/make_config.py .PHONY: build_configs -build_configs: build/conf/default-config.json build/conf/prod-uid2-config.json build/conf/integ-uid2-config.json build/conf/prod-euid-config.json build/conf/integ-euid-config.json build/conf/logback.xml +build_configs: build/conf/default-config.json build/conf/prod-uid2-config.json build/conf/integ-uid2-config.json build/conf/prod-euid-config.json build/conf/integ-euid-config.json build/conf/logback.xml build/conf/logback-debug.xml build/conf/default-config.json: build_artifacts ./scripts/aws/conf/default-config.json cp ./scripts/aws/conf/default-config.json ./build/conf/ @@ -57,6 +57,9 @@ build/conf/integ-euid-config.json: build_artifacts ./scripts/aws/conf/integ-euid build/conf/logback.xml: build_artifacts ./scripts/aws/conf/logback.xml cp ./scripts/aws/conf/logback.xml ./build/conf/ +build/conf/logback-debug.xml: build_artifacts ./scripts/aws/conf/logback-debug.xml + cp ./scripts/aws/conf/logback-debug.xml ./build/conf/ + build/Dockerfile: build_artifacts ./scripts/aws/Dockerfile cp ./scripts/aws/Dockerfile ./build/ diff --git a/scripts/aws/conf/logback-debug.xml b/scripts/aws/conf/logback-debug.xml new file mode 100644 index 000000000..c012f8d25 --- /dev/null +++ b/scripts/aws/conf/logback-debug.xml @@ -0,0 +1,15 @@ + + + + + + + REDACTED - S3 + \S+s3\.amazonaws\.com\/\S*X-Amz-Security-Token=\S+ + + + + + + + \ No newline at end of file diff --git a/scripts/aws/entrypoint.sh b/scripts/aws/entrypoint.sh index 9fd5e31ca..e5063b148 100755 --- a/scripts/aws/entrypoint.sh +++ b/scripts/aws/entrypoint.sh @@ -105,16 +105,17 @@ if [[ "$DEBUG_MODE" = "true" ]]; then -Djava.library.path=/app/lib \ -Dvertx-config-path="${FINAL_CONFIG}" \ -Dvertx.logger-delegate-factory-class-name=io.vertx.core.logging.SLF4JLogDelegateFactory \ + -Dlogback.configurationFile=./conf/logback-debug.xml \ -Dhttp_proxy=socks5://127.0.0.1:3305 \ -jar /app/"${JAR_NAME}"-"${JAR_VERSION}".jar else java \ - -XX:MaxRAMPercentage=95 -XX:-UseCompressedOops -XX:+PrintFlagsFinal \ - -Djava.security.egd=file:/dev/./urandom \ - -Djava.library.path=/app/lib \ - -Dvertx-config-path="${FINAL_CONFIG}" \ - -Dvertx.logger-delegate-factory-class-name=io.vertx.core.logging.SLF4JLogDelegateFactory \ - -Dlogback.configurationFile=./conf/logback.xml \ - -Dhttp_proxy=socks5://127.0.0.1:3305 \ - -jar /app/"${JAR_NAME}"-"${JAR_VERSION}".jar + -XX:MaxRAMPercentage=95 -XX:-UseCompressedOops -XX:+PrintFlagsFinal \ + -Djava.security.egd=file:/dev/./urandom \ + -Djava.library.path=/app/lib \ + -Dvertx-config-path="${FINAL_CONFIG}" \ + -Dvertx.logger-delegate-factory-class-name=io.vertx.core.logging.SLF4JLogDelegateFactory \ + -Dlogback.configurationFile=./conf/logback.xml \ + -Dhttp_proxy=socks5://127.0.0.1:3305 \ + -jar /app/"${JAR_NAME}"-"${JAR_VERSION}".jar fi \ No newline at end of file From ba116b3b3ae87d9f75029de3232c99dc2e67fb5f Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Mon, 7 Oct 2024 15:28:38 +0000 Subject: [PATCH 031/431] [CI Pipeline] Released Snapshot version: 5.40.52-alpha-28-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 33cc957bf..c28f60fc3 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.40.51-alpha-27-SNAPSHOT + 5.40.52-alpha-28-SNAPSHOT UTF-8 From 4803b7f7ad2f89f88773c6552d351665f40412bb Mon Sep 17 00:00:00 2001 From: "ian.nara" Date: Mon, 7 Oct 2024 12:41:11 -0600 Subject: [PATCH 032/431] operator debug --- scripts/aws/entrypoint.sh | 1 + scripts/aws/proxies.host.yaml | 5 +++++ scripts/aws/proxies.nitro.yaml | 5 +++++ 3 files changed, 11 insertions(+) diff --git a/scripts/aws/entrypoint.sh b/scripts/aws/entrypoint.sh index e5063b148..8c99c26b9 100755 --- a/scripts/aws/entrypoint.sh +++ b/scripts/aws/entrypoint.sh @@ -101,6 +101,7 @@ echo "Starting Java application..." if [[ "$DEBUG_MODE" = "true" ]]; then java \ -XX:MaxRAMPercentage=95 -XX:-UseCompressedOops -XX:+PrintFlagsFinal \ + -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000 \ -Djava.security.egd=file:/dev/./urandom \ -Djava.library.path=/app/lib \ -Dvertx-config-path="${FINAL_CONFIG}" \ diff --git a/scripts/aws/proxies.host.yaml b/scripts/aws/proxies.host.yaml index 5a2ae0623..da2e40b57 100644 --- a/scripts/aws/proxies.host.yaml +++ b/scripts/aws/proxies.host.yaml @@ -10,6 +10,11 @@ operator-service: listen: tcp://0.0.0.0:80 connect: vsock://42:8080 +operator-debug: + service: direct + listen: tcp://0.0.0.0:8000 + connect: vsock://42:8000 + operator-prometheus: service: direct listen: tcp://0.0.0.0:9080 diff --git a/scripts/aws/proxies.nitro.yaml b/scripts/aws/proxies.nitro.yaml index 0f459b150..e9b5e0cfc 100644 --- a/scripts/aws/proxies.nitro.yaml +++ b/scripts/aws/proxies.nitro.yaml @@ -5,6 +5,11 @@ uid-operator-in: listen: vsock://-1:8080 connect: tcp://127.0.0.1:8080 +uid-operator-debug-in: + service: direct + listen: vsock://-1:8000 + connect: tcp://127.0.0.1:8000 + prometheus-server: service: direct listen: vsock://-1:9080 From 0de31854847338903a288638423f88135e6ee5b4 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Mon, 7 Oct 2024 18:46:38 +0000 Subject: [PATCH 033/431] [CI Pipeline] Released Snapshot version: 5.40.53-alpha-29-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c28f60fc3..661748e55 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.40.52-alpha-28-SNAPSHOT + 5.40.53-alpha-29-SNAPSHOT UTF-8 From 7772761ea2d5726af8b46af5911e761427833055 Mon Sep 17 00:00:00 2001 From: Thomas Manson Date: Tue, 8 Oct 2024 10:22:38 +1100 Subject: [PATCH 034/431] Adding vsock and dante logging --- scripts/aws/eks-pod/entrypoint.sh | 2 +- scripts/aws/eks-pod/sockd_eks.conf | 8 ++++---- scripts/aws/entrypoint.sh | 2 +- scripts/aws/sockd.conf | 7 +++++-- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/scripts/aws/eks-pod/entrypoint.sh b/scripts/aws/eks-pod/entrypoint.sh index 3bdce6a25..7573d8a9d 100644 --- a/scripts/aws/eks-pod/entrypoint.sh +++ b/scripts/aws/eks-pod/entrypoint.sh @@ -30,7 +30,7 @@ function setup_vsockproxy() { VSOCK_PROXY=${VSOCK_PROXY:-/home/vsockpx} VSOCK_CONFIG=${VSOCK_CONFIG:-/home/proxies.host.yaml} VSOCK_THREADS=${VSOCK_THREADS:-$(( $(nproc) * 2 )) } - VSOCK_LOG_LEVEL=${VSOCK_LOG_LEVEL:-3} + VSOCK_LOG_LEVEL=0 echo "starting vsock proxy at $VSOCK_PROXY with $VSOCK_THREADS worker threads..." $VSOCK_PROXY -c $VSOCK_CONFIG --workers $VSOCK_THREADS --log-level $VSOCK_LOG_LEVEL --daemon echo "vsock proxy now running in background." diff --git a/scripts/aws/eks-pod/sockd_eks.conf b/scripts/aws/eks-pod/sockd_eks.conf index 0e588fb7c..f6b483958 100644 --- a/scripts/aws/eks-pod/sockd_eks.conf +++ b/scripts/aws/eks-pod/sockd_eks.conf @@ -1,6 +1,6 @@ -#logoutput: stdout +logoutput: stdout errorlog: stdout -#debug: 2 +debug: 2 internal: 127.0.0.1 port = 3306 external: eth0 user.notprivileged: ec2-user @@ -9,12 +9,12 @@ socksmethod: none client pass { from: 127.0.0.1/32 to: 127.0.0.1/32 - log: error # connect disconnect iooperation + log: connect disconnect tcpinfo # connect disconnect iooperation } socks pass { from: 127.0.0.1/32 to: 0.0.0.0/0 command: bind connect protocol: tcp - log: error + log: connect disconnect tcpinfo } \ No newline at end of file diff --git a/scripts/aws/entrypoint.sh b/scripts/aws/entrypoint.sh index 8c99c26b9..1f350776d 100755 --- a/scripts/aws/entrypoint.sh +++ b/scripts/aws/entrypoint.sh @@ -16,7 +16,7 @@ ifconfig lo 127.0.0.1 # -- start vsock proxy echo "Starting vsock proxy..." -/app/vsockpx --config /app/proxies.nitro.yaml --daemon --workers $(( $(nproc) * 2 )) --log-level 3 +/app/vsockpx --config /app/proxies.nitro.yaml --daemon --workers $(( $(nproc) * 2 )) --log-level 0 # -- load config from identity service echo "Loading config from identity service via proxy..." diff --git a/scripts/aws/sockd.conf b/scripts/aws/sockd.conf index 6e8814445..d5e039c20 100644 --- a/scripts/aws/sockd.conf +++ b/scripts/aws/sockd.conf @@ -1,3 +1,6 @@ +logoutput: stdout +errorlog: stdout +debug: 2 internal: 127.0.0.1 port = 3306 external: ens5 user.notprivileged: ec2-user @@ -6,12 +9,12 @@ socksmethod: none client pass { from: 127.0.0.1/32 to: 127.0.0.1/32 - log: error # connect disconnect iooperation + log: connect disconnect tcpinfo # connect disconnect iooperation } socks pass { from: 127.0.0.1/32 to: 0.0.0.0/0 command: bind connect protocol: tcp - log: error + log: connect disconnect tcpinfo } \ No newline at end of file From 482382312d0859ebf2750375163cd7112ce3362b Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Mon, 7 Oct 2024 23:23:32 +0000 Subject: [PATCH 035/431] [CI Pipeline] Released Snapshot version: 5.40.54-alpha-30-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 661748e55..333ea1162 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.40.53-alpha-29-SNAPSHOT + 5.40.54-alpha-30-SNAPSHOT UTF-8 From 407e548b1023437f423f9604eb2ede98189f5441 Mon Sep 17 00:00:00 2001 From: Thomas Manson Date: Tue, 8 Oct 2024 13:40:56 +1100 Subject: [PATCH 036/431] Temp change the CID --- scripts/aws/eks-pod/entrypoint.sh | 2 +- scripts/aws/eks-pod/proxies.host.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/aws/eks-pod/entrypoint.sh b/scripts/aws/eks-pod/entrypoint.sh index 7573d8a9d..8d87326f0 100644 --- a/scripts/aws/eks-pod/entrypoint.sh +++ b/scripts/aws/eks-pod/entrypoint.sh @@ -1,5 +1,5 @@ #!/bin/bash -eufx -CID=42 +CID=43 EIF_PATH=/home/uid2operator.eif MEMORY_MB=24576 CPU_COUNT=6 diff --git a/scripts/aws/eks-pod/proxies.host.yaml b/scripts/aws/eks-pod/proxies.host.yaml index 5a2ae0623..649cc97f0 100644 --- a/scripts/aws/eks-pod/proxies.host.yaml +++ b/scripts/aws/eks-pod/proxies.host.yaml @@ -8,12 +8,12 @@ socks5h-proxy: operator-service: service: direct listen: tcp://0.0.0.0:80 - connect: vsock://42:8080 + connect: vsock://43:8080 operator-prometheus: service: direct listen: tcp://0.0.0.0:9080 - connect: vsock://42:9080 + connect: vsock://43:9080 syslogng: service: direct From 3b65da65708437a449a5fae6af23e2699f02a446 Mon Sep 17 00:00:00 2001 From: Thomas Manson Date: Tue, 8 Oct 2024 14:13:02 +1100 Subject: [PATCH 037/431] Revert CID change --- scripts/aws/eks-pod/entrypoint.sh | 2 +- scripts/aws/eks-pod/proxies.host.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/aws/eks-pod/entrypoint.sh b/scripts/aws/eks-pod/entrypoint.sh index 8d87326f0..7573d8a9d 100644 --- a/scripts/aws/eks-pod/entrypoint.sh +++ b/scripts/aws/eks-pod/entrypoint.sh @@ -1,5 +1,5 @@ #!/bin/bash -eufx -CID=43 +CID=42 EIF_PATH=/home/uid2operator.eif MEMORY_MB=24576 CPU_COUNT=6 diff --git a/scripts/aws/eks-pod/proxies.host.yaml b/scripts/aws/eks-pod/proxies.host.yaml index 649cc97f0..5a2ae0623 100644 --- a/scripts/aws/eks-pod/proxies.host.yaml +++ b/scripts/aws/eks-pod/proxies.host.yaml @@ -8,12 +8,12 @@ socks5h-proxy: operator-service: service: direct listen: tcp://0.0.0.0:80 - connect: vsock://43:8080 + connect: vsock://42:8080 operator-prometheus: service: direct listen: tcp://0.0.0.0:9080 - connect: vsock://43:9080 + connect: vsock://42:9080 syslogng: service: direct From b7dd2547b9150353e16437cd81ccb2b2b22a5b93 Mon Sep 17 00:00:00 2001 From: Cody Constine Date: Tue, 8 Oct 2024 13:38:28 -0600 Subject: [PATCH 038/431] Adding debug port to eks pod --- scripts/aws/eks-pod/proxies.host.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/aws/eks-pod/proxies.host.yaml b/scripts/aws/eks-pod/proxies.host.yaml index 5a2ae0623..da2e40b57 100644 --- a/scripts/aws/eks-pod/proxies.host.yaml +++ b/scripts/aws/eks-pod/proxies.host.yaml @@ -10,6 +10,11 @@ operator-service: listen: tcp://0.0.0.0:80 connect: vsock://42:8080 +operator-debug: + service: direct + listen: tcp://0.0.0.0:8000 + connect: vsock://42:8000 + operator-prometheus: service: direct listen: tcp://0.0.0.0:9080 From efbc1ac464a4bb012eaa16cd6ae02f040fd6df2a Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Tue, 8 Oct 2024 19:41:09 +0000 Subject: [PATCH 039/431] [CI Pipeline] Released Snapshot version: 5.40.55-alpha-31-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 333ea1162..a3b752d15 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.40.54-alpha-30-SNAPSHOT + 5.40.55-alpha-31-SNAPSHOT UTF-8 From 579fdc19f6f59500a01646762f7288ddb28462f2 Mon Sep 17 00:00:00 2001 From: asloob qureshi Date: Thu, 10 Oct 2024 12:12:52 -0700 Subject: [PATCH 040/431] Remove assertions used to check argument validity in public methods --- src/main/java/com/uid2/operator/util/Tuple.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/uid2/operator/util/Tuple.java b/src/main/java/com/uid2/operator/util/Tuple.java index 491413de4..979e79b4e 100644 --- a/src/main/java/com/uid2/operator/util/Tuple.java +++ b/src/main/java/com/uid2/operator/util/Tuple.java @@ -6,8 +6,9 @@ public static class Tuple2 { private final T2 item2; public Tuple2(T1 item1, T2 item2) { - assert item1 != null; - assert item2 != null; + if (item1 == null || item2 == null) { + throw new NullPointerException(); + } this.item1 = item1; this.item2 = item2; @@ -34,9 +35,9 @@ public static class Tuple3 { private final T3 item3; public Tuple3(T1 item1, T2 item2, T3 item3) { - assert item1 != null; - assert item2 != null; - assert item3 != null; + if (item1 == null || item2 == null || item3 == null) { + throw new NullPointerException(); + } this.item1 = item1; this.item2 = item2; From 44093af76d8d0ed91c14abb6038fff36d4762f91 Mon Sep 17 00:00:00 2001 From: asloob qureshi Date: Thu, 10 Oct 2024 12:23:55 -0700 Subject: [PATCH 041/431] Exclude code and assertion from source needed only for tests --- .../operator/service/UIDOperatorService.java | 14 +++-------- .../uid2/operator/UIDOperatorServiceTest.java | 25 ++++++++++++++++--- 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/src/main/java/com/uid2/operator/service/UIDOperatorService.java b/src/main/java/com/uid2/operator/service/UIDOperatorService.java index 99d769e17..672cec238 100644 --- a/src/main/java/com/uid2/operator/service/UIDOperatorService.java +++ b/src/main/java/com/uid2/operator/service/UIDOperatorService.java @@ -47,9 +47,9 @@ public class UIDOperatorService implements IUIDOperatorService { private final Duration refreshIdentityAfter; private final OperatorIdentity operatorIdentity; - private final TokenVersion tokenVersionToUseIfNotV4; - private final int advertisingTokenV4Percentage; - private final Set siteIdsUsingV4Tokens; + protected final TokenVersion tokenVersionToUseIfNotV4; + protected final int advertisingTokenV4Percentage; + protected final Set siteIdsUsingV4Tokens; private final TokenVersion refreshTokenVersion; private final boolean identityV3Enabled; @@ -319,12 +319,4 @@ private GlobalOptoutResult getGlobalOptOutResult(UserIdentity userIdentity, bool Instant result = this.optOutStore.getLatestEntry(userIdentity); return new GlobalOptoutResult(result); } - - public TokenVersion getAdvertisingTokenVersionForTests(int siteId) { - assert this.advertisingTokenV4Percentage == 0 || this.advertisingTokenV4Percentage == 100; //we want tests to be deterministic - if (this.siteIdsUsingV4Tokens.contains(siteId)) { - return TokenVersion.V4; - } - return this.advertisingTokenV4Percentage == 100 ? TokenVersion.V4 : this.tokenVersionToUseIfNotV4; - } } diff --git a/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java b/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java index 79a8a104b..bc1d49bb2 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java @@ -3,17 +3,20 @@ import com.uid2.operator.model.*; import com.uid2.operator.service.EncodingUtils; import com.uid2.operator.service.EncryptedTokenEncoder; +import com.uid2.operator.service.ITokenEncoder; import com.uid2.operator.service.InputUtil; import com.uid2.operator.service.UIDOperatorService; import com.uid2.operator.store.IOptOutStore; import com.uid2.operator.vertx.OperatorShutdownHandler; import com.uid2.shared.store.CloudPath; +import com.uid2.shared.store.ISaltProvider; import com.uid2.shared.store.RotatingSaltProvider; import com.uid2.shared.cloud.EmbeddedResourceStorage; import com.uid2.shared.store.reader.RotatingKeysetKeyStore; import com.uid2.shared.store.reader.RotatingKeysetProvider; import com.uid2.shared.store.scope.GlobalScope; import com.uid2.shared.model.TokenVersion; +import io.vertx.core.Handler; import io.vertx.core.json.JsonObject; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -42,14 +45,28 @@ public class UIDOperatorServiceTest { EncryptedTokenEncoder tokenEncoder; JsonObject uid2Config; JsonObject euidConfig; - UIDOperatorService uid2Service; - UIDOperatorService euidService; + ExtendedUIDOperatorService uid2Service; + ExtendedUIDOperatorService euidService; Instant now; final int IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS = 600; final int REFRESH_TOKEN_EXPIRES_AFTER_SECONDS = 900; final int REFRESH_IDENTITY_TOKEN_AFTER_SECONDS = 300; + class ExtendedUIDOperatorService extends UIDOperatorService { + public ExtendedUIDOperatorService(JsonObject config, IOptOutStore optOutStore, ISaltProvider saltProvider, ITokenEncoder encoder, Clock clock, IdentityScope identityScope, Handler saltRetrievalResponseHandler) { + super(config, optOutStore, saltProvider, encoder, clock, identityScope, saltRetrievalResponseHandler); + } + + public TokenVersion getAdvertisingTokenVersionForTests(int siteId) { + assert this.advertisingTokenV4Percentage == 0 || this.advertisingTokenV4Percentage == 100; //we want tests to be deterministic + if (this.siteIdsUsingV4Tokens.contains(siteId)) { + return TokenVersion.V4; + } + return this.advertisingTokenV4Percentage == 100 ? TokenVersion.V4 : this.tokenVersionToUseIfNotV4; + } + } + @BeforeEach void setup() throws Exception { mocks = MockitoAnnotations.openMocks(this); @@ -84,7 +101,7 @@ void setup() throws Exception { uid2Config.put("advertising_token_v3", false); // prod is using v2 token version for now uid2Config.put("identity_v3", false); - uid2Service = new UIDOperatorService( + uid2Service = new ExtendedUIDOperatorService( uid2Config, optOutStore, saltProvider, @@ -102,7 +119,7 @@ void setup() throws Exception { euidConfig.put("advertising_token_v3", true); euidConfig.put("identity_v3", true); - euidService = new UIDOperatorService( + euidService = new ExtendedUIDOperatorService( euidConfig, optOutStore, saltProvider, From b91591be805f7b46975342d8a0868f358bcdc5f9 Mon Sep 17 00:00:00 2001 From: asloob qureshi Date: Thu, 10 Oct 2024 14:24:14 -0700 Subject: [PATCH 042/431] replace assert for refresh token length with exception --- .../uid2/operator/service/V2RequestUtil.java | 7 ++- .../com/uid2/operator/V2RequestUtilTest.java | 50 +++++++++++++++++-- 2 files changed, 52 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/uid2/operator/service/V2RequestUtil.java b/src/main/java/com/uid2/operator/service/V2RequestUtil.java index d9737323b..cd4983865 100644 --- a/src/main/java/com/uid2/operator/service/V2RequestUtil.java +++ b/src/main/java/com/uid2/operator/service/V2RequestUtil.java @@ -172,7 +172,12 @@ public static void handleRefreshTokenInResponseBody(JsonObject bodyJson, KeyMana .appendInt(refreshKey.getId()) .appendBytes(encrypted) .getBytes()); - assert modifiedToken.length() == V2_REFRESH_PAYLOAD_LENGTH; + if (modifiedToken.length() != V2_REFRESH_PAYLOAD_LENGTH) { + final String errorMsg = "Generated refresh token's length=" + modifiedToken.length() + + " is not equal to=" + V2_REFRESH_PAYLOAD_LENGTH; + LOGGER.error(errorMsg); + throw new IllegalArgumentException(errorMsg); + } bodyJson.put("refresh_token", modifiedToken); bodyJson.put("refresh_response_key", refreshResponseKey); diff --git a/src/test/java/com/uid2/operator/V2RequestUtilTest.java b/src/test/java/com/uid2/operator/V2RequestUtilTest.java index 008583cb8..dcf4fa38b 100644 --- a/src/test/java/com/uid2/operator/V2RequestUtilTest.java +++ b/src/test/java/com/uid2/operator/V2RequestUtilTest.java @@ -1,28 +1,40 @@ package com.uid2.operator; -import com.uid2.operator.service.V2RequestUtil; import ch.qos.logback.classic.Level; import ch.qos.logback.classic.Logger; import ch.qos.logback.classic.LoggerContext; +import com.uid2.operator.model.IdentityScope; +import com.uid2.operator.model.KeyManager; +import com.uid2.operator.service.V2RequestUtil; import com.uid2.shared.IClock; import com.uid2.shared.auth.ClientKey; +import com.uid2.shared.encryption.Random; +import com.uid2.shared.model.KeysetKey; import io.vertx.core.json.JsonObject; import org.junit.Test; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; import org.slf4j.LoggerFactory; import java.time.Instant; import java.util.Set; -import static org.junit.jupiter.api.Assertions.*; import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class V2RequestUtilTest { private static final String LOGGER_NAME = "com.uid2.operator.service.V2RequestUtil"; private static MemoryAppender memoryAppender; private IClock clock = mock(IClock.class); private Instant mockNow = Instant.parse("2024-03-20T04:02:46.130Z"); + private AutoCloseable mocks; + KeyManager keyManager = Mockito.mock(KeyManager.class); + KeysetKey refreshKey = Mockito.mock(KeysetKey.class); public void setupMemoryAppender() { Logger logger = (Logger)LoggerFactory.getLogger(LOGGER_NAME); @@ -33,10 +45,16 @@ public void setupMemoryAppender() { memoryAppender.start(); } + @BeforeEach + public void setup() { + mocks = MockitoAnnotations.openMocks(this); + } + @AfterEach - public void close() { + public void close() throws Exception { memoryAppender.reset(); memoryAppender.stop(); + mocks.close(); } @Test @@ -118,4 +136,28 @@ public void testParseRequestWithMalformedJson() { assertThat(memoryAppender.search("[ERROR] Invalid payload in body: Data is not valid json string.").size()).isEqualTo(1); assertThat(memoryAppender.checkNoThrowableLogged().size()).isEqualTo(1); } + + @Test + public void testHandleRefreshTokenInResponseBody() { + String response = "{\n" + + " \"identity\": {\n" + + " \"advertising_token\": \"A4AAABZBgXozOcvdoBLWXaJSltTRG27n1kFegS9IKt-wN8bUPIPKiUXu9gxOzB0CvYprD8-tJNJjYNUy_HegQ1DdWkHwTm9vz9C2PUPtWzZenVy3g5L3hrbD_c7GuA6M6suZAkQGgeRM-7ixjVK2iUKYs5fOgxqzAl21St-7Bm97mgUEoMmg37bW5-X9w3TVs6PAUgSF2DuQmmwVXeKIsmoQZA\",\n" + + " \"refresh_token\": \"AAAAFkKfY/PfFkWOByfIqQpP/nWp70ULyurGFQU7CUs5VWWhSgvzFRqXBes5DBqn6GKtwgKH/dF1Cx6Id951RnumXMJ5Oebw4vxQSvtGMNroN1B6HuPZcZiMnvDaTKjCZSAMd6Rc61pZzaQQ7wDKNP9NHNIzRmp7oziVlnEkT/sTJFfZZQPMFjWNqPy2nR0CFg8Zxui5ac6Ix9KEIFXOPM2v1O3kUm5E6x8MJ4vRLclK3NtAbWE3imauSpGSVlqG12hQKEBfN5CbcGRtdQGzdZoWjl8adZQdovufwulg59o8yKrEVPpL7wmoQ5oBaG9GG+FZMx4ttzkS/UlW+uk5qxUopeCRsuOSD/zWAsDDPP+6/FFuIMj+ftASZ7gXVaDraWqD\",\n" + + " \"identity_expires\": 1728595268736,\n" + + " \"refresh_expires\": 1731186368736,\n" + + " \"refresh_from\": 1728594668736,\n" + + " \"refresh_response_key\": \"sMRiJivNZJ6msQSvZhsVooG2T/xXTigaFRBPFHCPGQQ=\"\n" + + " }\n" + + "}"; + JsonObject jsonBody = new JsonObject(response); + when(keyManager.getRefreshKey()).thenReturn(refreshKey); + when(refreshKey.getId()).thenReturn(Integer.MAX_VALUE); + when(refreshKey.getKeyBytes()).thenReturn(Random.getRandomKeyBytes()); + try { + V2RequestUtil.handleRefreshTokenInResponseBody(jsonBody, keyManager, IdentityScope.UID2); + fail("IllegalArgumentException not thrown"); + } catch (IllegalArgumentException e) { + assertEquals("Generated refresh token's length=168 is not equal to=388", e.getMessage()); + } + } } From 0a82473468a77dca2719870a8177bf939edc991e Mon Sep 17 00:00:00 2001 From: asloob qureshi Date: Thu, 10 Oct 2024 14:29:27 -0700 Subject: [PATCH 043/431] Replace assert with NPE in statsCollector public methods --- .../com/uid2/operator/monitoring/StatsCollectorHandler.java | 4 +++- .../com/uid2/operator/monitoring/StatsCollectorVerticle.java | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/uid2/operator/monitoring/StatsCollectorHandler.java b/src/main/java/com/uid2/operator/monitoring/StatsCollectorHandler.java index 04a36d9c1..ebeb304d5 100644 --- a/src/main/java/com/uid2/operator/monitoring/StatsCollectorHandler.java +++ b/src/main/java/com/uid2/operator/monitoring/StatsCollectorHandler.java @@ -20,7 +20,9 @@ public StatsCollectorHandler(IStatsCollectorQueue _statCollectorQueue, Vertx ver @Override public void handle(RoutingContext routingContext) { - assert routingContext != null; + if (routingContext == null) { + throw new NullPointerException(); + } //setAuthClient() has not yet been called, so getAuthClient() would return null. This is resolved by using addBodyEndHandler() routingContext.addBodyEndHandler(v -> addStatsMessageToQueue(routingContext)); diff --git a/src/main/java/com/uid2/operator/monitoring/StatsCollectorVerticle.java b/src/main/java/com/uid2/operator/monitoring/StatsCollectorVerticle.java index 8e49ec1f8..e6742356a 100644 --- a/src/main/java/com/uid2/operator/monitoring/StatsCollectorVerticle.java +++ b/src/main/java/com/uid2/operator/monitoring/StatsCollectorVerticle.java @@ -88,7 +88,9 @@ public void handleMessage(Message message) { return; } - assert messageItem != null; + if (messageItem == null) { + throw new NullPointerException("Message could not be deserialized"); + } String path = messageItem.getPath(); String apiVersion = "v0"; From 21555c064945470fdf5da0be9d44de4628fd21f0 Mon Sep 17 00:00:00 2001 From: Thomas Manson Date: Fri, 11 Oct 2024 13:17:52 +1100 Subject: [PATCH 044/431] Update the handler for shutdown --- pom.xml | 2 +- src/main/java/com/uid2/operator/Main.java | 4 ++-- .../vertx/OperatorShutdownHandler.java | 10 ++++++---- .../operator/OperatorShutdownHandlerTest.java | 18 ++++++++++-------- 4 files changed, 19 insertions(+), 15 deletions(-) diff --git a/pom.xml b/pom.xml index 082c67876..8db01ac81 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ 2.1.0 2.1.0 2.1.0 - 7.19.0 + 7.19.1-SNAPSHOT ${project.version} 21 21 diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index dad32611d..6aa069604 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -467,14 +467,14 @@ public DistributionStatisticConfig configure(Meter.Id id, DistributionStatisticC .register(globalRegistry); } - private Map.Entry createUidClients(Vertx vertx, String attestationUrl, String clientApiToken, Handler> responseWatcher) throws Exception { + private Map.Entry createUidClients(Vertx vertx, String attestationUrl, String clientApiToken, Handler> responseWatcher) throws Exception { AttestationResponseHandler attestationResponseHandler = getAttestationTokenRetriever(vertx, attestationUrl, clientApiToken, responseWatcher); UidCoreClient coreClient = new UidCoreClient(clientApiToken, CloudUtils.defaultProxy, attestationResponseHandler); UidOptOutClient optOutClient = new UidOptOutClient(clientApiToken, CloudUtils.defaultProxy, attestationResponseHandler); return new AbstractMap.SimpleEntry<>(coreClient, optOutClient); } - private AttestationResponseHandler getAttestationTokenRetriever(Vertx vertx, String attestationUrl, String clientApiToken, Handler> responseWatcher) throws Exception { + private AttestationResponseHandler getAttestationTokenRetriever(Vertx vertx, String attestationUrl, String clientApiToken, Handler> responseWatcher) throws Exception { String enclavePlatform = this.config.getString(Const.Config.EnclavePlatformProp); String operatorType = this.config.getString(Const.Config.OperatorTypeProp, ""); diff --git a/src/main/java/com/uid2/operator/vertx/OperatorShutdownHandler.java b/src/main/java/com/uid2/operator/vertx/OperatorShutdownHandler.java index 113c14d3e..84075fb03 100644 --- a/src/main/java/com/uid2/operator/vertx/OperatorShutdownHandler.java +++ b/src/main/java/com/uid2/operator/vertx/OperatorShutdownHandler.java @@ -1,6 +1,8 @@ package com.uid2.operator.vertx; import com.uid2.operator.service.ShutdownService; +import com.uid2.shared.attest.AttestationResponseCode; +import lombok.extern.java.Log; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.utils.Pair; @@ -52,12 +54,12 @@ public void logSaltFailureAtInterval() { } } - public void handleAttestResponse(Pair response) { - if (response.left() == 401) { - LOGGER.error("core attestation failed with 401, shutting down operator, core response: " + response.right()); + public void handleAttestResponse(Pair response) { + if (response.left() == AttestationResponseCode.AttestationFailure) { + LOGGER.error("core attestation failed with AttestationFailure, shutting down operator, core response: {}", response.right()); this.shutdownService.Shutdown(1); } - if (response.left() == 200) { + if (response.left() == AttestationResponseCode.Success) { attestFailureStartTime.set(null); } else { Instant t = attestFailureStartTime.get(); diff --git a/src/test/java/com/uid2/operator/OperatorShutdownHandlerTest.java b/src/test/java/com/uid2/operator/OperatorShutdownHandlerTest.java index e4323226c..10a00b813 100644 --- a/src/test/java/com/uid2/operator/OperatorShutdownHandlerTest.java +++ b/src/test/java/com/uid2/operator/OperatorShutdownHandlerTest.java @@ -5,6 +5,7 @@ import ch.qos.logback.core.read.ListAppender; import com.uid2.operator.service.ShutdownService; import com.uid2.operator.vertx.OperatorShutdownHandler; +import com.uid2.shared.attest.AttestationResponseCode; import io.vertx.core.Vertx; import io.vertx.junit5.VertxExtension; import io.vertx.junit5.VertxTestContext; @@ -51,17 +52,18 @@ void afterEach() throws Exception { } @Test - void shutdownOnAttest401(VertxTestContext testContext) { + void shutdownOnAttestFailure(VertxTestContext testContext) { ListAppender logWatcher = new ListAppender<>(); logWatcher.start(); ((Logger) LoggerFactory.getLogger(OperatorShutdownHandler.class)).addAppender(logWatcher); // Revoke auth try { - this.operatorShutdownHandler.handleAttestResponse(Pair.of(401, "Unauthorized")); + this.operatorShutdownHandler.handleAttestResponse(Pair.of(AttestationResponseCode.AttestationFailure, "Unauthorized")); } catch (RuntimeException e) { verify(shutdownService).Shutdown(1); - Assertions.assertTrue(logWatcher.list.get(0).getFormattedMessage().contains("core attestation failed with 401, shutting down operator, core response: ")); + String message = logWatcher.list.get(0).getFormattedMessage(); + Assertions.assertEquals("core attestation failed with AttestationFailure, shutting down operator, core response: Unauthorized", logWatcher.list.get(0).getFormattedMessage()); testContext.completeNow(); } } @@ -72,11 +74,11 @@ void shutdownOnAttestFailedTooLong(VertxTestContext testContext) { logWatcher.start(); ((Logger) LoggerFactory.getLogger(OperatorShutdownHandler.class)).addAppender(logWatcher); - this.operatorShutdownHandler.handleAttestResponse(Pair.of(500, "")); + this.operatorShutdownHandler.handleAttestResponse(Pair.of(AttestationResponseCode.RetryableFailure, "")); when(clock.instant()).thenAnswer(i -> Instant.now().plus(12, ChronoUnit.HOURS).plusSeconds(60)); try { - this.operatorShutdownHandler.handleAttestResponse(Pair.of(500, "")); + this.operatorShutdownHandler.handleAttestResponse(Pair.of(AttestationResponseCode.RetryableFailure, "")); } catch (RuntimeException e) { verify(shutdownService).Shutdown(1); Assertions.assertTrue(logWatcher.list.get(0).getFormattedMessage().contains("core attestation has been in failed state for too long. shutting down operator")); @@ -90,13 +92,13 @@ void attestRecoverOnSuccess(VertxTestContext testContext) { logWatcher.start(); ((Logger) LoggerFactory.getLogger(OperatorShutdownHandler.class)).addAppender(logWatcher); - this.operatorShutdownHandler.handleAttestResponse(Pair.of(500, "")); + this.operatorShutdownHandler.handleAttestResponse(Pair.of(AttestationResponseCode.RetryableFailure, "")); when(clock.instant()).thenAnswer(i -> Instant.now().plus(6, ChronoUnit.HOURS)); - this.operatorShutdownHandler.handleAttestResponse(Pair.of(200, "")); + this.operatorShutdownHandler.handleAttestResponse(Pair.of(AttestationResponseCode.Success, "")); when(clock.instant()).thenAnswer(i -> Instant.now().plus(12, ChronoUnit.HOURS)); assertDoesNotThrow(() -> { - this.operatorShutdownHandler.handleAttestResponse(Pair.of(500, "")); + this.operatorShutdownHandler.handleAttestResponse(Pair.of(AttestationResponseCode.RetryableFailure, "")); }); verify(shutdownService, never()).Shutdown(anyInt()); testContext.completeNow(); From d45f4cf5c28e963b6a2945144de54254c3990568 Mon Sep 17 00:00:00 2001 From: Thomas Manson Date: Fri, 11 Oct 2024 13:27:36 +1100 Subject: [PATCH 045/431] Turn of the exit on error --- scripts/aws/eks-pod/entrypoint.sh | 4 ++-- scripts/aws/entrypoint.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/aws/eks-pod/entrypoint.sh b/scripts/aws/eks-pod/entrypoint.sh index 7573d8a9d..11f643c9b 100644 --- a/scripts/aws/eks-pod/entrypoint.sh +++ b/scripts/aws/eks-pod/entrypoint.sh @@ -1,4 +1,4 @@ -#!/bin/bash -eufx +#!/bin/bash -ufx CID=42 EIF_PATH=/home/uid2operator.eif MEMORY_MB=24576 @@ -30,7 +30,7 @@ function setup_vsockproxy() { VSOCK_PROXY=${VSOCK_PROXY:-/home/vsockpx} VSOCK_CONFIG=${VSOCK_CONFIG:-/home/proxies.host.yaml} VSOCK_THREADS=${VSOCK_THREADS:-$(( $(nproc) * 2 )) } - VSOCK_LOG_LEVEL=0 + VSOCK_LOG_LEVEL=1 echo "starting vsock proxy at $VSOCK_PROXY with $VSOCK_THREADS worker threads..." $VSOCK_PROXY -c $VSOCK_CONFIG --workers $VSOCK_THREADS --log-level $VSOCK_LOG_LEVEL --daemon echo "vsock proxy now running in background." diff --git a/scripts/aws/entrypoint.sh b/scripts/aws/entrypoint.sh index 1f350776d..5ae1df745 100755 --- a/scripts/aws/entrypoint.sh +++ b/scripts/aws/entrypoint.sh @@ -1,4 +1,4 @@ -#!/bin/bash -eufx +#!/bin/bash -ufx # This is the entrypoint for the Enclave. It is executed in all enclaves - EC2 and EKS @@ -16,7 +16,7 @@ ifconfig lo 127.0.0.1 # -- start vsock proxy echo "Starting vsock proxy..." -/app/vsockpx --config /app/proxies.nitro.yaml --daemon --workers $(( $(nproc) * 2 )) --log-level 0 +/app/vsockpx --config /app/proxies.nitro.yaml --daemon --workers $(( $(nproc) * 2 )) --log-level 1 # -- load config from identity service echo "Loading config from identity service via proxy..." From 912a6a93cf40d74d7d51d9ec51298b862b0692f9 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Fri, 11 Oct 2024 02:29:37 +0000 Subject: [PATCH 046/431] [CI Pipeline] Released Snapshot version: 5.40.56-alpha-34-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a3b752d15..04a610078 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.40.55-alpha-31-SNAPSHOT + 5.40.56-alpha-34-SNAPSHOT UTF-8 From bd6438af15e50da91c01366d74e250c5aa777c31 Mon Sep 17 00:00:00 2001 From: asloob qureshi Date: Fri, 11 Oct 2024 11:58:46 -0700 Subject: [PATCH 047/431] Convert assert in default switch case to exception. Refactor methods --- .../operator/vertx/UIDOperatorVerticle.java | 25 ++++++++----------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index 210b08571..8c69e7ccd 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -450,7 +450,7 @@ else if(emailHash != null) { input = InputUtil.normalizePhoneHash(phoneHash); } - if (!checkForInvalidTokenInput(input, rc)) { + if (!isTokenInputValid(input, rc)) { return; } @@ -869,7 +869,7 @@ private void handleTokenRefreshV2(RoutingContext rc) { private void handleTokenValidateV1(RoutingContext rc) { try { final InputUtil.InputVal input = this.phoneSupport ? getTokenInputV1(rc) : getTokenInput(rc); - if (!checkForInvalidTokenInput(input, rc)) { + if (!isTokenInputValid(input, rc)) { return; } if ((Arrays.equals(ValidateIdentityForEmailHash, input.getIdentityInput()) && input.getIdentityType() == IdentityType.Email) @@ -900,7 +900,7 @@ private void handleTokenValidateV2(RoutingContext rc) { final JsonObject req = (JsonObject) rc.data().get("request"); final InputUtil.InputVal input = getTokenInputV2(req); - if (!checkForInvalidTokenInput(input, rc)) { + if (!isTokenInputValid(input, rc)) { return; } if ((input.getIdentityType() == IdentityType.Email && Arrays.equals(ValidateIdentityForEmailHash, input.getIdentityInput())) @@ -932,17 +932,13 @@ private void handleTokenGenerateV1(RoutingContext rc) { try { final InputUtil.InputVal input = this.phoneSupport ? this.getTokenInputV1(rc) : this.getTokenInput(rc); platformType = getPlatformType(rc); - if (!checkForInvalidTokenInput(input, rc)) { - return; - } else { + if (isTokenInputValid(input, rc)) { final IdentityTokens t = this.idService.generateIdentity( new IdentityRequest( new PublisherIdentity(siteId, 0, 0), input.toUserIdentity(this.identityScope, 1, Instant.now()), OptoutCheckPolicy.defaultPolicy())); - //Integer.parseInt(rc.queryParam("privacy_bits").get(0)))); - ResponseUtil.Success(rc, toJsonV1(t)); recordTokenResponseStats(siteId, TokenResponseStatsCollector.Endpoint.GenerateV1, TokenResponseStatsCollector.ResponseStatus.Success, siteProvider, t.getAdvertisingTokenVersion(), platformType); } @@ -959,9 +955,7 @@ private void handleTokenGenerateV2(RoutingContext rc) { platformType = getPlatformType(rc); final InputUtil.InputVal input = this.getTokenInputV2(req); - if (!checkForInvalidTokenInput(input, rc)) { - return; - } else { + if (isTokenInputValid(input, rc)) { final String apiContact = getApiContact(rc); switch (validateUserConsent(req)) { @@ -978,8 +972,9 @@ private void handleTokenGenerateV2(RoutingContext rc) { break; } default: { - assert false : "Please update UIDOperatorVerticle.handleTokenGenerateV2 when changing UserConsentStatus"; - break; + final String errorMsg = "Please update UIDOperatorVerticle.handleTokenGenerateV2 when changing UserConsentStatus"; + LOGGER.error(errorMsg); + throw new IllegalStateException(errorMsg); } } @@ -1234,7 +1229,7 @@ private void handleBucketsV2(RoutingContext rc) { private void handleIdentityMapV1(RoutingContext rc) { final InputUtil.InputVal input = this.phoneSupport ? this.getTokenInputV1(rc) : this.getTokenInput(rc); - if (!checkForInvalidTokenInput(input, rc)) { + if (!isTokenInputValid(input, rc)) { return; } try { @@ -1363,7 +1358,7 @@ private InputUtil.InputVal getTokenInputV1(RoutingContext rc) { return null; } - private boolean checkForInvalidTokenInput(InputUtil.InputVal input, RoutingContext rc) { + private boolean isTokenInputValid(InputUtil.InputVal input, RoutingContext rc) { if (input == null) { String message = this.phoneSupport ? "Required Parameter Missing: exactly one of [email, email_hash, phone, phone_hash] must be specified" : "Required Parameter Missing: exactly one of email or email_hash must be specified"; ResponseUtil.ClientError(rc, message); From 6e964c6a934cf571efc3c10c12bc3adc3dfc0adb Mon Sep 17 00:00:00 2001 From: asloob qureshi Date: Fri, 11 Oct 2024 12:09:08 -0700 Subject: [PATCH 048/431] Refactor error messages to static string --- .../operator/vertx/UIDOperatorVerticle.java | 27 ++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index 8c69e7ccd..13d1d0c1c 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -45,7 +45,6 @@ import io.vertx.core.json.JsonArray; import io.vertx.core.json.JsonObject; import io.vertx.ext.web.AllowForwardHeaders; -import io.vertx.ext.web.Route; import io.vertx.ext.web.Router; import io.vertx.ext.web.RoutingContext; import io.vertx.ext.web.handler.BodyHandler; @@ -129,6 +128,10 @@ public class UIDOperatorVerticle extends AbstractVerticle { //"Android" is from https://github.com/IABTechLab/uid2-android-sdk/blob/ff93ebf597f5de7d440a84f7015a334ba4138ede/sdk/src/main/java/com/uid2/UID2Client.kt#L46 //"ios"/"tvos" is from https://github.com/IABTechLab/uid2-ios-sdk/blob/91c290d29a7093cfc209eca493d1fee80c17e16a/Sources/UID2/UID2Client.swift#L36-L38 private final static List SUPPORTED_IN_APP = Arrays.asList("Android", "ios", "tvos"); + + private static final String ERROR_INVALID_INPUT_WITH_PHONE_SUPPORT = "Required Parameter Missing: exactly one of [email, email_hash, phone, phone_hash] must be specified"; + private static final String ERROR_INVALID_INPUT_EMAIL_MISSING = "Required Parameter Missing: exactly one of email or email_hash must be specified"; + private static final String ERROR_INVALID_INPUT_EMAIL_TWICE = "Only one of email or email_hash can be specified"; public final static String ORIGIN_HEADER = "Origin"; public UIDOperatorVerticle(JsonObject config, @@ -1032,7 +1035,7 @@ private void handleTokenGenerate(RoutingContext rc) { final InputUtil.InputVal input = this.getTokenInput(rc); Integer siteId = null; if (input == null) { - SendClientErrorResponseAndRecordStats(ResponseStatus.ClientError, 400, rc, "Required Parameter Missing: exactly one of email or email_hash must be specified", siteId, TokenResponseStatsCollector.Endpoint.GenerateV0, TokenResponseStatsCollector.ResponseStatus.BadPayload, siteProvider, TokenResponseStatsCollector.PlatformType.Other); + SendClientErrorResponseAndRecordStats(ResponseStatus.ClientError, 400, rc, ERROR_INVALID_INPUT_EMAIL_MISSING, siteId, TokenResponseStatsCollector.Endpoint.GenerateV0, TokenResponseStatsCollector.ResponseStatus.BadPayload, siteProvider, TokenResponseStatsCollector.PlatformType.Other); return; } else if (!input.isValid()) { @@ -1250,7 +1253,7 @@ private void handleIdentityMap(RoutingContext rc) { try { if (input == null) { - ResponseUtil.ClientError(rc, "Required Parameter Missing: exactly one of email or email_hash must be specified"); + ResponseUtil.ClientError(rc, ERROR_INVALID_INPUT_EMAIL_MISSING); } else if (!input.isValid()) { ResponseUtil.ClientError(rc, "Invalid email or email_hash"); @@ -1360,7 +1363,7 @@ private InputUtil.InputVal getTokenInputV1(RoutingContext rc) { private boolean isTokenInputValid(InputUtil.InputVal input, RoutingContext rc) { if (input == null) { - String message = this.phoneSupport ? "Required Parameter Missing: exactly one of [email, email_hash, phone, phone_hash] must be specified" : "Required Parameter Missing: exactly one of email or email_hash must be specified"; + String message = this.phoneSupport ? ERROR_INVALID_INPUT_WITH_PHONE_SUPPORT : ERROR_INVALID_INPUT_EMAIL_MISSING; ResponseUtil.ClientError(rc, message); return false; } else if (!input.isValid()) { @@ -1376,11 +1379,11 @@ private InputUtil.InputVal[] getIdentityBulkInput(RoutingContext rc) { final JsonArray emailHashes = obj.getJsonArray("email_hash"); // FIXME TODO. Avoid Double Iteration. Turn to a decorator pattern if (emails == null && emailHashes == null) { - ResponseUtil.ClientError(rc, "Exactly one of email or email_hash must be specified"); + ResponseUtil.ClientError(rc, ERROR_INVALID_INPUT_EMAIL_MISSING); return null; } else if (emails != null && !emails.isEmpty()) { if (emailHashes != null && !emailHashes.isEmpty()) { - ResponseUtil.ClientError(rc, "Only one of email or email_hash can be specified"); + ResponseUtil.ClientError(rc, ERROR_INVALID_INPUT_EMAIL_TWICE); return null; } return createInputList(emails, false); @@ -1393,7 +1396,7 @@ private InputUtil.InputVal[] getIdentityBulkInput(RoutingContext rc) { private InputUtil.InputVal[] getIdentityBulkInputV1(RoutingContext rc) { final JsonObject obj = rc.body().asJsonObject(); if(obj.isEmpty()) { - ResponseUtil.ClientError(rc, "Exactly one of [email, email_hash, phone, phone_hash] must be specified"); + ResponseUtil.ClientError(rc, ERROR_INVALID_INPUT_WITH_PHONE_SUPPORT); return null; } final JsonArray emails = JsonParseUtils.parseArray(obj, "email", rc); @@ -1425,7 +1428,7 @@ private InputUtil.InputVal[] getIdentityBulkInputV1(RoutingContext rc) { } if (validInputs == 0 || nonEmptyInputs > 1) { - ResponseUtil.ClientError(rc, "Exactly one of [email, email_hash, phone, phone_hash] must be specified"); + ResponseUtil.ClientError(rc, ERROR_INVALID_INPUT_WITH_PHONE_SUPPORT); return null; } @@ -1506,9 +1509,9 @@ private void handleIdentityMapV2(RoutingContext rc) { final InputUtil.InputVal[] inputList = getIdentityMapV2Input(rc); if (inputList == null) { if (this.phoneSupport) - ResponseUtil.ClientError(rc, "Exactly one of [email, email_hash, phone, phone_hash] must be specified"); + ResponseUtil.ClientError(rc, ERROR_INVALID_INPUT_WITH_PHONE_SUPPORT); else - ResponseUtil.ClientError(rc, "Required Parameter Missing: exactly one of email or email_hash must be specified"); + ResponseUtil.ClientError(rc, ERROR_INVALID_INPUT_EMAIL_MISSING); return; } @@ -1574,11 +1577,11 @@ private void handleIdentityMapBatch(RoutingContext rc) { final JsonArray emails = obj.getJsonArray("email"); final JsonArray emailHashes = obj.getJsonArray("email_hash"); if (emails == null && emailHashes == null) { - ResponseUtil.ClientError(rc, "Exactly one of email or email_hash must be specified"); + ResponseUtil.ClientError(rc, ERROR_INVALID_INPUT_EMAIL_MISSING); return; } else if (emails != null && !emails.isEmpty()) { if (emailHashes != null && !emailHashes.isEmpty()) { - ResponseUtil.ClientError(rc, "Only one of email or email_hash can be specified"); + ResponseUtil.ClientError(rc, ERROR_INVALID_INPUT_EMAIL_TWICE); return; } inputList = createInputList(emails, false); From 9c249353128e331adb42db6a47036509155f969c Mon Sep 17 00:00:00 2001 From: asloob qureshi Date: Fri, 11 Oct 2024 12:14:05 -0700 Subject: [PATCH 049/431] refactor redundant code to call function --- .../com/uid2/operator/vertx/UIDOperatorVerticle.java | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index 13d1d0c1c..c515f33d7 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -1051,8 +1051,6 @@ else if (!input.isValid()) { input.toUserIdentity(this.identityScope, 1, Instant.now()), OptoutCheckPolicy.defaultPolicy())); - //Integer.parseInt(rc.queryParam("privacy_bits").get(0)))); - recordTokenResponseStats(siteId, TokenResponseStatsCollector.Endpoint.GenerateV0, TokenResponseStatsCollector.ResponseStatus.Success, siteProvider, t.getAdvertisingTokenVersion(), TokenResponseStatsCollector.PlatformType.Other); sendJsonResponse(rc, toJson(t)); @@ -1252,13 +1250,7 @@ private void handleIdentityMap(RoutingContext rc) { final InputUtil.InputVal input = this.getTokenInput(rc); try { - if (input == null) { - ResponseUtil.ClientError(rc, ERROR_INVALID_INPUT_EMAIL_MISSING); - } - else if (!input.isValid()) { - ResponseUtil.ClientError(rc, "Invalid email or email_hash"); - } - else { + if (isTokenInputValid(input, rc)) { final Instant now = Instant.now(); final MappedIdentity mappedIdentity = this.idService.map(input.toUserIdentity(this.identityScope, 0, now), now); rc.response().end(EncodingUtils.toBase64String(mappedIdentity.advertisingId)); From 7e5ad5dc5a50c7a9a01bf1b7d751ba8b0250455c Mon Sep 17 00:00:00 2001 From: Sunny Wu Date: Mon, 14 Oct 2024 13:44:57 +1100 Subject: [PATCH 050/431] 1. Refactor EncryptedTokenEncoder#encodeIntoIdentityResponse 2. Moved toJson methods into IdentityResponse from UIDOperatorVerticle class --- .../uid2/operator/model/IdentityResponse.java | 22 ++++++++++++ .../service/EncryptedTokenEncoder.java | 19 +++++++--- .../operator/vertx/UIDOperatorVerticle.java | 35 +++++-------------- 3 files changed, 45 insertions(+), 31 deletions(-) diff --git a/src/main/java/com/uid2/operator/model/IdentityResponse.java b/src/main/java/com/uid2/operator/model/IdentityResponse.java index 5c3b004ae..3b82d30e0 100644 --- a/src/main/java/com/uid2/operator/model/IdentityResponse.java +++ b/src/main/java/com/uid2/operator/model/IdentityResponse.java @@ -1,6 +1,7 @@ package com.uid2.operator.model; import com.uid2.shared.model.TokenVersion; +import io.vertx.core.json.JsonObject; import java.time.Instant; @@ -52,4 +53,25 @@ public Instant getRefreshFrom() { public boolean isNotValid() { return advertisingToken == null || advertisingToken.isEmpty(); } + + // for v1/v2 token/generate and token/refresh and client/generate (CSTG) endpoints + public JsonObject toJsonV1() { + final JsonObject json = new JsonObject(); + json.put("advertising_token", getAdvertisingToken()); + json.put("refresh_token", getRefreshToken()); + json.put("identity_expires", getIdentityExpires().toEpochMilli()); + json.put("refresh_expires", getRefreshExpires().toEpochMilli()); + json.put("refresh_from", getRefreshFrom().toEpochMilli()); + return json; + } + + // for the original/legacy token/generate and token/refresh endpoint + public JsonObject toJsonV0() { + final JsonObject json = new JsonObject(); + json.put("advertisement_token", getAdvertisingToken()); + json.put("advertising_token", getAdvertisingToken()); + json.put("refresh_token", getRefreshToken()); + + return json; + } } diff --git a/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java b/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java index ab70aeea3..1dc01777e 100644 --- a/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java +++ b/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java @@ -344,19 +344,30 @@ public static String bytesToBase64Token(byte[] advertisingTokenBytes, TokenVersi @Override public IdentityResponse encodeIntoIdentityResponse(AdvertisingTokenInput advertisingTokenInput, RefreshTokenInput refreshTokenInput, Instant refreshFrom, Instant asOf) { - final byte[] advertisingTokenBytes = encode(advertisingTokenInput, asOf); - final String base64AdvertisingToken = bytesToBase64Token(advertisingTokenBytes, advertisingTokenInput.version); - + final String base64AdvertisingToken = generateBase64AdvertisingToken(advertisingTokenInput, asOf); + final String base64RefreshToken = generateBase64RefreshToken(refreshTokenInput, asOf); return new IdentityResponse( base64AdvertisingToken, advertisingTokenInput.version, - EncodingUtils.toBase64String(encode(refreshTokenInput, asOf)), + base64RefreshToken, advertisingTokenInput.expiresAt, refreshTokenInput.expiresAt, refreshFrom ); } + private String generateBase64RefreshToken(RefreshTokenInput refreshTokenInput, Instant asOf) { + return EncodingUtils.toBase64String(encode(refreshTokenInput, asOf)); + } + + private String generateBase64AdvertisingToken(AdvertisingTokenInput advertisingTokenInput, Instant asOf) { + final byte[] advertisingTokenBytes = encode(advertisingTokenInput, asOf); + final String base64AdvertisingToken = bytesToBase64Token(advertisingTokenBytes, advertisingTokenInput.version); + return base64AdvertisingToken; + } + + + private byte[] encryptIdentityV2(SourcePublisher sourcePublisher, FirstLevelHashIdentity firstLevelHashIdentity, KeysetKey key) { return encryptIdentityV2(sourcePublisher, firstLevelHashIdentity.firstLevelHash, firstLevelHashIdentity.privacyBits, firstLevelHashIdentity.establishedAt, key); diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index 43585c664..65db2e4b8 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -479,7 +479,7 @@ else if(emailHash != null) { responseStatus = TokenResponseStatsCollector.ResponseStatus.OptOut; } else { //user not opted out and already generated valid identity token - response = ResponseUtil.SuccessV2(toJsonV1(identityResponse)); + response = ResponseUtil.SuccessV2(identityResponse.toJsonV1()); } //if returning an optout token or a successful identity token created originally if (responseStatus == TokenResponseStatsCollector.ResponseStatus.Success) { @@ -825,7 +825,7 @@ private void handleTokenRefreshV1(RoutingContext rc) { ResponseUtil.Error(ResponseStatus.UnknownError, 500, rc, "Unknown State"); } } else { - ResponseUtil.Success(rc, toJsonV1(r.getIdentityResponse())); + ResponseUtil.Success(rc, r.getIdentityResponse().toJsonV1()); this.recordRefreshDurationStats(siteId, getApiContact(rc), r.getDurationSinceLastRefresh(), rc.request().headers().contains(ORIGIN_HEADER)); } @@ -859,7 +859,7 @@ private void handleTokenRefreshV2(RoutingContext rc) { ResponseUtil.Error(ResponseStatus.UnknownError, 500, rc, "Unknown State"); } } else { - ResponseUtil.SuccessV2(rc, toJsonV1(r.getIdentityResponse())); + ResponseUtil.SuccessV2(rc, r.getIdentityResponse().toJsonV1()); this.recordRefreshDurationStats(siteId, getApiContact(rc), r.getDurationSinceLastRefresh(), rc.request().headers().contains(ORIGIN_HEADER)); } TokenResponseStatsCollector.recordRefresh(siteProvider, siteId, TokenResponseStatsCollector.Endpoint.RefreshV2, r, platformType); @@ -945,7 +945,7 @@ private void handleTokenGenerateV1(RoutingContext rc) { //Integer.parseInt(rc.queryParam("privacy_bits").get(0)))); - ResponseUtil.Success(rc, toJsonV1(t)); + ResponseUtil.Success(rc, t.toJsonV1()); recordTokenResponseStats(siteId, TokenResponseStatsCollector.Endpoint.GenerateV1, TokenResponseStatsCollector.ResponseStatus.Success, siteProvider, t.getAdvertisingTokenVersion(), platformType); } } catch (Exception e) { @@ -1015,14 +1015,14 @@ private void handleTokenGenerateV2(RoutingContext rc) { optOutTokenInput.toHashedDiiIdentity(this.identityScope, pb.getAsInt(), Instant.now()), OptoutCheckPolicy.DoNotRespect)); - ResponseUtil.SuccessV2(rc, toJsonV1(optOutTokens)); + ResponseUtil.SuccessV2(rc, optOutTokens.toJsonV1()); recordTokenResponseStats(siteId, TokenResponseStatsCollector.Endpoint.GenerateV2, TokenResponseStatsCollector.ResponseStatus.Success, siteProvider, optOutTokens.getAdvertisingTokenVersion(), platformType); } else { // new participant, or legacy specified policy/optout_check=1 ResponseUtil.SuccessNoBodyV2("optout", rc); recordTokenResponseStats(siteId, TokenResponseStatsCollector.Endpoint.GenerateV2, TokenResponseStatsCollector.ResponseStatus.OptOut, siteProvider, null, platformType); } } else { - ResponseUtil.SuccessV2(rc, toJsonV1(t)); + ResponseUtil.SuccessV2(rc, t.toJsonV1()); recordTokenResponseStats(siteId, TokenResponseStatsCollector.Endpoint.GenerateV2, TokenResponseStatsCollector.ResponseStatus.Success, siteProvider, t.getAdvertisingTokenVersion(), platformType); } } @@ -1058,7 +1058,7 @@ else if (!input.isValid()) { //Integer.parseInt(rc.queryParam("privacy_bits").get(0)))); recordTokenResponseStats(siteId, TokenResponseStatsCollector.Endpoint.GenerateV0, TokenResponseStatsCollector.ResponseStatus.Success, siteProvider, t.getAdvertisingTokenVersion(), TokenResponseStatsCollector.PlatformType.Other); - sendJsonResponse(rc, toJson(t)); + sendJsonResponse(rc, t.toJsonV0()); } catch (Exception e) { SendServerErrorResponseAndRecordStats(rc, "Unknown error while generating token", siteId, TokenResponseStatsCollector.Endpoint.GenerateV0, TokenResponseStatsCollector.ResponseStatus.Unknown, siteProvider, e, TokenResponseStatsCollector.PlatformType.Other); @@ -1076,7 +1076,7 @@ private void handleTokenRefresh(RoutingContext rc) { try { final RefreshResponse r = this.refreshIdentity(rc, tokenList.get(0)); - sendJsonResponse(rc, toJson(r.getIdentityResponse())); + sendJsonResponse(rc, r.getIdentityResponse().toJsonV0()); siteId = rc.get(Const.RoutingContextData.SiteId); if (r.isRefreshed()) { @@ -1986,16 +1986,6 @@ private TransparentConsentParseResult getUserConsentV2(JsonObject req) { } } - private JsonObject toJsonV1(IdentityResponse t) { - final JsonObject json = new JsonObject(); - json.put("advertising_token", t.getAdvertisingToken()); - json.put("refresh_token", t.getRefreshToken()); - json.put("identity_expires", t.getIdentityExpires().toEpochMilli()); - json.put("refresh_expires", t.getRefreshExpires().toEpochMilli()); - json.put("refresh_from", t.getRefreshFrom().toEpochMilli()); - return json; - } - private static MissingAclMode getMissingAclMode(ClientKey clientKey) { return clientKey.hasRole(Role.ID_READER) ? MissingAclMode.ALLOW_ALL : MissingAclMode.DENY_ALL; } @@ -2040,15 +2030,6 @@ private static JsonObject toJson(KeysetKey key) { return json; } - private JsonObject toJson(IdentityResponse t) { - final JsonObject json = new JsonObject(); - json.put("advertisement_token", t.getAdvertisingToken()); - json.put("advertising_token", t.getAdvertisingToken()); - json.put("refresh_token", t.getRefreshToken()); - - return json; - } - private void sendJsonResponse(RoutingContext rc, JsonObject json) { rc.response().putHeader(HttpHeaders.CONTENT_TYPE, "application/json") .end(json.encode()); From 131d203972adf6948380fb1c559de812eb5bc028 Mon Sep 17 00:00:00 2001 From: Sunny Wu Date: Mon, 14 Oct 2024 13:59:47 +1100 Subject: [PATCH 051/431] Use EncryptedTokenEncoder directly inside UIDOperatorService class instead of ITokenEncoder interfact so it's easier to jump to references and we haven't really created any other ITokenEncoder variants --- .../java/com/uid2/operator/service/UIDOperatorService.java | 4 ++-- .../java/com/uid2/operator/vertx/UIDOperatorVerticle.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/uid2/operator/service/UIDOperatorService.java b/src/main/java/com/uid2/operator/service/UIDOperatorService.java index 7b5c88e8e..b678d7f67 100644 --- a/src/main/java/com/uid2/operator/service/UIDOperatorService.java +++ b/src/main/java/com/uid2/operator/service/UIDOperatorService.java @@ -36,7 +36,7 @@ public class UIDOperatorService implements IUIDOperatorService { private static final Instant RefreshCutoff = LocalDateTime.parse("2021-03-08T17:00:00", DateTimeFormatter.ISO_LOCAL_DATE_TIME).toInstant(ZoneOffset.UTC); private final ISaltProvider saltProvider; private final IOptOutStore optOutStore; - private final ITokenEncoder encoder; + private final EncryptedTokenEncoder encoder; private final Clock clock; private final IdentityScope identityScope; private final FirstLevelHashIdentity testOptOutIdentityForEmail; @@ -58,7 +58,7 @@ public class UIDOperatorService implements IUIDOperatorService { private final Handler saltRetrievalResponseHandler; - public UIDOperatorService(JsonObject config, IOptOutStore optOutStore, ISaltProvider saltProvider, ITokenEncoder encoder, Clock clock, + public UIDOperatorService(JsonObject config, IOptOutStore optOutStore, ISaltProvider saltProvider, EncryptedTokenEncoder encoder, Clock clock, IdentityScope identityScope, Handler saltRetrievalResponseHandler) { this.saltProvider = saltProvider; this.encoder = encoder; diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index 65db2e4b8..32fe035f9 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -91,7 +91,7 @@ public class UIDOperatorVerticle extends AbstractVerticle { private final AuthMiddleware auth; private final ISiteStore siteProvider; private final IClientSideKeypairStore clientSideKeypairProvider; - private final ITokenEncoder encoder; + private final EncryptedTokenEncoder encoder; private final ISaltProvider saltProvider; private final IOptOutStore optOutStore; private final IClientKeyProvider clientKeyProvider; From b2fc597e6681dfd725b21002f32b8af1724a057a Mon Sep 17 00:00:00 2001 From: Thomas Manson Date: Mon, 14 Oct 2024 16:41:40 +1100 Subject: [PATCH 052/431] Update version of shared --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 8db01ac81..4a2390c5a 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ 2.1.0 2.1.0 2.1.0 - 7.19.1-SNAPSHOT + 7.19.2-alpha-151-SNAPSHOT ${project.version} 21 21 From 9a0d39e5004ded050f7111ab90ce31b386fb0e4b Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Mon, 14 Oct 2024 05:49:54 +0000 Subject: [PATCH 053/431] [CI Pipeline] Released Snapshot version: 5.40.87-alpha-108-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 4a2390c5a..4ecccdf61 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.40.86 + 5.40.87-alpha-108-SNAPSHOT UTF-8 From 0e621d3e88ab8fb5540e1315f6629cce2b5029e6 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Mon, 14 Oct 2024 23:31:56 +0000 Subject: [PATCH 054/431] [CI Pipeline] Released Snapshot version: 5.40.88-alpha-109-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 4ecccdf61..1bb14f203 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.40.87-alpha-108-SNAPSHOT + 5.40.88-alpha-109-SNAPSHOT UTF-8 From b507d8e8ae55c493c8ddf5391428fe66eb1d192d Mon Sep 17 00:00:00 2001 From: Sunny Wu Date: Tue, 15 Oct 2024 16:57:13 +1100 Subject: [PATCH 055/431] Fixed UserIdentity member variables as final and added constructor --- .../userIdentity/FirstLevelHashIdentity.java | 6 +----- .../model/userIdentity/HashedDiiIdentity.java | 6 +----- .../model/userIdentity/RawUidIdentity.java | 6 +----- .../model/userIdentity/UserIdentity.java | 21 +++++++++++-------- .../uid2/operator/UIDOperatorServiceTest.java | 6 +++--- 5 files changed, 18 insertions(+), 27 deletions(-) diff --git a/src/main/java/com/uid2/operator/model/userIdentity/FirstLevelHashIdentity.java b/src/main/java/com/uid2/operator/model/userIdentity/FirstLevelHashIdentity.java index 4286d2a0c..cac4f970e 100644 --- a/src/main/java/com/uid2/operator/model/userIdentity/FirstLevelHashIdentity.java +++ b/src/main/java/com/uid2/operator/model/userIdentity/FirstLevelHashIdentity.java @@ -12,12 +12,8 @@ public class FirstLevelHashIdentity extends UserIdentity { public FirstLevelHashIdentity(IdentityScope identityScope, IdentityType identityType, byte[] firstLevelHash, int privacyBits, Instant establishedAt, Instant refreshedAt) { - this.identityScope = identityScope; - this.identityType = identityType; + super(identityScope, identityType, privacyBits, establishedAt, refreshedAt); this.firstLevelHash = firstLevelHash; - this.privacyBits = privacyBits; - this.establishedAt = establishedAt; - this.refreshedAt = refreshedAt; } public boolean matches(FirstLevelHashIdentity that) { diff --git a/src/main/java/com/uid2/operator/model/userIdentity/HashedDiiIdentity.java b/src/main/java/com/uid2/operator/model/userIdentity/HashedDiiIdentity.java index c5b162315..25baaf308 100644 --- a/src/main/java/com/uid2/operator/model/userIdentity/HashedDiiIdentity.java +++ b/src/main/java/com/uid2/operator/model/userIdentity/HashedDiiIdentity.java @@ -12,11 +12,7 @@ public class HashedDiiIdentity extends UserIdentity{ public HashedDiiIdentity(IdentityScope identityScope, IdentityType identityType, byte[] hashedDii, int privacyBits, Instant establishedAt, Instant refreshedAt) { - this.identityScope = identityScope; - this.identityType = identityType; + super(identityScope, identityType, privacyBits, establishedAt, refreshedAt); this.hashedDii = hashedDii; - this.privacyBits = privacyBits; - this.establishedAt = establishedAt; - this.refreshedAt = refreshedAt; } } diff --git a/src/main/java/com/uid2/operator/model/userIdentity/RawUidIdentity.java b/src/main/java/com/uid2/operator/model/userIdentity/RawUidIdentity.java index 7af6edf5b..4e15c6ff0 100644 --- a/src/main/java/com/uid2/operator/model/userIdentity/RawUidIdentity.java +++ b/src/main/java/com/uid2/operator/model/userIdentity/RawUidIdentity.java @@ -12,12 +12,8 @@ public class RawUidIdentity extends UserIdentity { public RawUidIdentity(IdentityScope identityScope, IdentityType identityType, byte[] rawUid, int privacyBits, Instant establishedAt, Instant refreshedAt) { - this.identityScope = identityScope; - this.identityType = identityType; + super(identityScope, identityType, privacyBits, establishedAt, refreshedAt); this.rawUid = rawUid; - this.privacyBits = privacyBits; - this.establishedAt = establishedAt; - this.refreshedAt = refreshedAt; } public boolean matches(RawUidIdentity that) { diff --git a/src/main/java/com/uid2/operator/model/userIdentity/UserIdentity.java b/src/main/java/com/uid2/operator/model/userIdentity/UserIdentity.java index 0e16cc4ed..362b89fbe 100644 --- a/src/main/java/com/uid2/operator/model/userIdentity/UserIdentity.java +++ b/src/main/java/com/uid2/operator/model/userIdentity/UserIdentity.java @@ -8,14 +8,17 @@ //base class for all other HshedDii/FirstLevelHash/RawUIDIdentity class and define the basic common fields public class UserIdentity { - public IdentityScope identityScope; - public IdentityType identityType; - public int privacyBits; - public Instant establishedAt; - public Instant refreshedAt; + public final IdentityScope identityScope; + public final IdentityType identityType; + public final int privacyBits; + public final Instant establishedAt; + public final Instant refreshedAt; - public IdentityScope GetIdentityScope() { return identityScope; } - public IdentityType GetIdentityType() { return identityType; } - public Instant GetEstablishedAt() { return establishedAt; }; - public Instant GetIRefreshedAt() { return refreshedAt; } + public UserIdentity(IdentityScope identityScope, IdentityType identityType, int privacyBits, Instant establishedAt, Instant refreshedAt) { + this.identityScope = identityScope; + this.identityType = identityType; + this.privacyBits = privacyBits; + this.establishedAt = establishedAt; + this.refreshedAt = refreshedAt; + } } diff --git a/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java b/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java index 7b01b7180..6b2459eca 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java @@ -142,9 +142,9 @@ private AdvertisingTokenInput validateAndGetToken(EncryptedTokenEncoder tokenEnc private void assertIdentityScopeIdentityTypeAndEstablishedAt(UserIdentity expctedValues, UserIdentity actualValues) { - assertEquals(expctedValues.GetIdentityScope(), actualValues.GetIdentityScope()); - assertEquals(expctedValues.GetIdentityType(), actualValues.GetIdentityType()); - assertEquals(expctedValues.GetEstablishedAt(), actualValues.GetEstablishedAt()); + assertEquals(expctedValues.identityScope, actualValues.identityScope); + assertEquals(expctedValues.identityType, actualValues.identityType); + assertEquals(expctedValues.establishedAt, actualValues.establishedAt); } @ParameterizedTest From f5877b76a9c651b37a680cd3db31d7f9270c7ee9 Mon Sep 17 00:00:00 2001 From: Sunny Wu Date: Tue, 15 Oct 2024 17:45:46 +1100 Subject: [PATCH 056/431] 1. Addressed code review feedbacks 2. Renamed RawUidResult to RawUidResponse 3. rename some id variable to rawUid to make it clearer 4. more comments --- .../java/com/uid2/operator/IdentityConst.java | 7 +- .../uid2/operator/model/IdentityResponse.java | 4 +- ...{RawUidResult.java => RawUidResponse.java} | 7 +- .../userIdentity/FirstLevelHashIdentity.java | 2 +- .../model/userIdentity/HashedDiiIdentity.java | 6 +- .../service/EncryptedTokenEncoder.java | 15 ++--- .../operator/service/IUIDOperatorService.java | 4 +- .../operator/service/UIDOperatorService.java | 22 +++---- .../operator/store/CloudSyncOptOutStore.java | 2 +- .../com/uid2/operator/store/IOptOutStore.java | 5 +- .../operator/vertx/UIDOperatorVerticle.java | 22 +++---- .../uid2/operator/UIDOperatorServiceTest.java | 64 +++++++++---------- .../benchmark/IdentityMapBenchmark.java | 4 +- 13 files changed, 83 insertions(+), 81 deletions(-) rename src/main/java/com/uid2/operator/model/{RawUidResult.java => RawUidResponse.java} (66%) diff --git a/src/main/java/com/uid2/operator/IdentityConst.java b/src/main/java/com/uid2/operator/IdentityConst.java index 0263d4e76..9362ade6e 100644 --- a/src/main/java/com/uid2/operator/IdentityConst.java +++ b/src/main/java/com/uid2/operator/IdentityConst.java @@ -3,9 +3,11 @@ import com.uid2.operator.service.EncodingUtils; public class IdentityConst { + // DIIs for generating optout tokens for legacy participants - to be deprecated public static final String OptOutTokenIdentityForEmail = "optout@unifiedid.com"; public static final String OptOutTokenIdentityForPhone = "+00000000001"; + // DIIs for for testing with token/validate endpoint, see https://unifiedid.com/docs/endpoints/post-token-validate public static final String ValidateIdentityForEmail = "validate@example.com"; public static final String ValidateIdentityForPhone = "+12345678901"; public static final byte[] ValidateIdentityForEmailHash = EncodingUtils.getSha256Bytes(IdentityConst.ValidateIdentityForEmail); @@ -15,9 +17,8 @@ public class IdentityConst { public static final String OptOutIdentityForEmail = "optout@example.com"; public static final String OptOutIdentityForPhone = "+00000000000"; + // DIIs to use when you want to generate a UID token but when doing refresh token, you want to always get an optout response + // to test the optout handling workflow public static final String RefreshOptOutIdentityForEmail = "refresh-optout@example.com"; public static final String RefreshOptOutIdentityForPhone = "+00000000002"; - - - } diff --git a/src/main/java/com/uid2/operator/model/IdentityResponse.java b/src/main/java/com/uid2/operator/model/IdentityResponse.java index 78dee24ed..8b3ad4a43 100644 --- a/src/main/java/com/uid2/operator/model/IdentityResponse.java +++ b/src/main/java/com/uid2/operator/model/IdentityResponse.java @@ -6,7 +6,7 @@ import java.time.Instant; // this defines all the fields for the response of the /token/generate and /client/generate endpoints before they are -// json-ised +// jsonified public class IdentityResponse { public static IdentityResponse OptOutIdentityResponse = new IdentityResponse("", null, "", Instant.EPOCH, Instant.EPOCH, Instant.EPOCH); private final String advertisingToken; @@ -50,7 +50,7 @@ public Instant getRefreshFrom() { return refreshFrom; } - public boolean isOptedout() { + public boolean isOptedOut() { return advertisingToken == null || advertisingToken.isEmpty(); } diff --git a/src/main/java/com/uid2/operator/model/RawUidResult.java b/src/main/java/com/uid2/operator/model/RawUidResponse.java similarity index 66% rename from src/main/java/com/uid2/operator/model/RawUidResult.java rename to src/main/java/com/uid2/operator/model/RawUidResponse.java index 5348700e0..249bef4c5 100644 --- a/src/main/java/com/uid2/operator/model/RawUidResult.java +++ b/src/main/java/com/uid2/operator/model/RawUidResponse.java @@ -1,17 +1,18 @@ package com.uid2.operator.model; // Contains the computed raw UID and its bucket ID from identity/map request -public class RawUidResult { - public static RawUidResult OptoutIdentity = new RawUidResult(new byte[33], ""); +public class RawUidResponse { + public static RawUidResponse OptoutIdentity = new RawUidResponse(new byte[33], ""); // The raw UID is also known as Advertising Id (historically) public final byte[] rawUid; public final String bucketId; - public RawUidResult(byte[] rawUid, String bucketId) { + public RawUidResponse(byte[] rawUid, String bucketId) { this.rawUid = rawUid; this.bucketId = bucketId; } + // historically Optout is known as Logout public boolean isOptedOut() { return this.equals(OptoutIdentity) || this.bucketId == null || this.bucketId.isEmpty(); } diff --git a/src/main/java/com/uid2/operator/model/userIdentity/FirstLevelHashIdentity.java b/src/main/java/com/uid2/operator/model/userIdentity/FirstLevelHashIdentity.java index cac4f970e..64b8bcedd 100644 --- a/src/main/java/com/uid2/operator/model/userIdentity/FirstLevelHashIdentity.java +++ b/src/main/java/com/uid2/operator/model/userIdentity/FirstLevelHashIdentity.java @@ -6,7 +6,7 @@ import java.time.Instant; import java.util.Arrays; -// Contains a first level salted computed from Hashed DII (email/phone number) and applying salt to it +// Contains a first level salted hash computed from Hashed DII (email/phone number) public class FirstLevelHashIdentity extends UserIdentity { public final byte[] firstLevelHash; diff --git a/src/main/java/com/uid2/operator/model/userIdentity/HashedDiiIdentity.java b/src/main/java/com/uid2/operator/model/userIdentity/HashedDiiIdentity.java index 25baaf308..dad862f21 100644 --- a/src/main/java/com/uid2/operator/model/userIdentity/HashedDiiIdentity.java +++ b/src/main/java/com/uid2/operator/model/userIdentity/HashedDiiIdentity.java @@ -5,9 +5,9 @@ import java.time.Instant; -// Contains a hash computed from a raw email/phone number DII input or the hash is provided by the UID Participant -// directly -public class HashedDiiIdentity extends UserIdentity{ +// Contains a hash DII, +// This hash can either be computed from a raw email/phone number DII input or provided by the UID Participant directly +public class HashedDiiIdentity extends UserIdentity { public final byte[] hashedDii; public HashedDiiIdentity(IdentityScope identityScope, IdentityType identityType, byte[] hashedDii, int privacyBits, diff --git a/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java b/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java index 89ac7cc1e..7accc0ac5 100644 --- a/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java +++ b/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java @@ -217,7 +217,7 @@ public AdvertisingTokenInput decodeAdvertisingTokenV2(Buffer b) { final int siteId = b3.getInt(0); final int length = b3.getInt(4); - final byte[] getRawUid = EncodingUtils.fromBase64(b3.slice(8, 8 + length).getBytes()); + final byte[] rawUid = EncodingUtils.fromBase64(b3.slice(8, 8 + length).getBytes()); final int privacyBits = b3.getInt(8 + length); final long establishedMillis = b3.getLong(8 + length + 4); @@ -228,7 +228,7 @@ public AdvertisingTokenInput decodeAdvertisingTokenV2(Buffer b) { Instant.ofEpochMilli(expiresMillis), new OperatorIdentity(0, OperatorType.Service, 0, masterKeyId), new SourcePublisher(siteId, siteKeyId, 0), - new RawUidIdentity(IdentityScope.UID2, IdentityType.Email, getRawUid, privacyBits, + new RawUidIdentity(IdentityScope.UID2, IdentityType.Email, rawUid, privacyBits, Instant.ofEpochMilli(establishedMillis), null) ); @@ -253,11 +253,11 @@ public AdvertisingTokenInput decodeAdvertisingTokenV3orV4(Buffer b, byte[] bytes final int privacyBits = sitePayload.getInt(16); final Instant establishedAt = Instant.ofEpochMilli(sitePayload.getLong(20)); final Instant refreshedAt = Instant.ofEpochMilli(sitePayload.getLong(28)); - final byte[] id = sitePayload.slice(36, sitePayload.length()).getBytes(); - final IdentityScope identityScope = id.length == 32 ? IdentityScope.UID2 : decodeIdentityScopeV3(id[0]); - final IdentityType identityType = id.length == 32 ? IdentityType.Email : decodeIdentityTypeV3(id[0]); + final byte[] rawUid = sitePayload.slice(36, sitePayload.length()).getBytes(); + final IdentityScope identityScope = rawUid.length == 32 ? IdentityScope.UID2 : decodeIdentityScopeV3(rawUid[0]); + final IdentityType identityType = rawUid.length == 32 ? IdentityType.Email : decodeIdentityTypeV3(rawUid[0]); - if (id.length > 32) + if (rawUid.length > 32) { if (identityScope != decodeIdentityScopeV3(b.getByte(0))) { throw new ClientInputValidationException("Failed decoding advertisingTokenV3: Identity scope mismatch"); @@ -269,7 +269,7 @@ public AdvertisingTokenInput decodeAdvertisingTokenV3orV4(Buffer b, byte[] bytes return new AdvertisingTokenInput( tokenVersion, createdAt, expiresAt, operatorIdentity, sourcePublisher, - new RawUidIdentity(identityScope, identityType, id, privacyBits, establishedAt, refreshedAt) + new RawUidIdentity(identityScope, identityType, rawUid, privacyBits, establishedAt, refreshedAt) ); } @@ -371,7 +371,6 @@ private String generateBase64AdvertisingToken(AdvertisingTokenInput advertisingT private byte[] encryptIdentityV2(SourcePublisher sourcePublisher, FirstLevelHashIdentity firstLevelHashIdentity, KeysetKey key) { return encryptIdentityV2(sourcePublisher, firstLevelHashIdentity.firstLevelHash, firstLevelHashIdentity.privacyBits, firstLevelHashIdentity.establishedAt, key); - } private byte[] encryptIdentityV2(SourcePublisher sourcePublisher, RawUidIdentity rawUidIdentity, diff --git a/src/main/java/com/uid2/operator/service/IUIDOperatorService.java b/src/main/java/com/uid2/operator/service/IUIDOperatorService.java index c2bf091d2..38624848d 100644 --- a/src/main/java/com/uid2/operator/service/IUIDOperatorService.java +++ b/src/main/java/com/uid2/operator/service/IUIDOperatorService.java @@ -16,10 +16,10 @@ public interface IUIDOperatorService { RefreshResponse refreshIdentity(RefreshTokenInput refreshTokenInput); - RawUidResult mapIdentity(MapRequest request); + RawUidResponse mapIdentity(MapRequest request); @Deprecated - RawUidResult map(HashedDiiIdentity hashedDiiIdentity, Instant asOf); + RawUidResponse map(HashedDiiIdentity hashedDiiIdentity, Instant asOf); List getModifiedBuckets(Instant sinceTimestamp); diff --git a/src/main/java/com/uid2/operator/service/UIDOperatorService.java b/src/main/java/com/uid2/operator/service/UIDOperatorService.java index c11b36710..4c397885c 100644 --- a/src/main/java/com/uid2/operator/service/UIDOperatorService.java +++ b/src/main/java/com/uid2/operator/service/UIDOperatorService.java @@ -160,18 +160,18 @@ public RefreshResponse refreshIdentity(RefreshTokenInput token) { } @Override - public RawUidResult mapIdentity(MapRequest request) { + public RawUidResponse mapIdentity(MapRequest request) { final FirstLevelHashIdentity firstLevelHashIdentity = getFirstLevelHashIdentity(request.hashedDiiIdentity, request.asOf); if (request.shouldCheckOptOut() && getGlobalOptOutResult(firstLevelHashIdentity, false).isOptedOut()) { - return RawUidResult.OptoutIdentity; + return RawUidResponse.OptoutIdentity; } else { return generateRawUid(firstLevelHashIdentity, request.asOf); } } @Override - public RawUidResult map(HashedDiiIdentity diiIdentity, Instant asOf) { + public RawUidResponse map(HashedDiiIdentity diiIdentity, Instant asOf) { final FirstLevelHashIdentity firstLevelHashIdentity = getFirstLevelHashIdentity(diiIdentity, asOf); return generateRawUid(firstLevelHashIdentity, asOf); } @@ -194,9 +194,9 @@ private ISaltProvider.ISaltSnapshot getSaltProviderSnapshot(Instant asOf) { @Override public void invalidateTokensAsync(HashedDiiIdentity diiIdentity, Instant asOf, Handler> handler) { final FirstLevelHashIdentity hashedDiiIdentity = getFirstLevelHashIdentity(diiIdentity, asOf); - final RawUidResult rawUidResult = generateRawUid(hashedDiiIdentity, asOf); + final RawUidResponse rawUidResponse = generateRawUid(hashedDiiIdentity, asOf); - this.optOutStore.addEntry(hashedDiiIdentity, rawUidResult.rawUid, r -> { + this.optOutStore.addEntry(hashedDiiIdentity, rawUidResponse.rawUid, r -> { if (r.succeeded()) { handler.handle(Future.succeededFuture(r.result())); } else { @@ -208,10 +208,10 @@ public void invalidateTokensAsync(HashedDiiIdentity diiIdentity, Instant asOf, H @Override public boolean advertisingTokenMatches(String advertisingToken, HashedDiiIdentity diiIdentity, Instant asOf) { final FirstLevelHashIdentity firstLevelHashIdentity = getFirstLevelHashIdentity(diiIdentity, asOf); - final RawUidResult rawUidResult = generateRawUid(firstLevelHashIdentity, asOf); + final RawUidResponse rawUidResponse = generateRawUid(firstLevelHashIdentity, asOf); final AdvertisingTokenInput token = this.encoder.decodeAdvertisingToken(advertisingToken); - return Arrays.equals(rawUidResult.rawUid, token.rawUidIdentity.rawUid); + return Arrays.equals(rawUidResponse.rawUid, token.rawUidIdentity.rawUid); } @Override @@ -238,10 +238,10 @@ private byte[] getFirstLevelHash(byte[] identityHash, Instant asOf) { return TokenUtils.getFirstLevelHash(identityHash, getSaltProviderSnapshot(asOf).getFirstLevelSalt()); } - private RawUidResult generateRawUid(FirstLevelHashIdentity firstLevelHashIdentity, Instant asOf) { + private RawUidResponse generateRawUid(FirstLevelHashIdentity firstLevelHashIdentity, Instant asOf) { final SaltEntry rotatingSalt = getSaltProviderSnapshot(asOf).getRotatingSalt(firstLevelHashIdentity.firstLevelHash); - return new RawUidResult( + return new RawUidResponse( this.identityV3Enabled ? TokenUtils.getRawUidV3(firstLevelHashIdentity.identityScope, firstLevelHashIdentity.identityType, firstLevelHashIdentity.firstLevelHash, rotatingSalt.getSalt()) @@ -252,10 +252,10 @@ private RawUidResult generateRawUid(FirstLevelHashIdentity firstLevelHashIdentit private IdentityResponse generateIdentity(SourcePublisher sourcePublisher, FirstLevelHashIdentity firstLevelHashIdentity) { final Instant nowUtc = EncodingUtils.NowUTCMillis(this.clock); - final RawUidResult rawUidResult = generateRawUid(firstLevelHashIdentity, nowUtc); + final RawUidResponse rawUidResponse = generateRawUid(firstLevelHashIdentity, nowUtc); final RawUidIdentity rawUidIdentity = new RawUidIdentity(firstLevelHashIdentity.identityScope, firstLevelHashIdentity.identityType, - rawUidResult.rawUid, firstLevelHashIdentity.privacyBits, firstLevelHashIdentity.establishedAt, nowUtc); + rawUidResponse.rawUid, firstLevelHashIdentity.privacyBits, firstLevelHashIdentity.establishedAt, nowUtc); return this.encoder.encodeIntoIdentityResponse( this.createAdvertisingTokenInput(sourcePublisher, rawUidIdentity, nowUtc), diff --git a/src/main/java/com/uid2/operator/store/CloudSyncOptOutStore.java b/src/main/java/com/uid2/operator/store/CloudSyncOptOutStore.java index 1808175a0..e33106949 100644 --- a/src/main/java/com/uid2/operator/store/CloudSyncOptOutStore.java +++ b/src/main/java/com/uid2/operator/store/CloudSyncOptOutStore.java @@ -94,7 +94,7 @@ public void addEntry(FirstLevelHashIdentity firstLevelHashIdentity, byte[] adver this.webClient.get(remoteApiPort, remoteApiHost, remoteApiPath). addQueryParam("identity_hash", EncodingUtils.toBase64String(firstLevelHashIdentity.firstLevelHash)) - .addQueryParam("advertising_id", EncodingUtils.toBase64String(advertisingId)) + .addQueryParam("advertising_id", EncodingUtils.toBase64String(advertisingId)) // advertising id aka raw UID .putHeader("Authorization", remoteApiBearerToken) .as(BodyCodec.string()) .send(ar -> { diff --git a/src/main/java/com/uid2/operator/store/IOptOutStore.java b/src/main/java/com/uid2/operator/store/IOptOutStore.java index f767b316e..995939c70 100644 --- a/src/main/java/com/uid2/operator/store/IOptOutStore.java +++ b/src/main/java/com/uid2/operator/store/IOptOutStore.java @@ -9,9 +9,10 @@ public interface IOptOutStore { /** - * Get latest Opt-out record with respect to the UID (hashed identity) + * Get latest opt-out record * - * @param firstLevelHashIdentity@return The timestamp of latest opt-out record. NULL if no record. + * @param firstLevelHashIdentity The first level hash of a DII Hash + * @return The timestamp of latest opt-out record. NULL if no record. */ Instant getLatestEntry(FirstLevelHashIdentity firstLevelHashIdentity); diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index 404cd92de..58c6703a1 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -474,7 +474,7 @@ else if(emailHash != null) { JsonObject response; TokenResponseStatsCollector.ResponseStatus responseStatus = TokenResponseStatsCollector.ResponseStatus.Success; - if (identityResponse.isOptedout()) { + if (identityResponse.isOptedOut()) { response = ResponseUtil.SuccessNoBodyV2(ResponseStatus.OptOut); responseStatus = TokenResponseStatsCollector.ResponseStatus.OptOut; } @@ -999,7 +999,7 @@ private void handleTokenGenerateV2(RoutingContext rc) { input.toHashedDiiIdentity(this.identityScope, 1, Instant.now()), OptoutCheckPolicy.respectOptOut())); - if (t.isOptedout()) { + if (t.isOptedOut()) { if (optoutCheckPolicy.getItem1() == OptoutCheckPolicy.DoNotRespect) { // only legacy can use this policy final InputUtil.InputVal optOutTokenInput = input.getIdentityType() == IdentityType.Email ? InputUtil.InputVal.validEmail(OptOutTokenIdentityForEmail, OptOutTokenIdentityForEmail) @@ -1241,11 +1241,11 @@ private void handleIdentityMapV1(RoutingContext rc) { } try { final Instant now = Instant.now(); - final RawUidResult rawUidResult = this.idService.map(input.toHashedDiiIdentity(this.identityScope, 0, now), now); + final RawUidResponse rawUidResponse = this.idService.map(input.toHashedDiiIdentity(this.identityScope, 0, now), now); final JsonObject jsonObject = new JsonObject(); jsonObject.put("identifier", input.getProvided()); - jsonObject.put("advertising_id", EncodingUtils.toBase64String(rawUidResult.rawUid)); - jsonObject.put("bucket_id", rawUidResult.bucketId); + jsonObject.put("advertising_id", EncodingUtils.toBase64String(rawUidResponse.rawUid)); + jsonObject.put("bucket_id", rawUidResponse.bucketId); ResponseUtil.Success(rc, jsonObject); } catch (Exception e) { ResponseUtil.Error(ResponseStatus.UnknownError, 500, rc, "Unknown State", e); @@ -1264,8 +1264,8 @@ else if (!input.isValid()) { } else { final Instant now = Instant.now(); - final RawUidResult rawUidResult = this.idService.map(input.toHashedDiiIdentity(this.identityScope, 0, now), now); - rc.response().end(EncodingUtils.toBase64String(rawUidResult.rawUid)); + final RawUidResponse rawUidResponse = this.idService.map(input.toHashedDiiIdentity(this.identityScope, 0, now), now); + rc.response().end(EncodingUtils.toBase64String(rawUidResponse.rawUid)); } } catch (Exception ex) { LOGGER.error("Unexpected error while mapping identity", ex); @@ -1460,13 +1460,13 @@ private JsonObject handleIdentityMapCommon(RoutingContext rc, InputUtil.InputVal for (int i = 0; i < count; ++i) { final InputUtil.InputVal input = inputList[i]; if (input != null && input.isValid()) { - final RawUidResult rawUidResult = idService.mapIdentity( + final RawUidResponse rawUidResponse = idService.mapIdentity( new MapRequest( input.toHashedDiiIdentity(this.identityScope, 0, now), OptoutCheckPolicy.respectOptOut(), now)); - if (rawUidResult.isOptedOut()) { + if (rawUidResponse.isOptedOut()) { final JsonObject resp = new JsonObject(); resp.put("identifier", input.getProvided()); resp.put("reason", "optout"); @@ -1475,8 +1475,8 @@ private JsonObject handleIdentityMapCommon(RoutingContext rc, InputUtil.InputVal } else { final JsonObject resp = new JsonObject(); resp.put("identifier", input.getProvided()); - resp.put("advertising_id", EncodingUtils.toBase64String(rawUidResult.rawUid)); - resp.put("bucket_id", rawUidResult.bucketId); + resp.put("advertising_id", EncodingUtils.toBase64String(rawUidResponse.rawUid)); + resp.put("bucket_id", rawUidResponse.bucketId); mapped.add(resp); } } else { diff --git a/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java b/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java index 6b2459eca..5196dfaf2 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java @@ -211,7 +211,7 @@ public void testTestOptOutKey_DoNotRespectOptout() { verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); assertNotNull(identityResponse); - assertFalse(identityResponse.isOptedout()); + assertFalse(identityResponse.isOptedOut()); final RefreshTokenInput refreshTokenInput = this.tokenEncoder.decodeRefreshToken(identityResponse.getRefreshToken()); assertEquals(RefreshResponse.Optout, uid2Service.refreshIdentity(refreshTokenInput)); @@ -227,7 +227,7 @@ public void testTestOptOutKey_RespectOptout() { OptoutCheckPolicy.RespectOptOut ); final IdentityResponse identityResponse = uid2Service.generateIdentity(identityRequest); - assertTrue(identityResponse.isOptedout()); + assertTrue(identityResponse.isOptedOut()); verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); } @@ -327,27 +327,27 @@ public void testIdentityMapForOptOutUser(IdentityType type, String identity, Ide when(optOutStore.getLatestEntry(any(FirstLevelHashIdentity.class))) .thenReturn(Instant.now().minus(1, ChronoUnit.HOURS)); - final RawUidResult rawUidResult; - final RawUidResult rawUidResultShouldBeOptOut; + final RawUidResponse rawUidResponse; + final RawUidResponse rawUidResponseShouldBeOptOut; if (scope == IdentityScope.UID2) { verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); - rawUidResult = uid2Service.mapIdentity(mapRequestForceMap); + rawUidResponse = uid2Service.mapIdentity(mapRequestForceMap); reset(shutdownHandler); - rawUidResultShouldBeOptOut = uid2Service.mapIdentity(mapRequestRespectOptOut); + rawUidResponseShouldBeOptOut = uid2Service.mapIdentity(mapRequestRespectOptOut); } else { verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); - rawUidResult = euidService.mapIdentity(mapRequestForceMap); + rawUidResponse = euidService.mapIdentity(mapRequestForceMap); reset(shutdownHandler); - rawUidResultShouldBeOptOut = euidService.mapIdentity(mapRequestRespectOptOut); + rawUidResponseShouldBeOptOut = euidService.mapIdentity(mapRequestRespectOptOut); } verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); - assertNotNull(rawUidResult); - assertFalse(rawUidResult.isOptedOut()); - assertNotNull(rawUidResultShouldBeOptOut); - assertTrue(rawUidResultShouldBeOptOut.isOptedOut()); + assertNotNull(rawUidResponse); + assertFalse(rawUidResponse.isOptedOut()); + assertNotNull(rawUidResponseShouldBeOptOut); + assertTrue(rawUidResponseShouldBeOptOut.isOptedOut()); } private enum TestIdentityInputType { @@ -434,17 +434,17 @@ void testSpecialIdentityOptOutIdentityMap(TestIdentityInputType type, String id, // identity has no optout record, ensure map still returns optout when(this.optOutStore.getLatestEntry(any())).thenReturn(null); - final RawUidResult rawUidResult; + final RawUidResponse rawUidResponse; if(scope == IdentityScope.EUID) { - rawUidResult = euidService.mapIdentity(mapRequestRespectOptOut); + rawUidResponse = euidService.mapIdentity(mapRequestRespectOptOut); } else { - rawUidResult = uid2Service.mapIdentity(mapRequestRespectOptOut); + rawUidResponse = uid2Service.mapIdentity(mapRequestRespectOptOut); } verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); - assertNotNull(rawUidResult); - assertTrue(rawUidResult.isOptedOut()); + assertNotNull(rawUidResponse); + assertTrue(rawUidResponse.isOptedOut()); } @ParameterizedTest @@ -548,17 +548,17 @@ void testSpecialIdentityRefreshOptOutIdentityMap(TestIdentityInputType type, Str // all identities have optout records, ensure refresh-optout identities still map when(this.optOutStore.getLatestEntry(any())).thenReturn(Instant.now()); - final RawUidResult rawUidResult; + final RawUidResponse rawUidResponse; if(scope == IdentityScope.EUID) { - rawUidResult = euidService.mapIdentity(mapRequestRespectOptOut); + rawUidResponse = euidService.mapIdentity(mapRequestRespectOptOut); } else { - rawUidResult = uid2Service.mapIdentity(mapRequestRespectOptOut); + rawUidResponse = uid2Service.mapIdentity(mapRequestRespectOptOut); } verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); - assertNotNull(rawUidResult); - assertFalse(rawUidResult.isOptedOut()); + assertNotNull(rawUidResponse); + assertFalse(rawUidResponse.isOptedOut()); } @ParameterizedTest @@ -619,17 +619,17 @@ void testSpecialIdentityValidateIdentityMap(TestIdentityInputType type, String i // all identities have optout records, ensure validate identities still get mapped when(this.optOutStore.getLatestEntry(any())).thenReturn(Instant.now()); - final RawUidResult rawUidResult; + final RawUidResponse rawUidResponse; if(scope == IdentityScope.EUID) { - rawUidResult = euidService.mapIdentity(mapRequestRespectOptOut); + rawUidResponse = euidService.mapIdentity(mapRequestRespectOptOut); } else { - rawUidResult = uid2Service.mapIdentity(mapRequestRespectOptOut); + rawUidResponse = uid2Service.mapIdentity(mapRequestRespectOptOut); } verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); - assertNotNull(rawUidResult); - assertFalse(rawUidResult.isOptedOut()); + assertNotNull(rawUidResponse); + assertFalse(rawUidResponse.isOptedOut()); } @ParameterizedTest @@ -737,18 +737,18 @@ void testExpiredSaltsNotifiesShutdownHandler(TestIdentityInputType type, String inputVal.toHashedDiiIdentity(scope, 0, this.now), OptoutCheckPolicy.RespectOptOut, now); - final RawUidResult rawUidResult; + final RawUidResponse rawUidResponse; reset(shutdownHandler); if(scope == IdentityScope.EUID) { - rawUidResult = euidService.mapIdentity(mapRequest); + rawUidResponse = euidService.mapIdentity(mapRequest); } else { - rawUidResult = uid2Service.mapIdentity(mapRequest); + rawUidResponse = uid2Service.mapIdentity(mapRequest); } verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(true); verify(shutdownHandler, never()).handleSaltRetrievalResponse(false); - assertNotNull(rawUidResult); - assertFalse(rawUidResult.isOptedOut()); + assertNotNull(rawUidResponse); + assertFalse(rawUidResponse.isOptedOut()); } } diff --git a/src/test/java/com/uid2/operator/benchmark/IdentityMapBenchmark.java b/src/test/java/com/uid2/operator/benchmark/IdentityMapBenchmark.java index acf5b8e2e..011b80a2e 100644 --- a/src/test/java/com/uid2/operator/benchmark/IdentityMapBenchmark.java +++ b/src/test/java/com/uid2/operator/benchmark/IdentityMapBenchmark.java @@ -25,13 +25,13 @@ public class IdentityMapBenchmark { @Benchmark @BenchmarkMode(Mode.Throughput) - public RawUidResult IdentityMapRawThroughput() { + public RawUidResponse IdentityMapRawThroughput() { return uidService.map(firstLevelHashIdentities[(idx++) & 65535], Instant.now()); } @Benchmark @BenchmarkMode(Mode.Throughput) - public RawUidResult IdentityMapWithOptOutThroughput() { + public RawUidResponse IdentityMapWithOptOutThroughput() { return uidService.mapIdentity(new MapRequest(firstLevelHashIdentities[(idx++) & 65535], OptoutCheckPolicy.RespectOptOut, Instant.now())); } } From ccc639f00502eff02183a0e4271ffca101a97bd4 Mon Sep 17 00:00:00 2001 From: Sunny Wu Date: Tue, 15 Oct 2024 18:00:33 +1100 Subject: [PATCH 057/431] Made UserIdentity abstract first --- .../java/com/uid2/operator/model/userIdentity/UserIdentity.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/uid2/operator/model/userIdentity/UserIdentity.java b/src/main/java/com/uid2/operator/model/userIdentity/UserIdentity.java index 362b89fbe..1391b7d75 100644 --- a/src/main/java/com/uid2/operator/model/userIdentity/UserIdentity.java +++ b/src/main/java/com/uid2/operator/model/userIdentity/UserIdentity.java @@ -6,7 +6,7 @@ import java.time.Instant; //base class for all other HshedDii/FirstLevelHash/RawUIDIdentity class and define the basic common fields -public class UserIdentity { +public abstract class UserIdentity { public final IdentityScope identityScope; public final IdentityType identityType; From dabff48084f78fc22c9f641ac696ab137ba81e6f Mon Sep 17 00:00:00 2001 From: Sunny Wu Date: Tue, 15 Oct 2024 18:16:01 +1100 Subject: [PATCH 058/431] Fixed a few id variable to rawUid/firstLevelHash to make it clear. Fixed the UserIdentity/FirstLevelHashIdentity class uses to HashedDiiIdentity instead in IdentityMapBenchmark and TokenEndecBenchmark --- .../uid2/operator/service/EncryptedTokenEncoder.java | 4 ++-- .../java/com/uid2/operator/service/TokenUtils.java | 8 ++++---- .../com/uid2/operator/benchmark/BenchmarkCommon.java | 8 ++++---- .../operator/benchmark/IdentityMapBenchmark.java | 8 ++++---- .../uid2/operator/benchmark/TokenEndecBenchmark.java | 12 ++++++------ 5 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java b/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java index 7accc0ac5..c5a405f69 100644 --- a/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java +++ b/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java @@ -149,7 +149,7 @@ private RefreshTokenInput decodeRefreshTokenV3(Buffer b, byte[] bytes) { final Instant establishedAt = Instant.ofEpochMilli(b2.getLong(49)); final IdentityScope identityScope = decodeIdentityScopeV3(b2.getByte(57)); final IdentityType identityType = decodeIdentityTypeV3(b2.getByte(57)); - final byte[] id = b2.getBytes(58, 90); + final byte[] firstLevelHash = b2.getBytes(58, 90); if (identityScope != decodeIdentityScopeV3(b.getByte(0))) { throw new ClientInputValidationException("Failed to decode refreshTokenV3: Identity scope mismatch"); @@ -160,7 +160,7 @@ private RefreshTokenInput decodeRefreshTokenV3(Buffer b, byte[] bytes) { return new RefreshTokenInput( TokenVersion.V3, createdAt, expiresAt, operatorIdentity, sourcePublisher, - new FirstLevelHashIdentity(identityScope, identityType, id, privacyBits, establishedAt, null)); + new FirstLevelHashIdentity(identityScope, identityType, firstLevelHash, privacyBits, establishedAt, null)); } @Override diff --git a/src/main/java/com/uid2/operator/service/TokenUtils.java b/src/main/java/com/uid2/operator/service/TokenUtils.java index b42ac3c5f..ef532e578 100644 --- a/src/main/java/com/uid2/operator/service/TokenUtils.java +++ b/src/main/java/com/uid2/operator/service/TokenUtils.java @@ -41,10 +41,10 @@ public static byte[] getRawUidV2FromIdentityHash(String identityString, String f public static byte[] getRawUidV3(IdentityScope scope, IdentityType type, byte[] firstLevelHash, String rotatingSalt) { final byte[] sha = EncodingUtils.getSha256Bytes(EncodingUtils.toBase64String(firstLevelHash), rotatingSalt); - final byte[] id = new byte[33]; - id[0] = (byte)(encodeIdentityScope(scope) | encodeIdentityType(type)); - System.arraycopy(sha, 0, id, 1, 32); - return id; + final byte[] rawUid = new byte[33]; + rawUid[0] = (byte)(encodeIdentityScope(scope) | encodeIdentityType(type)); + System.arraycopy(sha, 0, rawUid, 1, 32); + return rawUid; } public static byte[] getRawUidV3FromIdentity(IdentityScope scope, IdentityType type, String identityString, String firstLevelSalt, String rotatingSalt) { diff --git a/src/test/java/com/uid2/operator/benchmark/BenchmarkCommon.java b/src/test/java/com/uid2/operator/benchmark/BenchmarkCommon.java index 00c2e2cac..1c25faecd 100644 --- a/src/test/java/com/uid2/operator/benchmark/BenchmarkCommon.java +++ b/src/test/java/com/uid2/operator/benchmark/BenchmarkCommon.java @@ -150,12 +150,12 @@ static ICloudStorage make1mOptOutEntryStorage(String salt, List out_gene return storage; } - static HashedDiiIdentity[] createUserIdentities() { + static HashedDiiIdentity[] createHashedDiiIdentities() { HashedDiiIdentity[] arr = new HashedDiiIdentity[65536]; for (int i = 0; i < 65536; i++) { - final byte[] id = new byte[33]; - new Random().nextBytes(id); - arr[i] = new HashedDiiIdentity(IdentityScope.UID2, IdentityType.Email, id, 0, + final byte[] diiHash = new byte[33]; + new Random().nextBytes(diiHash); + arr[i] = new HashedDiiIdentity(IdentityScope.UID2, IdentityType.Email, diiHash, 0, Instant.now().minusSeconds(120), Instant.now().minusSeconds(60)); } return arr; diff --git a/src/test/java/com/uid2/operator/benchmark/IdentityMapBenchmark.java b/src/test/java/com/uid2/operator/benchmark/IdentityMapBenchmark.java index 011b80a2e..880875fc0 100644 --- a/src/test/java/com/uid2/operator/benchmark/IdentityMapBenchmark.java +++ b/src/test/java/com/uid2/operator/benchmark/IdentityMapBenchmark.java @@ -11,13 +11,13 @@ public class IdentityMapBenchmark { private static final IUIDOperatorService uidService; - private static final HashedDiiIdentity[] firstLevelHashIdentities; + private static final HashedDiiIdentity[] hashedDiiIdentities; private static int idx = 0; static { try { uidService = BenchmarkCommon.createUidOperatorService(); - firstLevelHashIdentities = BenchmarkCommon.createUserIdentities(); + hashedDiiIdentities = BenchmarkCommon.createHashedDiiIdentities(); } catch (Exception e) { throw new RuntimeException(e); } @@ -26,12 +26,12 @@ public class IdentityMapBenchmark { @Benchmark @BenchmarkMode(Mode.Throughput) public RawUidResponse IdentityMapRawThroughput() { - return uidService.map(firstLevelHashIdentities[(idx++) & 65535], Instant.now()); + return uidService.map(hashedDiiIdentities[(idx++) & 65535], Instant.now()); } @Benchmark @BenchmarkMode(Mode.Throughput) public RawUidResponse IdentityMapWithOptOutThroughput() { - return uidService.mapIdentity(new MapRequest(firstLevelHashIdentities[(idx++) & 65535], OptoutCheckPolicy.RespectOptOut, Instant.now())); + return uidService.mapIdentity(new MapRequest(hashedDiiIdentities[(idx++) & 65535], OptoutCheckPolicy.RespectOptOut, Instant.now())); } } diff --git a/src/test/java/com/uid2/operator/benchmark/TokenEndecBenchmark.java b/src/test/java/com/uid2/operator/benchmark/TokenEndecBenchmark.java index 0e5f62b89..e907f638c 100644 --- a/src/test/java/com/uid2/operator/benchmark/TokenEndecBenchmark.java +++ b/src/test/java/com/uid2/operator/benchmark/TokenEndecBenchmark.java @@ -14,7 +14,7 @@ public class TokenEndecBenchmark { private static final IUIDOperatorService uidService; - private static final HashedDiiIdentity[] userIdentities; + private static final HashedDiiIdentity[] hashedDiiIdentities; private static final SourcePublisher publisher; private static final EncryptedTokenEncoder encoder; private static final IdentityResponse[] generatedTokens; @@ -23,11 +23,11 @@ public class TokenEndecBenchmark { static { try { uidService = BenchmarkCommon.createUidOperatorService(); - userIdentities = BenchmarkCommon.createUserIdentities(); + hashedDiiIdentities = BenchmarkCommon.createHashedDiiIdentities(); publisher = BenchmarkCommon.createSourcePublisher(); encoder = BenchmarkCommon.createTokenEncoder(); generatedTokens = createAdvertisingTokens(); - if (generatedTokens.length < 65536 || userIdentities.length < 65536) { + if (generatedTokens.length < 65536 || hashedDiiIdentities.length < 65536) { throw new IllegalStateException("must create more than 65535 test candidates."); } } catch (Exception e) { @@ -37,11 +37,11 @@ public class TokenEndecBenchmark { static IdentityResponse[] createAdvertisingTokens() { List tokens = new ArrayList<>(); - for (int i = 0; i < userIdentities.length; i++) { + for (int i = 0; i < hashedDiiIdentities.length; i++) { tokens.add( uidService.generateIdentity(new IdentityRequest( publisher, - userIdentities[i], + hashedDiiIdentities[i], OptoutCheckPolicy.DoNotRespect))); } return tokens.toArray(new IdentityResponse[tokens.size()]); @@ -52,7 +52,7 @@ static IdentityResponse[] createAdvertisingTokens() { public IdentityResponse TokenGenerationBenchmark() { return uidService.generateIdentity(new IdentityRequest( publisher, - userIdentities[(idx++) & 65535], + hashedDiiIdentities[(idx++) & 65535], OptoutCheckPolicy.DoNotRespect)); } From 60abeedd04f142cfee9409485a71e3b6605aac08 Mon Sep 17 00:00:00 2001 From: asloob qureshi Date: Tue, 15 Oct 2024 12:42:58 -0700 Subject: [PATCH 059/431] Convert asserts to exceptions in OptOut code --- .../operator/store/CloudSyncOptOutStore.java | 40 ++++++++++++++----- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/uid2/operator/store/CloudSyncOptOutStore.java b/src/main/java/com/uid2/operator/store/CloudSyncOptOutStore.java index 5ca87cc37..2de86cc85 100644 --- a/src/main/java/com/uid2/operator/store/CloudSyncOptOutStore.java +++ b/src/main/java/com/uid2/operator/store/CloudSyncOptOutStore.java @@ -525,7 +525,11 @@ private IndexUpdateMessage getIndexUpdateMessage(Instant now, Collection ium.addDeltaFile(f); else if (OptOutUtils.isPartitionFile(f)) ium.addPartitionFile(f); - else assert false; + else { + final String errorMsg = "File to index " + f + " is not of type delta or partition"; + LOGGER.error(errorMsg); + throw new IllegalStateException(errorMsg); + } } Collection indexedNonSynthetic = indexedFiles.stream() @@ -538,7 +542,12 @@ else if (OptOutUtils.isPartitionFile(f)) Instant tsOld = OptOutUtils.lastPartitionTimestamp(indexedNonSynthetic); Instant tsNew = OptOutUtils.lastPartitionTimestamp(newNonSynthetic); - assert tsOld == Instant.EPOCH || tsNew == Instant.EPOCH || tsOld.isBefore(tsNew); + if (tsOld != Instant.EPOCH && tsNew != Instant.EPOCH && tsOld.isBefore(tsNew)) { + final String errorMsg = "Last partition timestamp of indexed files " + tsOld.getEpochSecond() + + " is after last partition of non-indexed files " + tsNew.getEpochSecond(); + LOGGER.error(errorMsg); + throw new IllegalStateException(errorMsg); + } // if there are new partitions in this update, let index delete some in-mem delta caches that is old if (tsNew != Instant.EPOCH) { tsNew = tsNew.minusSeconds(fileUtils.lookbackGracePeriod()); @@ -594,15 +603,20 @@ private OptOutStoreSnapshot updateIndexInternal(IndexUpdateContext iuc) { try { if (numPartitions == 0) { // if update doesn't have a new partition, simply update heap with new log data - assert iuc.getDeltasToRemove().size() == 0; + if (!iuc.getDeltasToRemove().isEmpty()) { + final String errorMsg = "Invalid number of Deltas to remove=" + iuc.getDeltasToRemove().size(); + LOGGER.error(errorMsg); + throw new IllegalStateException(errorMsg); + } return this.processDeltas(iuc); } else if (numPartitions > 1) { - // should not load more than 1 partition at a time, unless during service bootstrap - assert this.iteration == 0; + if (this.iteration != 0) { + final String errorMsg = "Should not load more than 1 partition at a time, unless during service bootstrap. Current iteration " + this.iteration; + LOGGER.error(errorMsg); + throw new IllegalStateException(errorMsg); + } return this.processPartitions(iuc); } else { - // array size cannot be a negative value - assert numPartitions == 1; return this.processPartitions(iuc); } } finally { @@ -628,7 +642,11 @@ private OptOutStoreSnapshot processDeltasImpl(IndexUpdateContext iuc) { // this is thread-safe, as heap is not being used // and bloomfilter can tolerate false positive for (byte[] data : loadedData) { - assert data.length != 0; + if (data.length == 0) { + final String errorMsg = "Loaded delta file has 0 size"; + LOGGER.error(errorMsg); + throw new IllegalStateException(errorMsg); + } OptOutCollection newLog = new OptOutCollection(data); this.heap.add(newLog); @@ -679,7 +697,11 @@ private OptOutStoreSnapshot processPartitionsImpl(IndexUpdateContext iuc) { } for (String key : sortedPartitionFiles) { byte[] data = iuc.loadedPartitions.get(key); - assert data.length != 0; + if (data.length == 0) { + final String errorMsg = "Loaded partition file has 0 size"; + LOGGER.error(errorMsg); + throw new IllegalStateException(errorMsg); + } newPartitions[snapIndex++] = new OptOutPartition(data); } From ce6bf6064f360626ed6b630ee3eda341149c6e5b Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Tue, 15 Oct 2024 22:11:26 +0000 Subject: [PATCH 060/431] [CI Pipeline] Released Snapshot version: 5.40.87-alpha-110-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 082c67876..65bac8849 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.40.86 + 5.40.87-alpha-110-SNAPSHOT UTF-8 From 19c6c7a4d4034829ceeb4ace3bcef2f16ed024ca Mon Sep 17 00:00:00 2001 From: asloob qureshi Date: Tue, 15 Oct 2024 17:31:05 -0700 Subject: [PATCH 061/431] Fix partition timestamp check --- src/main/java/com/uid2/operator/store/CloudSyncOptOutStore.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/uid2/operator/store/CloudSyncOptOutStore.java b/src/main/java/com/uid2/operator/store/CloudSyncOptOutStore.java index 2de86cc85..8cd54ad28 100644 --- a/src/main/java/com/uid2/operator/store/CloudSyncOptOutStore.java +++ b/src/main/java/com/uid2/operator/store/CloudSyncOptOutStore.java @@ -542,7 +542,7 @@ else if (OptOutUtils.isPartitionFile(f)) Instant tsOld = OptOutUtils.lastPartitionTimestamp(indexedNonSynthetic); Instant tsNew = OptOutUtils.lastPartitionTimestamp(newNonSynthetic); - if (tsOld != Instant.EPOCH && tsNew != Instant.EPOCH && tsOld.isBefore(tsNew)) { + if (tsOld != Instant.EPOCH && tsNew != Instant.EPOCH && !tsOld.isBefore(tsNew)) { final String errorMsg = "Last partition timestamp of indexed files " + tsOld.getEpochSecond() + " is after last partition of non-indexed files " + tsNew.getEpochSecond(); LOGGER.error(errorMsg); From 3b1718ca7c9ec7efb1cd08e1639d1c7f2f9fcfd6 Mon Sep 17 00:00:00 2001 From: Sunny Wu Date: Wed, 16 Oct 2024 17:04:07 +1100 Subject: [PATCH 062/431] 1. Removed the UserIdentity's refreshedAt field as that's redundant and currently set to the same value as the AdvertisingTokenInput/RefreshTokenInput#createdAt timestamp. 2. Renamed some `token` variable names to adTokenInput and refreshTokenInput to make it clearer --- .../userIdentity/FirstLevelHashIdentity.java | 4 +- .../model/userIdentity/HashedDiiIdentity.java | 4 +- .../model/userIdentity/RawUidIdentity.java | 4 +- .../model/userIdentity/UserIdentity.java | 4 +- .../service/EncryptedTokenEncoder.java | 10 ++--- .../com/uid2/operator/service/InputUtil.java | 1 - .../operator/service/UIDOperatorService.java | 6 +-- .../com/uid2/operator/TokenEncodingTest.java | 38 +++++++++---------- .../uid2/operator/UIDOperatorServiceTest.java | 3 +- .../operator/benchmark/BenchmarkCommon.java | 2 +- 10 files changed, 36 insertions(+), 40 deletions(-) diff --git a/src/main/java/com/uid2/operator/model/userIdentity/FirstLevelHashIdentity.java b/src/main/java/com/uid2/operator/model/userIdentity/FirstLevelHashIdentity.java index 64b8bcedd..d48607a4c 100644 --- a/src/main/java/com/uid2/operator/model/userIdentity/FirstLevelHashIdentity.java +++ b/src/main/java/com/uid2/operator/model/userIdentity/FirstLevelHashIdentity.java @@ -11,8 +11,8 @@ public class FirstLevelHashIdentity extends UserIdentity { public final byte[] firstLevelHash; public FirstLevelHashIdentity(IdentityScope identityScope, IdentityType identityType, byte[] firstLevelHash, int privacyBits, - Instant establishedAt, Instant refreshedAt) { - super(identityScope, identityType, privacyBits, establishedAt, refreshedAt); + Instant establishedAt) { + super(identityScope, identityType, privacyBits, establishedAt); this.firstLevelHash = firstLevelHash; } diff --git a/src/main/java/com/uid2/operator/model/userIdentity/HashedDiiIdentity.java b/src/main/java/com/uid2/operator/model/userIdentity/HashedDiiIdentity.java index dad862f21..6f7ced3c5 100644 --- a/src/main/java/com/uid2/operator/model/userIdentity/HashedDiiIdentity.java +++ b/src/main/java/com/uid2/operator/model/userIdentity/HashedDiiIdentity.java @@ -11,8 +11,8 @@ public class HashedDiiIdentity extends UserIdentity { public final byte[] hashedDii; public HashedDiiIdentity(IdentityScope identityScope, IdentityType identityType, byte[] hashedDii, int privacyBits, - Instant establishedAt, Instant refreshedAt) { - super(identityScope, identityType, privacyBits, establishedAt, refreshedAt); + Instant establishedAt) { + super(identityScope, identityType, privacyBits, establishedAt); this.hashedDii = hashedDii; } } diff --git a/src/main/java/com/uid2/operator/model/userIdentity/RawUidIdentity.java b/src/main/java/com/uid2/operator/model/userIdentity/RawUidIdentity.java index 4e15c6ff0..80bbbd182 100644 --- a/src/main/java/com/uid2/operator/model/userIdentity/RawUidIdentity.java +++ b/src/main/java/com/uid2/operator/model/userIdentity/RawUidIdentity.java @@ -11,8 +11,8 @@ public class RawUidIdentity extends UserIdentity { public final byte[] rawUid; public RawUidIdentity(IdentityScope identityScope, IdentityType identityType, byte[] rawUid, int privacyBits, - Instant establishedAt, Instant refreshedAt) { - super(identityScope, identityType, privacyBits, establishedAt, refreshedAt); + Instant establishedAt) { + super(identityScope, identityType, privacyBits, establishedAt); this.rawUid = rawUid; } diff --git a/src/main/java/com/uid2/operator/model/userIdentity/UserIdentity.java b/src/main/java/com/uid2/operator/model/userIdentity/UserIdentity.java index 1391b7d75..b32da6385 100644 --- a/src/main/java/com/uid2/operator/model/userIdentity/UserIdentity.java +++ b/src/main/java/com/uid2/operator/model/userIdentity/UserIdentity.java @@ -12,13 +12,11 @@ public abstract class UserIdentity { public final IdentityType identityType; public final int privacyBits; public final Instant establishedAt; - public final Instant refreshedAt; - public UserIdentity(IdentityScope identityScope, IdentityType identityType, int privacyBits, Instant establishedAt, Instant refreshedAt) { + public UserIdentity(IdentityScope identityScope, IdentityType identityType, int privacyBits, Instant establishedAt) { this.identityScope = identityScope; this.identityType = identityType; this.privacyBits = privacyBits; this.establishedAt = establishedAt; - this.refreshedAt = refreshedAt; } } diff --git a/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java b/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java index 1a74a794a..fe2af06b3 100644 --- a/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java +++ b/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java @@ -55,7 +55,7 @@ private byte[] encodeIntoAdvertisingTokenV3(AdvertisingTokenInput t, KeysetKey m encodePublisherRequesterV3(sitePayload, t.sourcePublisher); sitePayload.appendInt(t.rawUidIdentity.privacyBits); sitePayload.appendLong(t.rawUidIdentity.establishedAt.toEpochMilli()); - sitePayload.appendLong(t.rawUidIdentity.refreshedAt.toEpochMilli()); + sitePayload.appendLong(t.createdAt.toEpochMilli()); sitePayload.appendBytes(t.rawUidIdentity.rawUid); // 32 or 33 bytes final Buffer masterPayload = Buffer.buffer(130); @@ -127,7 +127,7 @@ private RefreshTokenInput decodeRefreshTokenV2(Buffer b) { new OperatorIdentity(0, OperatorType.Service, 0, 0), new SourcePublisher(siteId, 0, 0), new FirstLevelHashIdentity(IdentityScope.UID2, IdentityType.Email, identity, privacyBits, - Instant.ofEpochMilli(establishedMillis), null)); + Instant.ofEpochMilli(establishedMillis))); } private RefreshTokenInput decodeRefreshTokenV3(Buffer b, byte[] bytes) { @@ -160,7 +160,7 @@ private RefreshTokenInput decodeRefreshTokenV3(Buffer b, byte[] bytes) { return new RefreshTokenInput( TokenVersion.V3, createdAt, expiresAt, operatorIdentity, sourcePublisher, - new FirstLevelHashIdentity(identityScope, identityType, firstLevelHash, privacyBits, establishedAt, null)); + new FirstLevelHashIdentity(identityScope, identityType, firstLevelHash, privacyBits, establishedAt)); } @Override @@ -229,7 +229,7 @@ public AdvertisingTokenInput decodeAdvertisingTokenV2(Buffer b) { new OperatorIdentity(0, OperatorType.Service, 0, masterKeyId), new SourcePublisher(siteId, siteKeyId, 0), new RawUidIdentity(IdentityScope.UID2, IdentityType.Email, rawUid, privacyBits, - Instant.ofEpochMilli(establishedMillis), null) + Instant.ofEpochMilli(establishedMillis)) ); } catch (Exception e) { @@ -269,7 +269,7 @@ public AdvertisingTokenInput decodeAdvertisingTokenV3orV4(Buffer b, byte[] bytes return new AdvertisingTokenInput( tokenVersion, createdAt, expiresAt, operatorIdentity, sourcePublisher, - new RawUidIdentity(identityScope, identityType, rawUid, privacyBits, establishedAt, refreshedAt) + new RawUidIdentity(identityScope, identityType, rawUid, privacyBits, establishedAt) ); } diff --git a/src/main/java/com/uid2/operator/service/InputUtil.java b/src/main/java/com/uid2/operator/service/InputUtil.java index ff9b3647b..fe63146f6 100644 --- a/src/main/java/com/uid2/operator/service/InputUtil.java +++ b/src/main/java/com/uid2/operator/service/InputUtil.java @@ -267,7 +267,6 @@ public HashedDiiIdentity toHashedDiiIdentity(IdentityScope identityScope, int pr this.identityType, getIdentityInput(), privacyBits, - establishedAt, establishedAt); } } diff --git a/src/main/java/com/uid2/operator/service/UIDOperatorService.java b/src/main/java/com/uid2/operator/service/UIDOperatorService.java index 4c397885c..d8d1f8957 100644 --- a/src/main/java/com/uid2/operator/service/UIDOperatorService.java +++ b/src/main/java/com/uid2/operator/service/UIDOperatorService.java @@ -110,7 +110,7 @@ public IdentityResponse generateIdentity(IdentityRequest request) { final byte[] firstLevelHash = getFirstLevelHash(request.hashedDiiIdentity.hashedDii, now); final FirstLevelHashIdentity firstLevelHashIdentity = new FirstLevelHashIdentity( request.hashedDiiIdentity.identityScope, request.hashedDiiIdentity.identityType, firstLevelHash, request.hashedDiiIdentity.privacyBits, - request.hashedDiiIdentity.establishedAt, request.hashedDiiIdentity.refreshedAt); + request.hashedDiiIdentity.establishedAt); if (request.shouldCheckOptOut() && getGlobalOptOutResult(firstLevelHashIdentity, false).isOptedOut()) { return IdentityResponse.OptOutIdentityResponse; @@ -231,7 +231,7 @@ private FirstLevelHashIdentity getFirstLevelHashIdentity(HashedDiiIdentity hashe private FirstLevelHashIdentity getFirstLevelHashIdentity(IdentityScope identityScope, IdentityType identityType, byte[] identityHash, Instant asOf) { final byte[] firstLevelHash = getFirstLevelHash(identityHash, asOf); - return new FirstLevelHashIdentity(identityScope, identityType, firstLevelHash, 0, null, null); + return new FirstLevelHashIdentity(identityScope, identityType, firstLevelHash, 0, null); } private byte[] getFirstLevelHash(byte[] identityHash, Instant asOf) { @@ -255,7 +255,7 @@ private IdentityResponse generateIdentity(SourcePublisher sourcePublisher, First final RawUidResponse rawUidResponse = generateRawUid(firstLevelHashIdentity, nowUtc); final RawUidIdentity rawUidIdentity = new RawUidIdentity(firstLevelHashIdentity.identityScope, firstLevelHashIdentity.identityType, - rawUidResponse.rawUid, firstLevelHashIdentity.privacyBits, firstLevelHashIdentity.establishedAt, nowUtc); + rawUidResponse.rawUid, firstLevelHashIdentity.privacyBits, firstLevelHashIdentity.establishedAt); return this.encoder.encodeIntoIdentityResponse( this.createAdvertisingTokenInput(sourcePublisher, rawUidIdentity, nowUtc), diff --git a/src/test/java/com/uid2/operator/TokenEncodingTest.java b/src/test/java/com/uid2/operator/TokenEncodingTest.java index b53f79b4f..656d50f98 100644 --- a/src/test/java/com/uid2/operator/TokenEncodingTest.java +++ b/src/test/java/com/uid2/operator/TokenEncodingTest.java @@ -54,29 +54,29 @@ public void testRefreshTokenEncoding(TokenVersion tokenVersion) { final byte[] firstLevelHash = TokenUtils.getFirstLevelHashFromIdentity("test@example.com", "some-salt"); - final RefreshTokenInput token = new RefreshTokenInput(tokenVersion, + final RefreshTokenInput refreshTokenInput = new RefreshTokenInput(tokenVersion, now, now.plusSeconds(360), new OperatorIdentity(101, OperatorType.Service, 102, 103), new SourcePublisher(111, 112, 113), - new FirstLevelHashIdentity(IdentityScope.UID2, IdentityType.Email, firstLevelHash, 121, now, now.minusSeconds(122)) + new FirstLevelHashIdentity(IdentityScope.UID2, IdentityType.Email, firstLevelHash, 121, now) ); if (tokenVersion == TokenVersion.V4) { - Assert.assertThrows(Exception.class, () -> encoder.encodeIntoRefreshToken(token, now)); + Assert.assertThrows(Exception.class, () -> encoder.encodeIntoRefreshToken(refreshTokenInput, now)); return; //V4 not supported for RefreshTokens } - final byte[] encodedBytes = encoder.encodeIntoRefreshToken(token, now); + final byte[] encodedBytes = encoder.encodeIntoRefreshToken(refreshTokenInput, now); final RefreshTokenInput decoded = encoder.decodeRefreshToken(EncodingUtils.toBase64String(encodedBytes)); assertEquals(tokenVersion, decoded.version); - assertEquals(token.createdAt, decoded.createdAt); + assertEquals(refreshTokenInput.createdAt, decoded.createdAt); int addSeconds = (tokenVersion == TokenVersion.V2) ? 60 : 0; //todo: why is there a 60 second buffer in encodeV2() but not in encodeV3()? - assertEquals(token.expiresAt.plusSeconds(addSeconds), decoded.expiresAt); - assertTrue(token.firstLevelHashIdentity.matches(decoded.firstLevelHashIdentity)); - assertEquals(token.firstLevelHashIdentity.privacyBits, decoded.firstLevelHashIdentity.privacyBits); - assertEquals(token.firstLevelHashIdentity.establishedAt, decoded.firstLevelHashIdentity.establishedAt); - assertEquals(token.sourcePublisher.siteId, decoded.sourcePublisher.siteId); + assertEquals(refreshTokenInput.expiresAt.plusSeconds(addSeconds), decoded.expiresAt); + assertTrue(refreshTokenInput.firstLevelHashIdentity.matches(decoded.firstLevelHashIdentity)); + assertEquals(refreshTokenInput.firstLevelHashIdentity.privacyBits, decoded.firstLevelHashIdentity.privacyBits); + assertEquals(refreshTokenInput.firstLevelHashIdentity.establishedAt, decoded.firstLevelHashIdentity.establishedAt); + assertEquals(refreshTokenInput.sourcePublisher.siteId, decoded.sourcePublisher.siteId); Buffer b = Buffer.buffer(encodedBytes); int keyId = b.getInt(tokenVersion == TokenVersion.V2 ? 25 : 2); @@ -95,25 +95,25 @@ public void testAdvertisingTokenEncodings(TokenVersion tokenVersion) { final byte[] rawUid = UIDOperatorVerticleTest.getRawUid(IdentityType.Email, "test@example.com", IdentityScope.UID2, tokenVersion != TokenVersion.V2); - final AdvertisingTokenInput token = new AdvertisingTokenInput( + final AdvertisingTokenInput adTokenInput = new AdvertisingTokenInput( tokenVersion, now, now.plusSeconds(60), new OperatorIdentity(101, OperatorType.Service, 102, 103), new SourcePublisher(111, 112, 113), - new RawUidIdentity(IdentityScope.UID2, IdentityType.Email, rawUid, 121, now, now.minusSeconds(122)) + new RawUidIdentity(IdentityScope.UID2, IdentityType.Email, rawUid, 121, now) ); - final byte[] encodedBytes = encoder.encodeIntoAdvertisingToken(token, now); + final byte[] encodedBytes = encoder.encodeIntoAdvertisingToken(adTokenInput, now); final AdvertisingTokenInput decoded = encoder.decodeAdvertisingToken(EncryptedTokenEncoder.bytesToBase64Token(encodedBytes, tokenVersion)); assertEquals(tokenVersion, decoded.version); - assertEquals(token.createdAt, decoded.createdAt); - assertEquals(token.expiresAt, decoded.expiresAt); - assertTrue(token.rawUidIdentity.matches(decoded.rawUidIdentity)); - assertEquals(token.rawUidIdentity.privacyBits, decoded.rawUidIdentity.privacyBits); - assertEquals(token.rawUidIdentity.establishedAt, decoded.rawUidIdentity.establishedAt); - assertEquals(token.sourcePublisher.siteId, decoded.sourcePublisher.siteId); + assertEquals(adTokenInput.createdAt, decoded.createdAt); + assertEquals(adTokenInput.expiresAt, decoded.expiresAt); + assertTrue(adTokenInput.rawUidIdentity.matches(decoded.rawUidIdentity)); + assertEquals(adTokenInput.rawUidIdentity.privacyBits, decoded.rawUidIdentity.privacyBits); + assertEquals(adTokenInput.rawUidIdentity.establishedAt, decoded.rawUidIdentity.establishedAt); + assertEquals(adTokenInput.sourcePublisher.siteId, decoded.sourcePublisher.siteId); Buffer b = Buffer.buffer(encodedBytes); int keyId = b.getInt(tokenVersion == TokenVersion.V2 ? 1 : 2); //TODO - extract master key from token should be a helper function diff --git a/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java b/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java index 5196dfaf2..efc7c0578 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java @@ -129,8 +129,7 @@ private HashedDiiIdentity createHashedDiiIdentity(String rawIdentityHash, Identi type, rawIdentityHash.getBytes(StandardCharsets.UTF_8), 0, - this.now.minusSeconds(234), - this.now.plusSeconds(12345) + this.now.minusSeconds(234) ); } diff --git a/src/test/java/com/uid2/operator/benchmark/BenchmarkCommon.java b/src/test/java/com/uid2/operator/benchmark/BenchmarkCommon.java index 1c25faecd..58f555818 100644 --- a/src/test/java/com/uid2/operator/benchmark/BenchmarkCommon.java +++ b/src/test/java/com/uid2/operator/benchmark/BenchmarkCommon.java @@ -156,7 +156,7 @@ static HashedDiiIdentity[] createHashedDiiIdentities() { final byte[] diiHash = new byte[33]; new Random().nextBytes(diiHash); arr[i] = new HashedDiiIdentity(IdentityScope.UID2, IdentityType.Email, diiHash, 0, - Instant.now().minusSeconds(120), Instant.now().minusSeconds(60)); + Instant.now().minusSeconds(120)); } return arr; } From 0ab0a79f29a0ca294e9a0587bfed2cf16002790a Mon Sep 17 00:00:00 2001 From: Sunny Wu Date: Wed, 16 Oct 2024 18:42:22 +1100 Subject: [PATCH 063/431] 1. Removed privacyBits and establishedAt from UserIdentity/HashedDiiIdentity/RawUidIdentity but only kept inside FirstLevelHashIdentity - these fields are only relevant when a first level hash is generated for token generation and then passed along into ad token generation logic. 2. As a result, privacyBits and establishedAt are generated from either a. in a brand new token generation call (by default should be 1 and the timestamp at the time of the call) b. or, during token refresh logic, these fields will be inherited from previous refresh token. 3. Updated a lot of codes as a result (more refinement required such as not hardcoding the "1, Instant.now()" in a lot of places) --- .../operator/model/AdvertisingTokenInput.java | 8 +- .../uid2/operator/model/IdentityRequest.java | 11 ++- .../userIdentity/FirstLevelHashIdentity.java | 12 ++- .../model/userIdentity/HashedDiiIdentity.java | 5 +- .../model/userIdentity/RawUidIdentity.java | 5 +- .../model/userIdentity/UserIdentity.java | 6 +- .../service/EncryptedTokenEncoder.java | 24 +++--- .../com/uid2/operator/service/InputUtil.java | 6 +- .../operator/service/UIDOperatorService.java | 13 ++-- .../operator/vertx/UIDOperatorVerticle.java | 38 +++++----- .../com/uid2/operator/TokenEncodingTest.java | 8 +- .../uid2/operator/UIDOperatorServiceTest.java | 75 ++++++++++--------- .../operator/UIDOperatorVerticleTest.java | 42 +++++------ .../operator/benchmark/BenchmarkCommon.java | 3 +- .../benchmark/TokenEndecBenchmark.java | 5 +- 15 files changed, 144 insertions(+), 117 deletions(-) diff --git a/src/main/java/com/uid2/operator/model/AdvertisingTokenInput.java b/src/main/java/com/uid2/operator/model/AdvertisingTokenInput.java index b5ffcb89a..2d32dc4f4 100644 --- a/src/main/java/com/uid2/operator/model/AdvertisingTokenInput.java +++ b/src/main/java/com/uid2/operator/model/AdvertisingTokenInput.java @@ -10,12 +10,18 @@ public class AdvertisingTokenInput extends VersionedToken { public final SourcePublisher sourcePublisher; public final RawUidIdentity rawUidIdentity; + public final int privacyBits; + public final Instant establishedAt; + public AdvertisingTokenInput(TokenVersion version, Instant createdAt, Instant expiresAt, OperatorIdentity operatorIdentity, - SourcePublisher sourcePublisher, RawUidIdentity rawUidIdentity) { + SourcePublisher sourcePublisher, RawUidIdentity rawUidIdentity, int privacyBits, + Instant establishedAt) { super(version, createdAt, expiresAt); this.operatorIdentity = operatorIdentity; this.sourcePublisher = sourcePublisher; this.rawUidIdentity = rawUidIdentity; + this.privacyBits = privacyBits; + this.establishedAt = establishedAt; } } diff --git a/src/main/java/com/uid2/operator/model/IdentityRequest.java b/src/main/java/com/uid2/operator/model/IdentityRequest.java index e9a0c96cb..a4f7c9b1d 100644 --- a/src/main/java/com/uid2/operator/model/IdentityRequest.java +++ b/src/main/java/com/uid2/operator/model/IdentityRequest.java @@ -2,19 +2,28 @@ import com.uid2.operator.model.userIdentity.HashedDiiIdentity; +import java.time.Instant; + public final class IdentityRequest { public final SourcePublisher sourcePublisher; public final HashedDiiIdentity hashedDiiIdentity; public final OptoutCheckPolicy optoutCheckPolicy; + public final int privacyBits; + public final Instant establishedAt; + public IdentityRequest( SourcePublisher sourcePublisher, HashedDiiIdentity hashedDiiIdentity, - OptoutCheckPolicy tokenGeneratePolicy) + OptoutCheckPolicy tokenGeneratePolicy, + int privacyBits, + Instant establishedAt) { this.sourcePublisher = sourcePublisher; this.hashedDiiIdentity = hashedDiiIdentity; this.optoutCheckPolicy = tokenGeneratePolicy; + this.privacyBits = privacyBits; + this.establishedAt = establishedAt; } public boolean shouldCheckOptOut() { diff --git a/src/main/java/com/uid2/operator/model/userIdentity/FirstLevelHashIdentity.java b/src/main/java/com/uid2/operator/model/userIdentity/FirstLevelHashIdentity.java index d48607a4c..5eb13526a 100644 --- a/src/main/java/com/uid2/operator/model/userIdentity/FirstLevelHashIdentity.java +++ b/src/main/java/com/uid2/operator/model/userIdentity/FirstLevelHashIdentity.java @@ -10,10 +10,20 @@ public class FirstLevelHashIdentity extends UserIdentity { public final byte[] firstLevelHash; + // for brand new token generation, it should contain 1 + // if the first level hash is from token/refresh call, it will inherit from the refresh token + public final int privacyBits; + + // for brand new token generation, it should be the time it is generated + // if the first level hash is from token/refresh call, it will be when the raw UID was originally created in the earliest token generation + public final Instant establishedAt; + public FirstLevelHashIdentity(IdentityScope identityScope, IdentityType identityType, byte[] firstLevelHash, int privacyBits, Instant establishedAt) { - super(identityScope, identityType, privacyBits, establishedAt); + super(identityScope, identityType); this.firstLevelHash = firstLevelHash; + this.privacyBits = privacyBits; + this.establishedAt = establishedAt; } public boolean matches(FirstLevelHashIdentity that) { diff --git a/src/main/java/com/uid2/operator/model/userIdentity/HashedDiiIdentity.java b/src/main/java/com/uid2/operator/model/userIdentity/HashedDiiIdentity.java index 6f7ced3c5..a9a6efe0a 100644 --- a/src/main/java/com/uid2/operator/model/userIdentity/HashedDiiIdentity.java +++ b/src/main/java/com/uid2/operator/model/userIdentity/HashedDiiIdentity.java @@ -10,9 +10,8 @@ public class HashedDiiIdentity extends UserIdentity { public final byte[] hashedDii; - public HashedDiiIdentity(IdentityScope identityScope, IdentityType identityType, byte[] hashedDii, int privacyBits, - Instant establishedAt) { - super(identityScope, identityType, privacyBits, establishedAt); + public HashedDiiIdentity(IdentityScope identityScope, IdentityType identityType, byte[] hashedDii) { + super(identityScope, identityType); this.hashedDii = hashedDii; } } diff --git a/src/main/java/com/uid2/operator/model/userIdentity/RawUidIdentity.java b/src/main/java/com/uid2/operator/model/userIdentity/RawUidIdentity.java index 80bbbd182..b86740a05 100644 --- a/src/main/java/com/uid2/operator/model/userIdentity/RawUidIdentity.java +++ b/src/main/java/com/uid2/operator/model/userIdentity/RawUidIdentity.java @@ -10,9 +10,8 @@ public class RawUidIdentity extends UserIdentity { public final byte[] rawUid; - public RawUidIdentity(IdentityScope identityScope, IdentityType identityType, byte[] rawUid, int privacyBits, - Instant establishedAt) { - super(identityScope, identityType, privacyBits, establishedAt); + public RawUidIdentity(IdentityScope identityScope, IdentityType identityType, byte[] rawUid) { + super(identityScope, identityType); this.rawUid = rawUid; } diff --git a/src/main/java/com/uid2/operator/model/userIdentity/UserIdentity.java b/src/main/java/com/uid2/operator/model/userIdentity/UserIdentity.java index b32da6385..8221a8c61 100644 --- a/src/main/java/com/uid2/operator/model/userIdentity/UserIdentity.java +++ b/src/main/java/com/uid2/operator/model/userIdentity/UserIdentity.java @@ -10,13 +10,9 @@ public abstract class UserIdentity { public final IdentityScope identityScope; public final IdentityType identityType; - public final int privacyBits; - public final Instant establishedAt; - public UserIdentity(IdentityScope identityScope, IdentityType identityType, int privacyBits, Instant establishedAt) { + public UserIdentity(IdentityScope identityScope, IdentityType identityType) { this.identityScope = identityScope; this.identityType = identityType; - this.privacyBits = privacyBits; - this.establishedAt = establishedAt; } } diff --git a/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java b/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java index fe2af06b3..67fac4d1d 100644 --- a/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java +++ b/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java @@ -41,7 +41,7 @@ private byte[] encodeIntoAdvertisingTokenV2(AdvertisingTokenInput t, KeysetKey m Buffer b2 = Buffer.buffer(); b2.appendLong(t.expiresAt.toEpochMilli()); - encodeSiteIdentityV2(b2, t.sourcePublisher, t.rawUidIdentity, siteKey); + encodeSiteIdentityV2(b2, t.sourcePublisher, t.rawUidIdentity, siteKey, t.privacyBits, t.establishedAt); final byte[] encryptedId = AesCbc.encrypt(b2.getBytes(), masterKey).getPayload(); @@ -53,8 +53,8 @@ private byte[] encodeIntoAdvertisingTokenV2(AdvertisingTokenInput t, KeysetKey m private byte[] encodeIntoAdvertisingTokenV3(AdvertisingTokenInput t, KeysetKey masterKey, KeysetKey siteKey) { final Buffer sitePayload = Buffer.buffer(69); encodePublisherRequesterV3(sitePayload, t.sourcePublisher); - sitePayload.appendInt(t.rawUidIdentity.privacyBits); - sitePayload.appendLong(t.rawUidIdentity.establishedAt.toEpochMilli()); + sitePayload.appendInt(t.privacyBits); + sitePayload.appendLong(t.establishedAt.toEpochMilli()); sitePayload.appendLong(t.createdAt.toEpochMilli()); sitePayload.appendBytes(t.rawUidIdentity.rawUid); // 32 or 33 bytes @@ -228,8 +228,9 @@ public AdvertisingTokenInput decodeAdvertisingTokenV2(Buffer b) { Instant.ofEpochMilli(expiresMillis), new OperatorIdentity(0, OperatorType.Service, 0, masterKeyId), new SourcePublisher(siteId, siteKeyId, 0), - new RawUidIdentity(IdentityScope.UID2, IdentityType.Email, rawUid, privacyBits, - Instant.ofEpochMilli(establishedMillis)) + new RawUidIdentity(IdentityScope.UID2, IdentityType.Email, rawUid), + privacyBits, + Instant.ofEpochMilli(establishedMillis) ); } catch (Exception e) { @@ -269,7 +270,8 @@ public AdvertisingTokenInput decodeAdvertisingTokenV3orV4(Buffer b, byte[] bytes return new AdvertisingTokenInput( tokenVersion, createdAt, expiresAt, operatorIdentity, sourcePublisher, - new RawUidIdentity(identityScope, identityType, rawUid, privacyBits, establishedAt) + new RawUidIdentity(identityScope, identityType, rawUid), + privacyBits, establishedAt ); } @@ -330,9 +332,9 @@ public byte[] encodeIntoRefreshTokenV3(RefreshTokenInput t, KeysetKey serviceKey } private void encodeSiteIdentityV2(Buffer b, SourcePublisher sourcePublisher, RawUidIdentity rawUidIdentity, - KeysetKey siteEncryptionKey) { + KeysetKey siteEncryptionKey, int privacyBits, Instant establishedAt) { b.appendInt(siteEncryptionKey.getId()); - final byte[] encryptedIdentity = encryptIdentityV2(sourcePublisher, rawUidIdentity, siteEncryptionKey); + final byte[] encryptedIdentity = encryptIdentityV2(sourcePublisher, rawUidIdentity, siteEncryptionKey, privacyBits, establishedAt); b.appendBytes(encryptedIdentity); } @@ -370,9 +372,9 @@ private byte[] encryptIdentityV2(SourcePublisher sourcePublisher, FirstLevelHash } private byte[] encryptIdentityV2(SourcePublisher sourcePublisher, RawUidIdentity rawUidIdentity, - KeysetKey key) { - return encryptIdentityV2(sourcePublisher, rawUidIdentity.rawUid, rawUidIdentity.privacyBits, - rawUidIdentity.establishedAt, key); + KeysetKey key, int privacyBits, Instant establishedAt) { + return encryptIdentityV2(sourcePublisher, rawUidIdentity.rawUid, privacyBits, + establishedAt, key); } diff --git a/src/main/java/com/uid2/operator/service/InputUtil.java b/src/main/java/com/uid2/operator/service/InputUtil.java index fe63146f6..8e44e5717 100644 --- a/src/main/java/com/uid2/operator/service/InputUtil.java +++ b/src/main/java/com/uid2/operator/service/InputUtil.java @@ -261,13 +261,11 @@ public boolean isValid() { return valid; } - public HashedDiiIdentity toHashedDiiIdentity(IdentityScope identityScope, int privacyBits, Instant establishedAt) { + public HashedDiiIdentity toHashedDiiIdentity(IdentityScope identityScope) { return new HashedDiiIdentity( identityScope, this.identityType, - getIdentityInput(), - privacyBits, - establishedAt); + getIdentityInput()); } } diff --git a/src/main/java/com/uid2/operator/service/UIDOperatorService.java b/src/main/java/com/uid2/operator/service/UIDOperatorService.java index d8d1f8957..eed0c0bb8 100644 --- a/src/main/java/com/uid2/operator/service/UIDOperatorService.java +++ b/src/main/java/com/uid2/operator/service/UIDOperatorService.java @@ -109,8 +109,8 @@ public IdentityResponse generateIdentity(IdentityRequest request) { final Instant now = EncodingUtils.NowUTCMillis(this.clock); final byte[] firstLevelHash = getFirstLevelHash(request.hashedDiiIdentity.hashedDii, now); final FirstLevelHashIdentity firstLevelHashIdentity = new FirstLevelHashIdentity( - request.hashedDiiIdentity.identityScope, request.hashedDiiIdentity.identityType, firstLevelHash, request.hashedDiiIdentity.privacyBits, - request.hashedDiiIdentity.establishedAt); + request.hashedDiiIdentity.identityScope, request.hashedDiiIdentity.identityType, firstLevelHash, request.privacyBits, + request.establishedAt); if (request.shouldCheckOptOut() && getGlobalOptOutResult(firstLevelHashIdentity, false).isOptedOut()) { return IdentityResponse.OptOutIdentityResponse; @@ -255,10 +255,10 @@ private IdentityResponse generateIdentity(SourcePublisher sourcePublisher, First final RawUidResponse rawUidResponse = generateRawUid(firstLevelHashIdentity, nowUtc); final RawUidIdentity rawUidIdentity = new RawUidIdentity(firstLevelHashIdentity.identityScope, firstLevelHashIdentity.identityType, - rawUidResponse.rawUid, firstLevelHashIdentity.privacyBits, firstLevelHashIdentity.establishedAt); + rawUidResponse.rawUid); return this.encoder.encodeIntoIdentityResponse( - this.createAdvertisingTokenInput(sourcePublisher, rawUidIdentity, nowUtc), + this.createAdvertisingTokenInput(sourcePublisher, rawUidIdentity, nowUtc, firstLevelHashIdentity.privacyBits, firstLevelHashIdentity.establishedAt), this.createRefreshTokenInput(sourcePublisher, firstLevelHashIdentity, nowUtc), nowUtc.plusMillis(refreshIdentityAfter.toMillis()), nowUtc @@ -277,7 +277,7 @@ private RefreshTokenInput createRefreshTokenInput(SourcePublisher sourcePublishe } private AdvertisingTokenInput createAdvertisingTokenInput(SourcePublisher sourcePublisher, RawUidIdentity rawUidIdentity, - Instant now) { + Instant now, int privacyBits, Instant establishedAt) { TokenVersion tokenVersion; if (siteIdsUsingV4Tokens.contains(sourcePublisher.siteId)) { tokenVersion = TokenVersion.V4; @@ -291,7 +291,8 @@ private AdvertisingTokenInput createAdvertisingTokenInput(SourcePublisher source } tokenVersion = (pseudoRandomNumber <= this.advertisingTokenV4Percentage) ? TokenVersion.V4 : this.tokenVersionToUseIfNotV4; } - return new AdvertisingTokenInput(tokenVersion, now, now.plusMillis(identityExpiresAfter.toMillis()), this.operatorIdentity, sourcePublisher, rawUidIdentity); + return new AdvertisingTokenInput(tokenVersion, now, now.plusMillis(identityExpiresAfter.toMillis()), this.operatorIdentity, sourcePublisher, rawUidIdentity, + privacyBits, establishedAt); } static protected class GlobalOptoutResult { diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index 58c6703a1..143de537d 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -465,8 +465,8 @@ else if(emailHash != null) { identityResponse = this.idService.generateIdentity( new IdentityRequest( new SourcePublisher(clientSideKeypair.getSiteId(), 0, 0), - input.toHashedDiiIdentity(this.identityScope, privacyBits.getAsInt(), Instant.now()), - OptoutCheckPolicy.RespectOptOut)); + input.toHashedDiiIdentity(this.identityScope), + OptoutCheckPolicy.RespectOptOut, privacyBits.getAsInt(), Instant.now())); } catch (KeyManager.NoActiveKeyException e){ SendServerErrorResponseAndRecordStats(rc, "No active encryption key available", clientSideKeypair.getSiteId(), TokenResponseStatsCollector.Endpoint.ClientSideTokenGenerateV2, TokenResponseStatsCollector.ResponseStatus.NoActiveKey, siteProvider, e, platformType); return; @@ -878,7 +878,7 @@ private void handleTokenValidateV1(RoutingContext rc) { || (Arrays.equals(ValidateIdentityForPhoneHash, input.getIdentityInput()) && input.getIdentityType() == IdentityType.Phone)) { try { final Instant now = Instant.now(); - if (this.idService.advertisingTokenMatches(rc.queryParam("token").get(0), input.toHashedDiiIdentity(this.identityScope, 0, now), now)) { + if (this.idService.advertisingTokenMatches(rc.queryParam("token").get(0), input.toHashedDiiIdentity(this.identityScope), now)) { ResponseUtil.Success(rc, Boolean.TRUE); } else { ResponseUtil.Success(rc, Boolean.FALSE); @@ -911,7 +911,7 @@ private void handleTokenValidateV2(RoutingContext rc) { final Instant now = Instant.now(); final String token = req.getString("token"); - if (this.idService.advertisingTokenMatches(token, input.toHashedDiiIdentity(this.identityScope, 0, now), now)) { + if (this.idService.advertisingTokenMatches(token, input.toHashedDiiIdentity(this.identityScope), now)) { ResponseUtil.SuccessV2(rc, Boolean.TRUE); } else { ResponseUtil.SuccessV2(rc, Boolean.FALSE); @@ -940,8 +940,8 @@ private void handleTokenGenerateV1(RoutingContext rc) { final IdentityResponse t = this.idService.generateIdentity( new IdentityRequest( new SourcePublisher(siteId, 0, 0), - input.toHashedDiiIdentity(this.identityScope, 1, Instant.now()), - OptoutCheckPolicy.defaultPolicy())); + input.toHashedDiiIdentity(this.identityScope), + OptoutCheckPolicy.defaultPolicy(), 1, Instant.now())); //Integer.parseInt(rc.queryParam("privacy_bits").get(0)))); @@ -996,8 +996,8 @@ private void handleTokenGenerateV2(RoutingContext rc) { final IdentityResponse t = this.idService.generateIdentity( new IdentityRequest( new SourcePublisher(siteId, 0, 0), - input.toHashedDiiIdentity(this.identityScope, 1, Instant.now()), - OptoutCheckPolicy.respectOptOut())); + input.toHashedDiiIdentity(this.identityScope), + OptoutCheckPolicy.respectOptOut(),1, Instant.now())); if (t.isOptedOut()) { if (optoutCheckPolicy.getItem1() == OptoutCheckPolicy.DoNotRespect) { // only legacy can use this policy @@ -1012,8 +1012,8 @@ private void handleTokenGenerateV2(RoutingContext rc) { final IdentityResponse optOutTokens = this.idService.generateIdentity( new IdentityRequest( new SourcePublisher(siteId, 0, 0), - optOutTokenInput.toHashedDiiIdentity(this.identityScope, pb.getAsInt(), Instant.now()), - OptoutCheckPolicy.DoNotRespect)); + optOutTokenInput.toHashedDiiIdentity(this.identityScope), + OptoutCheckPolicy.DoNotRespect, pb.getAsInt(), Instant.now())); ResponseUtil.SuccessV2(rc, optOutTokens.toJsonV1()); recordTokenResponseStats(siteId, TokenResponseStatsCollector.Endpoint.GenerateV2, TokenResponseStatsCollector.ResponseStatus.Success, siteProvider, optOutTokens.getAdvertisingTokenVersion(), platformType); @@ -1052,8 +1052,8 @@ else if (!input.isValid()) { final IdentityResponse t = this.idService.generateIdentity( new IdentityRequest( new SourcePublisher(siteId, 0, 0), - input.toHashedDiiIdentity(this.identityScope, 1, Instant.now()), - OptoutCheckPolicy.defaultPolicy())); + input.toHashedDiiIdentity(this.identityScope), + OptoutCheckPolicy.defaultPolicy(), 1, Instant.now())); //Integer.parseInt(rc.queryParam("privacy_bits").get(0)))); @@ -1094,7 +1094,7 @@ private void handleValidate(RoutingContext rc) { if (input != null && input.isValid() && Arrays.equals(ValidateIdentityForEmailHash, input.getIdentityInput())) { try { final Instant now = Instant.now(); - if (this.idService.advertisingTokenMatches(rc.queryParam("token").get(0), input.toHashedDiiIdentity(this.identityScope, 0, now), now)) { + if (this.idService.advertisingTokenMatches(rc.queryParam("token").get(0), input.toHashedDiiIdentity(this.identityScope), now)) { rc.response().end("true"); } else { rc.response().end("false"); @@ -1115,7 +1115,7 @@ private void handleLogoutAsync(RoutingContext rc) { final InputUtil.InputVal input = this.phoneSupport ? getTokenInputV1(rc) : getTokenInput(rc); if (input.isValid()) { final Instant now = Instant.now(); - this.idService.invalidateTokensAsync(input.toHashedDiiIdentity(this.identityScope, 0, now), now, ar -> { + this.idService.invalidateTokensAsync(input.toHashedDiiIdentity(this.identityScope), now, ar -> { if (ar.succeeded()) { rc.response().end("OK"); } else { @@ -1134,7 +1134,7 @@ private Future handleLogoutAsyncV2(RoutingContext rc) { final Instant now = Instant.now(); Promise promise = Promise.promise(); - this.idService.invalidateTokensAsync(input.toHashedDiiIdentity(this.identityScope, 0, now), now, ar -> { + this.idService.invalidateTokensAsync(input.toHashedDiiIdentity(this.identityScope), now, ar -> { if (ar.succeeded()) { JsonObject body = new JsonObject(); body.put("optout", "OK"); @@ -1156,7 +1156,7 @@ private void handleOptOutGet(RoutingContext rc) { if (input.isValid()) { try { final Instant now = Instant.now(); - final HashedDiiIdentity hashedDiiIdentity = input.toHashedDiiIdentity(this.identityScope, 0, now); + final HashedDiiIdentity hashedDiiIdentity = input.toHashedDiiIdentity(this.identityScope); final Instant result = this.idService.getLatestOptoutEntry(hashedDiiIdentity, now); long timestamp = result == null ? -1 : result.getEpochSecond(); rc.response().setStatusCode(200) @@ -1241,7 +1241,7 @@ private void handleIdentityMapV1(RoutingContext rc) { } try { final Instant now = Instant.now(); - final RawUidResponse rawUidResponse = this.idService.map(input.toHashedDiiIdentity(this.identityScope, 0, now), now); + final RawUidResponse rawUidResponse = this.idService.map(input.toHashedDiiIdentity(this.identityScope), now); final JsonObject jsonObject = new JsonObject(); jsonObject.put("identifier", input.getProvided()); jsonObject.put("advertising_id", EncodingUtils.toBase64String(rawUidResponse.rawUid)); @@ -1264,7 +1264,7 @@ else if (!input.isValid()) { } else { final Instant now = Instant.now(); - final RawUidResponse rawUidResponse = this.idService.map(input.toHashedDiiIdentity(this.identityScope, 0, now), now); + final RawUidResponse rawUidResponse = this.idService.map(input.toHashedDiiIdentity(this.identityScope), now); rc.response().end(EncodingUtils.toBase64String(rawUidResponse.rawUid)); } } catch (Exception ex) { @@ -1462,7 +1462,7 @@ private JsonObject handleIdentityMapCommon(RoutingContext rc, InputUtil.InputVal if (input != null && input.isValid()) { final RawUidResponse rawUidResponse = idService.mapIdentity( new MapRequest( - input.toHashedDiiIdentity(this.identityScope, 0, now), + input.toHashedDiiIdentity(this.identityScope), OptoutCheckPolicy.respectOptOut(), now)); diff --git a/src/test/java/com/uid2/operator/TokenEncodingTest.java b/src/test/java/com/uid2/operator/TokenEncodingTest.java index 656d50f98..b2255e1d0 100644 --- a/src/test/java/com/uid2/operator/TokenEncodingTest.java +++ b/src/test/java/com/uid2/operator/TokenEncodingTest.java @@ -101,7 +101,9 @@ public void testAdvertisingTokenEncodings(TokenVersion tokenVersion) { now.plusSeconds(60), new OperatorIdentity(101, OperatorType.Service, 102, 103), new SourcePublisher(111, 112, 113), - new RawUidIdentity(IdentityScope.UID2, IdentityType.Email, rawUid, 121, now) + new RawUidIdentity(IdentityScope.UID2, IdentityType.Email, rawUid), + 121, + now ); final byte[] encodedBytes = encoder.encodeIntoAdvertisingToken(adTokenInput, now); @@ -111,8 +113,8 @@ public void testAdvertisingTokenEncodings(TokenVersion tokenVersion) { assertEquals(adTokenInput.createdAt, decoded.createdAt); assertEquals(adTokenInput.expiresAt, decoded.expiresAt); assertTrue(adTokenInput.rawUidIdentity.matches(decoded.rawUidIdentity)); - assertEquals(adTokenInput.rawUidIdentity.privacyBits, decoded.rawUidIdentity.privacyBits); - assertEquals(adTokenInput.rawUidIdentity.establishedAt, decoded.rawUidIdentity.establishedAt); + assertEquals(adTokenInput.privacyBits, decoded.privacyBits); + assertEquals(adTokenInput.establishedAt, decoded.establishedAt); assertEquals(adTokenInput.sourcePublisher.siteId, decoded.sourcePublisher.siteId); Buffer b = Buffer.buffer(encodedBytes); diff --git a/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java b/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java index efc7c0578..5c3b23433 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java @@ -127,9 +127,7 @@ private HashedDiiIdentity createHashedDiiIdentity(String rawIdentityHash, Identi return new HashedDiiIdentity( scope, type, - rawIdentityHash.getBytes(StandardCharsets.UTF_8), - 0, - this.now.minusSeconds(234) + rawIdentityHash.getBytes(StandardCharsets.UTF_8) ); } @@ -139,11 +137,10 @@ private AdvertisingTokenInput validateAndGetToken(EncryptedTokenEncoder tokenEnc return tokenEncoder.decodeAdvertisingToken(advertisingTokenString); } - private void assertIdentityScopeIdentityTypeAndEstablishedAt(UserIdentity expctedValues, - UserIdentity actualValues) { + private void assertIdentityScopeIdentityType(UserIdentity expctedValues, + UserIdentity actualValues) { assertEquals(expctedValues.identityScope, actualValues.identityScope); assertEquals(expctedValues.identityType, actualValues.identityType); - assertEquals(expctedValues.establishedAt, actualValues.establishedAt); } @ParameterizedTest @@ -152,7 +149,8 @@ public void testGenerateAndRefresh(int siteId, TokenVersion tokenVersion) { final IdentityRequest identityRequest = new IdentityRequest( new SourcePublisher(siteId, 124, 125), createHashedDiiIdentity("test-email-hash", IdentityScope.UID2, IdentityType.Email), - OptoutCheckPolicy.DoNotRespect + OptoutCheckPolicy.DoNotRespect, 0, + this.now.minusSeconds(234) ); final IdentityResponse identityResponse = uid2Service.generateIdentity(identityRequest); verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); @@ -162,13 +160,15 @@ public void testGenerateAndRefresh(int siteId, TokenVersion tokenVersion) { UIDOperatorVerticleTest.validateAdvertisingToken(identityResponse.getAdvertisingToken(), tokenVersion, IdentityScope.UID2, IdentityType.Email); AdvertisingTokenInput advertisingTokenInput = tokenEncoder.decodeAdvertisingToken(identityResponse.getAdvertisingToken());assertEquals(this.now.plusSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS), advertisingTokenInput.expiresAt); assertEquals(identityRequest.sourcePublisher.siteId, advertisingTokenInput.sourcePublisher.siteId); - assertIdentityScopeIdentityTypeAndEstablishedAt(identityRequest.hashedDiiIdentity, advertisingTokenInput.rawUidIdentity); + assertIdentityScopeIdentityType(identityRequest.hashedDiiIdentity, advertisingTokenInput.rawUidIdentity); + assertEquals(identityRequest.establishedAt, advertisingTokenInput.establishedAt); RefreshTokenInput refreshTokenInput = tokenEncoder.decodeRefreshToken(identityResponse.getRefreshToken()); assertEquals(this.now, refreshTokenInput.createdAt); assertEquals(this.now.plusSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), refreshTokenInput.expiresAt); assertEquals(identityRequest.sourcePublisher.siteId, refreshTokenInput.sourcePublisher.siteId); - assertIdentityScopeIdentityTypeAndEstablishedAt(identityRequest.hashedDiiIdentity, refreshTokenInput.firstLevelHashIdentity); + assertIdentityScopeIdentityType(identityRequest.hashedDiiIdentity, refreshTokenInput.firstLevelHashIdentity); + assertEquals(identityRequest.establishedAt, refreshTokenInput.firstLevelHashIdentity.establishedAt); setNow(Instant.now().plusSeconds(200)); @@ -184,8 +184,9 @@ public void testGenerateAndRefresh(int siteId, TokenVersion tokenVersion) { AdvertisingTokenInput advertisingTokenInput2 = tokenEncoder.decodeAdvertisingToken(refreshResponse.getIdentityResponse().getAdvertisingToken()); assertEquals(this.now.plusSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS), advertisingTokenInput2.expiresAt); assertEquals(advertisingTokenInput.sourcePublisher.siteId, advertisingTokenInput2.sourcePublisher.siteId); - assertIdentityScopeIdentityTypeAndEstablishedAt(advertisingTokenInput.rawUidIdentity, + assertIdentityScopeIdentityType(advertisingTokenInput.rawUidIdentity, advertisingTokenInput2.rawUidIdentity); + assertEquals(advertisingTokenInput.establishedAt, advertisingTokenInput2.establishedAt); assertArrayEquals(advertisingTokenInput.rawUidIdentity.rawUid, advertisingTokenInput2.rawUidIdentity.rawUid); @@ -193,7 +194,8 @@ public void testGenerateAndRefresh(int siteId, TokenVersion tokenVersion) { assertEquals(this.now, refreshTokenInput2.createdAt); assertEquals(this.now.plusSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), refreshTokenInput2.expiresAt); assertEquals(refreshTokenInput.sourcePublisher.siteId, refreshTokenInput2.sourcePublisher.siteId); - assertIdentityScopeIdentityTypeAndEstablishedAt(refreshTokenInput.firstLevelHashIdentity, refreshTokenInput2.firstLevelHashIdentity); + assertIdentityScopeIdentityType(refreshTokenInput.firstLevelHashIdentity, refreshTokenInput2.firstLevelHashIdentity); + assertEquals(refreshTokenInput.firstLevelHashIdentity.establishedAt, refreshTokenInput2.firstLevelHashIdentity.establishedAt); assertArrayEquals(refreshTokenInput.firstLevelHashIdentity.firstLevelHash, refreshTokenInput2.firstLevelHashIdentity.firstLevelHash); } @@ -203,8 +205,8 @@ public void testTestOptOutKey_DoNotRespectOptout() { final IdentityRequest identityRequest = new IdentityRequest( new SourcePublisher(123, 124, 125), - inputVal.toHashedDiiIdentity(IdentityScope.UID2, 0, this.now), - OptoutCheckPolicy.DoNotRespect + inputVal.toHashedDiiIdentity(IdentityScope.UID2), + OptoutCheckPolicy.DoNotRespect, 0, this.now ); final IdentityResponse identityResponse = uid2Service.generateIdentity(identityRequest); verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); @@ -222,8 +224,8 @@ public void testTestOptOutKey_RespectOptout() { final IdentityRequest identityRequest = new IdentityRequest( new SourcePublisher(123, 124, 125), - inputVal.toHashedDiiIdentity(IdentityScope.UID2, 0, this.now), - OptoutCheckPolicy.RespectOptOut + inputVal.toHashedDiiIdentity(IdentityScope.UID2), + OptoutCheckPolicy.RespectOptOut, 0, this.now ); final IdentityResponse identityResponse = uid2Service.generateIdentity(identityRequest); assertTrue(identityResponse.isOptedOut()); @@ -238,8 +240,8 @@ public void testTestOptOutKeyIdentityScopeMismatch() { final IdentityRequest identityRequest = new IdentityRequest( new SourcePublisher(123, 124, 125), - inputVal.toHashedDiiIdentity(IdentityScope.EUID, 0, this.now), - OptoutCheckPolicy.DoNotRespect + inputVal.toHashedDiiIdentity(IdentityScope.EUID), + OptoutCheckPolicy.DoNotRespect, 0, this.now ); final IdentityResponse identityResponse = euidService.generateIdentity(identityRequest); verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); @@ -264,12 +266,14 @@ public void testGenerateTokenForOptOutUser(IdentityType type, String id, Identit final IdentityRequest identityRequestForceGenerate = new IdentityRequest( new SourcePublisher(123, 124, 125), hashedDiiIdentity, - OptoutCheckPolicy.DoNotRespect); + OptoutCheckPolicy.DoNotRespect,0, + this.now.minusSeconds(234)); final IdentityRequest identityRequestRespectOptOut = new IdentityRequest( new SourcePublisher(123, 124, 125), hashedDiiIdentity, - OptoutCheckPolicy.RespectOptOut); + OptoutCheckPolicy.RespectOptOut, 0, + this.now.minusSeconds(234)); // the clock value shouldn't matter here when(optOutStore.getLatestEntry(any(FirstLevelHashIdentity.class))) @@ -394,8 +398,8 @@ void testSpecialIdentityOptOutTokenGenerate(TestIdentityInputType type, String i final IdentityRequest identityRequest = new IdentityRequest( new SourcePublisher(123, 124, 125), - inputVal.toHashedDiiIdentity(scope, 0, this.now), - OptoutCheckPolicy.RespectOptOut + inputVal.toHashedDiiIdentity(scope), + OptoutCheckPolicy.RespectOptOut, 0, this.now ); // identity has no optout record, ensure generate still returns optout @@ -426,7 +430,8 @@ void testSpecialIdentityOptOutIdentityMap(TestIdentityInputType type, String id, InputUtil.InputVal inputVal = generateInputVal(type, id); final MapRequest mapRequestRespectOptOut = new MapRequest( - inputVal.toHashedDiiIdentity(scope, 0, this.now), + inputVal.toHashedDiiIdentity(scope), +// inputVal.toHashedDiiIdentity(scope, 0, this.now), OptoutCheckPolicy.RespectOptOut, now); @@ -460,8 +465,8 @@ void testSpecialIdentityOptOutTokenRefresh(TestIdentityInputType type, String id final IdentityRequest identityRequest = new IdentityRequest( new SourcePublisher(123, 124, 125), - inputVal.toHashedDiiIdentity(scope, 0, this.now), - OptoutCheckPolicy.DoNotRespect + inputVal.toHashedDiiIdentity(scope), + OptoutCheckPolicy.DoNotRespect, 0, this.now ); IdentityResponse identityResponse; @@ -499,8 +504,8 @@ void testSpecialIdentityRefreshOptOutGenerate(TestIdentityInputType type, String final IdentityRequest identityRequest = new IdentityRequest( new SourcePublisher(123, 124, 125), - inputVal.toHashedDiiIdentity(scope, 0, this.now), - OptoutCheckPolicy.RespectOptOut + inputVal.toHashedDiiIdentity(scope), + OptoutCheckPolicy.RespectOptOut, 0, this.now ); // identity has optout record, ensure still generates @@ -540,7 +545,7 @@ void testSpecialIdentityRefreshOptOutIdentityMap(TestIdentityInputType type, Str InputUtil.InputVal inputVal = generateInputVal(type, id); final MapRequest mapRequestRespectOptOut = new MapRequest( - inputVal.toHashedDiiIdentity(scope, 0, this.now), + inputVal.toHashedDiiIdentity(scope), OptoutCheckPolicy.RespectOptOut, now); @@ -574,8 +579,8 @@ void testSpecialIdentityValidateGenerate(TestIdentityInputType type, String id, final IdentityRequest identityRequest = new IdentityRequest( new SourcePublisher(123, 124, 125), - inputVal.toHashedDiiIdentity(scope, 0, this.now), - OptoutCheckPolicy.RespectOptOut + inputVal.toHashedDiiIdentity(scope), + OptoutCheckPolicy.RespectOptOut, 0, this.now ); // all identities have optout records, ensure validate identities still get generated @@ -611,7 +616,7 @@ void testSpecialIdentityValidateIdentityMap(TestIdentityInputType type, String i InputUtil.InputVal inputVal = generateInputVal(type, id); final MapRequest mapRequestRespectOptOut = new MapRequest( - inputVal.toHashedDiiIdentity(scope, 0, this.now), + inputVal.toHashedDiiIdentity(scope), OptoutCheckPolicy.RespectOptOut, now); @@ -642,8 +647,8 @@ void testNormalIdentityOptIn(TestIdentityInputType type, String id, IdentityScop InputUtil.InputVal inputVal = generateInputVal(type, id); final IdentityRequest identityRequest = new IdentityRequest( new SourcePublisher(123, 124, 125), - inputVal.toHashedDiiIdentity(scope, 0, this.now), - OptoutCheckPolicy.DoNotRespect + inputVal.toHashedDiiIdentity(scope), + OptoutCheckPolicy.DoNotRespect, 1, Instant.now() ); IdentityResponse identityResponse; if(scope == IdentityScope.EUID) { @@ -703,8 +708,8 @@ void testExpiredSaltsNotifiesShutdownHandler(TestIdentityInputType type, String final IdentityRequest identityRequest = new IdentityRequest( new SourcePublisher(123, 124, 125), - inputVal.toHashedDiiIdentity(scope, 0, this.now), - OptoutCheckPolicy.RespectOptOut); + inputVal.toHashedDiiIdentity(scope), + OptoutCheckPolicy.RespectOptOut, 0, this.now); IdentityResponse identityResponse; AdvertisingTokenInput advertisingTokenInput; @@ -733,7 +738,7 @@ void testExpiredSaltsNotifiesShutdownHandler(TestIdentityInputType type, String assertNotEquals(RefreshResponse.Optout, refreshResponse); final MapRequest mapRequest = new MapRequest( - inputVal.toHashedDiiIdentity(scope, 0, this.now), + inputVal.toHashedDiiIdentity(scope), OptoutCheckPolicy.RespectOptOut, now); final RawUidResponse rawUidResponse; diff --git a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java index ea8385779..1e09c6ea7 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java @@ -1180,10 +1180,10 @@ void tokenGenerateOptOutToken(String policyParameterKey, String identity, Identi String advertisingTokenString = body.getString("advertising_token"); final Instant now = Instant.now(); final String token = advertisingTokenString; - final boolean matchedOptedOutIdentity = this.uidOperatorVerticle.getIdService().advertisingTokenMatches(token, optOutTokenInput.toHashedDiiIdentity(getIdentityScope(), 0, now), now); + final boolean matchedOptedOutIdentity = this.uidOperatorVerticle.getIdService().advertisingTokenMatches(token, optOutTokenInput.toHashedDiiIdentity(getIdentityScope()), now); assertTrue(matchedOptedOutIdentity); - assertFalse(PrivacyBits.fromInt(advertisingTokenInput.rawUidIdentity.privacyBits).isClientSideTokenGenerated()); - assertTrue(PrivacyBits.fromInt(advertisingTokenInput.rawUidIdentity.privacyBits).isClientSideTokenOptedOut()); + assertFalse(PrivacyBits.fromInt(advertisingTokenInput.privacyBits).isClientSideTokenGenerated()); + assertTrue(PrivacyBits.fromInt(advertisingTokenInput.privacyBits).isClientSideTokenOptedOut()); assertTokenStatusMetrics( 201, @@ -1229,8 +1229,8 @@ void tokenGenerateForEmail(String apiVersion, Vertx vertx, VertxTestContext test AdvertisingTokenInput advertisingTokenInput = validateAndGetToken(encoder, body, IdentityType.Email); - assertFalse(PrivacyBits.fromInt(advertisingTokenInput.rawUidIdentity.privacyBits).isClientSideTokenGenerated()); - assertFalse(PrivacyBits.fromInt(advertisingTokenInput.rawUidIdentity.privacyBits).isClientSideTokenOptedOut()); + assertFalse(PrivacyBits.fromInt(advertisingTokenInput.privacyBits).isClientSideTokenGenerated()); + assertFalse(PrivacyBits.fromInt(advertisingTokenInput.privacyBits).isClientSideTokenOptedOut()); assertEquals(clientSiteId, advertisingTokenInput.sourcePublisher.siteId); assertArrayEquals(getRawUidFromIdentity(IdentityType.Email, emailAddress, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenInput.rawUidIdentity.rawUid); @@ -1271,8 +1271,8 @@ void tokenGenerateForEmailHash(String apiVersion, Vertx vertx, VertxTestContext AdvertisingTokenInput advertisingTokenInput = validateAndGetToken(encoder, body, IdentityType.Email); - assertFalse(PrivacyBits.fromInt(advertisingTokenInput.rawUidIdentity.privacyBits).isClientSideTokenGenerated()); - assertFalse(PrivacyBits.fromInt(advertisingTokenInput.rawUidIdentity.privacyBits).isClientSideTokenOptedOut()); + assertFalse(PrivacyBits.fromInt(advertisingTokenInput.privacyBits).isClientSideTokenGenerated()); + assertFalse(PrivacyBits.fromInt(advertisingTokenInput.privacyBits).isClientSideTokenOptedOut()); assertEquals(clientSiteId, advertisingTokenInput.sourcePublisher.siteId); assertArrayEquals(getRawUidFromIdentityHash(IdentityType.Email, emailHash, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenInput.rawUidIdentity.rawUid); @@ -1315,8 +1315,8 @@ void tokenGenerateThenRefresh(String apiVersion, Vertx vertx, VertxTestContext t AdvertisingTokenInput advertisingTokenInput = validateAndGetToken(encoder, refreshBody, IdentityType.Email); - assertFalse(PrivacyBits.fromInt(advertisingTokenInput.rawUidIdentity.privacyBits).isClientSideTokenGenerated()); - assertFalse(PrivacyBits.fromInt(advertisingTokenInput.rawUidIdentity.privacyBits).isClientSideTokenOptedOut()); + assertFalse(PrivacyBits.fromInt(advertisingTokenInput.privacyBits).isClientSideTokenGenerated()); + assertFalse(PrivacyBits.fromInt(advertisingTokenInput.privacyBits).isClientSideTokenOptedOut()); assertEquals(clientSiteId, advertisingTokenInput.sourcePublisher.siteId); assertArrayEquals(getRawUidFromIdentity(IdentityType.Email, emailAddress, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenInput.rawUidIdentity.rawUid); @@ -1374,8 +1374,8 @@ void tokenGenerateThenRefreshSaltsExpired(String apiVersion, Vertx vertx, VertxT AdvertisingTokenInput advertisingTokenInput = validateAndGetToken(encoder, refreshBody, IdentityType.Email); - assertFalse(PrivacyBits.fromInt(advertisingTokenInput.rawUidIdentity.privacyBits).isClientSideTokenGenerated()); - assertFalse(PrivacyBits.fromInt(advertisingTokenInput.rawUidIdentity.privacyBits).isClientSideTokenOptedOut()); + assertFalse(PrivacyBits.fromInt(advertisingTokenInput.privacyBits).isClientSideTokenGenerated()); + assertFalse(PrivacyBits.fromInt(advertisingTokenInput.privacyBits).isClientSideTokenOptedOut()); assertEquals(clientSiteId, advertisingTokenInput.sourcePublisher.siteId); assertArrayEquals(getRawUidFromIdentity(IdentityType.Email, emailAddress, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenInput.rawUidIdentity.rawUid); @@ -1586,8 +1586,8 @@ void tokenGenerateSaltsExpired(String apiVersion, Vertx vertx, VertxTestContext AdvertisingTokenInput advertisingTokenInput = validateAndGetToken(encoder, body, IdentityType.Email); - assertFalse(PrivacyBits.fromInt(advertisingTokenInput.rawUidIdentity.privacyBits).isClientSideTokenGenerated()); - assertFalse(PrivacyBits.fromInt(advertisingTokenInput.rawUidIdentity.privacyBits).isClientSideTokenOptedOut()); + assertFalse(PrivacyBits.fromInt(advertisingTokenInput.privacyBits).isClientSideTokenGenerated()); + assertFalse(PrivacyBits.fromInt(advertisingTokenInput.privacyBits).isClientSideTokenOptedOut()); assertEquals(clientSiteId, advertisingTokenInput.sourcePublisher.siteId); assertArrayEquals(getRawUidFromIdentity(IdentityType.Email, emailAddress, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenInput.rawUidIdentity.rawUid); @@ -2498,8 +2498,8 @@ void tokenGenerateForPhone(String apiVersion, Vertx vertx, VertxTestContext test AdvertisingTokenInput advertisingTokenInput = validateAndGetToken(encoder, body, IdentityType.Phone); - assertFalse(PrivacyBits.fromInt(advertisingTokenInput.rawUidIdentity.privacyBits).isClientSideTokenGenerated()); - assertFalse(PrivacyBits.fromInt(advertisingTokenInput.rawUidIdentity.privacyBits).isClientSideTokenOptedOut()); + assertFalse(PrivacyBits.fromInt(advertisingTokenInput.privacyBits).isClientSideTokenGenerated()); + assertFalse(PrivacyBits.fromInt(advertisingTokenInput.privacyBits).isClientSideTokenOptedOut()); assertEquals(clientSiteId, advertisingTokenInput.sourcePublisher.siteId); assertArrayEquals(getRawUidFromIdentity(IdentityType.Phone, phone, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenInput.rawUidIdentity.rawUid); @@ -2537,8 +2537,8 @@ void tokenGenerateForPhoneHash(String apiVersion, Vertx vertx, VertxTestContext AdvertisingTokenInput advertisingTokenInput = validateAndGetToken(encoder, body, IdentityType.Phone); - assertFalse(PrivacyBits.fromInt(advertisingTokenInput.rawUidIdentity.privacyBits).isClientSideTokenGenerated()); - assertFalse(PrivacyBits.fromInt(advertisingTokenInput.rawUidIdentity.privacyBits).isClientSideTokenOptedOut()); + assertFalse(PrivacyBits.fromInt(advertisingTokenInput.privacyBits).isClientSideTokenGenerated()); + assertFalse(PrivacyBits.fromInt(advertisingTokenInput.privacyBits).isClientSideTokenOptedOut()); assertEquals(clientSiteId, advertisingTokenInput.sourcePublisher.siteId); assertArrayEquals(getRawUidFromIdentity(IdentityType.Phone, phone, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenInput.rawUidIdentity.rawUid); @@ -2581,8 +2581,8 @@ void tokenGenerateThenRefreshForPhone(String apiVersion, Vertx vertx, VertxTestC AdvertisingTokenInput advertisingTokenInput = validateAndGetToken(encoder, refreshBody, IdentityType.Phone); - assertFalse(PrivacyBits.fromInt(advertisingTokenInput.rawUidIdentity.privacyBits).isClientSideTokenGenerated()); - assertFalse(PrivacyBits.fromInt(advertisingTokenInput.rawUidIdentity.privacyBits).isClientSideTokenOptedOut()); + assertFalse(PrivacyBits.fromInt(advertisingTokenInput.privacyBits).isClientSideTokenGenerated()); + assertFalse(PrivacyBits.fromInt(advertisingTokenInput.privacyBits).isClientSideTokenOptedOut()); assertEquals(clientSiteId, advertisingTokenInput.sourcePublisher.siteId); assertArrayEquals(getRawUidFromIdentity(IdentityType.Phone, phone, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenInput.rawUidIdentity.rawUid); @@ -4010,7 +4010,7 @@ void cstgUserOptsOutAfterTokenGenerate(String id, IdentityType identityType, Ver // When we refresh the token the user has opted out. when(optOutStore.getLatestEntry(any(FirstLevelHashIdentity.class))) - .thenReturn(advertisingTokenInput.rawUidIdentity.establishedAt.plusSeconds(1)); + .thenReturn(advertisingTokenInput.establishedAt.plusSeconds(1)); sendTokenRefresh("v2", vertx, testContext, genBody.getString("refresh_token"), genBody.getString("refresh_response_key"), 200, refreshRespJson -> { assertEquals("optout", refreshRespJson.getString("status")); @@ -4202,7 +4202,7 @@ private void assertAreClientSideGeneratedTokens(AdvertisingTokenInput advertisin } private void assertAreClientSideGeneratedTokens(AdvertisingTokenInput advertisingTokenInput, RefreshTokenInput refreshTokenInput, int siteId, IdentityType identityType, String identity, boolean expectedOptOut) { - final PrivacyBits advertisingTokenPrivacyBits = PrivacyBits.fromInt(advertisingTokenInput.rawUidIdentity.privacyBits); + final PrivacyBits advertisingTokenPrivacyBits = PrivacyBits.fromInt(advertisingTokenInput.privacyBits); final PrivacyBits refreshTokenPrivacyBits = PrivacyBits.fromInt(refreshTokenInput.firstLevelHashIdentity.privacyBits); final byte[] rawUid = getRawUidFromIdentity(identityType, diff --git a/src/test/java/com/uid2/operator/benchmark/BenchmarkCommon.java b/src/test/java/com/uid2/operator/benchmark/BenchmarkCommon.java index 58f555818..1c6153d22 100644 --- a/src/test/java/com/uid2/operator/benchmark/BenchmarkCommon.java +++ b/src/test/java/com/uid2/operator/benchmark/BenchmarkCommon.java @@ -155,8 +155,7 @@ static HashedDiiIdentity[] createHashedDiiIdentities() { for (int i = 0; i < 65536; i++) { final byte[] diiHash = new byte[33]; new Random().nextBytes(diiHash); - arr[i] = new HashedDiiIdentity(IdentityScope.UID2, IdentityType.Email, diiHash, 0, - Instant.now().minusSeconds(120)); + arr[i] = new HashedDiiIdentity(IdentityScope.UID2, IdentityType.Email, diiHash); } return arr; } diff --git a/src/test/java/com/uid2/operator/benchmark/TokenEndecBenchmark.java b/src/test/java/com/uid2/operator/benchmark/TokenEndecBenchmark.java index e907f638c..bc8fddb27 100644 --- a/src/test/java/com/uid2/operator/benchmark/TokenEndecBenchmark.java +++ b/src/test/java/com/uid2/operator/benchmark/TokenEndecBenchmark.java @@ -8,6 +8,7 @@ import org.openjdk.jmh.annotations.BenchmarkMode; import org.openjdk.jmh.annotations.Mode; +import java.time.Instant; import java.util.ArrayList; import java.util.List; @@ -42,7 +43,7 @@ static IdentityResponse[] createAdvertisingTokens() { uidService.generateIdentity(new IdentityRequest( publisher, hashedDiiIdentities[i], - OptoutCheckPolicy.DoNotRespect))); + OptoutCheckPolicy.DoNotRespect, 1, Instant.now()))); } return tokens.toArray(new IdentityResponse[tokens.size()]); } @@ -53,7 +54,7 @@ public IdentityResponse TokenGenerationBenchmark() { return uidService.generateIdentity(new IdentityRequest( publisher, hashedDiiIdentities[(idx++) & 65535], - OptoutCheckPolicy.DoNotRespect)); + OptoutCheckPolicy.DoNotRespect, 1, Instant.now())); } @Benchmark From 7538bfbc86394d78a5233bb388c7a9dd440b5704 Mon Sep 17 00:00:00 2001 From: Sunny Wu Date: Wed, 16 Oct 2024 19:08:48 +1100 Subject: [PATCH 064/431] added comments --- .../java/com/uid2/operator/service/EncryptedTokenEncoder.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java b/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java index 67fac4d1d..f3e22c5b5 100644 --- a/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java +++ b/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java @@ -55,6 +55,7 @@ private byte[] encodeIntoAdvertisingTokenV3(AdvertisingTokenInput t, KeysetKey m encodePublisherRequesterV3(sitePayload, t.sourcePublisher); sitePayload.appendInt(t.privacyBits); sitePayload.appendLong(t.establishedAt.toEpochMilli()); + // this is the refreshedAt field in the spec - but effectively it is the time this advertising token is generated sitePayload.appendLong(t.createdAt.toEpochMilli()); sitePayload.appendBytes(t.rawUidIdentity.rawUid); // 32 or 33 bytes @@ -253,6 +254,7 @@ public AdvertisingTokenInput decodeAdvertisingTokenV3orV4(Buffer b, byte[] bytes final SourcePublisher sourcePublisher = decodeSourcePublisherV3(sitePayload, 0); final int privacyBits = sitePayload.getInt(16); final Instant establishedAt = Instant.ofEpochMilli(sitePayload.getLong(20)); + // refreshedAt is currently not used final Instant refreshedAt = Instant.ofEpochMilli(sitePayload.getLong(28)); final byte[] rawUid = sitePayload.slice(36, sitePayload.length()).getBytes(); final IdentityScope identityScope = rawUid.length == 32 ? IdentityScope.UID2 : decodeIdentityScopeV3(rawUid[0]); From 249c25dedd2f81414503ac56a19c6dd1039e2b1e Mon Sep 17 00:00:00 2001 From: Sunny Wu Date: Wed, 16 Oct 2024 19:23:42 +1100 Subject: [PATCH 065/431] fixed variable name --- .../java/com/uid2/operator/service/UIDOperatorService.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/uid2/operator/service/UIDOperatorService.java b/src/main/java/com/uid2/operator/service/UIDOperatorService.java index eed0c0bb8..2b4288151 100644 --- a/src/main/java/com/uid2/operator/service/UIDOperatorService.java +++ b/src/main/java/com/uid2/operator/service/UIDOperatorService.java @@ -193,10 +193,10 @@ private ISaltProvider.ISaltSnapshot getSaltProviderSnapshot(Instant asOf) { @Override public void invalidateTokensAsync(HashedDiiIdentity diiIdentity, Instant asOf, Handler> handler) { - final FirstLevelHashIdentity hashedDiiIdentity = getFirstLevelHashIdentity(diiIdentity, asOf); - final RawUidResponse rawUidResponse = generateRawUid(hashedDiiIdentity, asOf); + final FirstLevelHashIdentity firstLevelHashIdentity = getFirstLevelHashIdentity(diiIdentity, asOf); + final RawUidResponse rawUidResponse = generateRawUid(firstLevelHashIdentity, asOf); - this.optOutStore.addEntry(hashedDiiIdentity, rawUidResponse.rawUid, r -> { + this.optOutStore.addEntry(firstLevelHashIdentity, rawUidResponse.rawUid, r -> { if (r.succeeded()) { handler.handle(Future.succeededFuture(r.result())); } else { From a89044648d9d3121ff7dff274c7fd9f352b1de47 Mon Sep 17 00:00:00 2001 From: asloob qureshi Date: Wed, 16 Oct 2024 10:19:31 -0700 Subject: [PATCH 066/431] Changed assertion to warn --- .../java/com/uid2/operator/store/CloudSyncOptOutStore.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/com/uid2/operator/store/CloudSyncOptOutStore.java b/src/main/java/com/uid2/operator/store/CloudSyncOptOutStore.java index 8cd54ad28..663abb821 100644 --- a/src/main/java/com/uid2/operator/store/CloudSyncOptOutStore.java +++ b/src/main/java/com/uid2/operator/store/CloudSyncOptOutStore.java @@ -612,8 +612,7 @@ private OptOutStoreSnapshot updateIndexInternal(IndexUpdateContext iuc) { } else if (numPartitions > 1) { if (this.iteration != 0) { final String errorMsg = "Should not load more than 1 partition at a time, unless during service bootstrap. Current iteration " + this.iteration; - LOGGER.error(errorMsg); - throw new IllegalStateException(errorMsg); + LOGGER.warn(errorMsg); } return this.processPartitions(iuc); } else { From dff8dae70d53ea81fecf7a216f4f93774dec7d70 Mon Sep 17 00:00:00 2001 From: Ian Nara Date: Wed, 16 Oct 2024 15:18:33 -0600 Subject: [PATCH 067/431] remove sleep and reset version --- pom.xml | 2 +- scripts/aws/eks-pod/entrypoint.sh | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 04a610078..8a1c418f7 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.40.56-alpha-34-SNAPSHOT + 5.40.48 UTF-8 diff --git a/scripts/aws/eks-pod/entrypoint.sh b/scripts/aws/eks-pod/entrypoint.sh index 11f643c9b..a28ebfe60 100644 --- a/scripts/aws/eks-pod/entrypoint.sh +++ b/scripts/aws/eks-pod/entrypoint.sh @@ -16,7 +16,6 @@ function terminate_old_enclave() { else nitro-cli describe-enclaves echo "No running enclaves to terminate." - sleep 30 fi } From 29ae2c50615dd179a6fe151d2357bd76ff15abe6 Mon Sep 17 00:00:00 2001 From: Ian Nara Date: Thu, 17 Oct 2024 13:19:43 -0600 Subject: [PATCH 068/431] safe debugging enclave --- scripts/aws/eks-pod/entrypoint.sh | 3 +-- scripts/aws/eks-pod/sockd_eks.conf | 8 ++++---- scripts/aws/entrypoint.sh | 5 ++--- scripts/aws/proxies.host.yaml | 5 ----- scripts/aws/proxies.nitro.yaml | 5 ----- scripts/aws/sockd.conf | 7 ++----- 6 files changed, 9 insertions(+), 24 deletions(-) diff --git a/scripts/aws/eks-pod/entrypoint.sh b/scripts/aws/eks-pod/entrypoint.sh index a28ebfe60..c51f57cc9 100644 --- a/scripts/aws/eks-pod/entrypoint.sh +++ b/scripts/aws/eks-pod/entrypoint.sh @@ -1,4 +1,4 @@ -#!/bin/bash -ufx +#!/bin/bash -eufx CID=42 EIF_PATH=/home/uid2operator.eif MEMORY_MB=24576 @@ -121,7 +121,6 @@ wait_for_config update_config run_enclave -nitro-cli describe-enclaves sleep 60s set +x ENCLAVE_ID=$(nitro-cli describe-enclaves | jq -r ".[0].EnclaveID") diff --git a/scripts/aws/eks-pod/sockd_eks.conf b/scripts/aws/eks-pod/sockd_eks.conf index f6b483958..0e588fb7c 100644 --- a/scripts/aws/eks-pod/sockd_eks.conf +++ b/scripts/aws/eks-pod/sockd_eks.conf @@ -1,6 +1,6 @@ -logoutput: stdout +#logoutput: stdout errorlog: stdout -debug: 2 +#debug: 2 internal: 127.0.0.1 port = 3306 external: eth0 user.notprivileged: ec2-user @@ -9,12 +9,12 @@ socksmethod: none client pass { from: 127.0.0.1/32 to: 127.0.0.1/32 - log: connect disconnect tcpinfo # connect disconnect iooperation + log: error # connect disconnect iooperation } socks pass { from: 127.0.0.1/32 to: 0.0.0.0/0 command: bind connect protocol: tcp - log: connect disconnect tcpinfo + log: error } \ No newline at end of file diff --git a/scripts/aws/entrypoint.sh b/scripts/aws/entrypoint.sh index 5ae1df745..e5063b148 100755 --- a/scripts/aws/entrypoint.sh +++ b/scripts/aws/entrypoint.sh @@ -1,4 +1,4 @@ -#!/bin/bash -ufx +#!/bin/bash -eufx # This is the entrypoint for the Enclave. It is executed in all enclaves - EC2 and EKS @@ -16,7 +16,7 @@ ifconfig lo 127.0.0.1 # -- start vsock proxy echo "Starting vsock proxy..." -/app/vsockpx --config /app/proxies.nitro.yaml --daemon --workers $(( $(nproc) * 2 )) --log-level 1 +/app/vsockpx --config /app/proxies.nitro.yaml --daemon --workers $(( $(nproc) * 2 )) --log-level 3 # -- load config from identity service echo "Loading config from identity service via proxy..." @@ -101,7 +101,6 @@ echo "Starting Java application..." if [[ "$DEBUG_MODE" = "true" ]]; then java \ -XX:MaxRAMPercentage=95 -XX:-UseCompressedOops -XX:+PrintFlagsFinal \ - -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000 \ -Djava.security.egd=file:/dev/./urandom \ -Djava.library.path=/app/lib \ -Dvertx-config-path="${FINAL_CONFIG}" \ diff --git a/scripts/aws/proxies.host.yaml b/scripts/aws/proxies.host.yaml index da2e40b57..5a2ae0623 100644 --- a/scripts/aws/proxies.host.yaml +++ b/scripts/aws/proxies.host.yaml @@ -10,11 +10,6 @@ operator-service: listen: tcp://0.0.0.0:80 connect: vsock://42:8080 -operator-debug: - service: direct - listen: tcp://0.0.0.0:8000 - connect: vsock://42:8000 - operator-prometheus: service: direct listen: tcp://0.0.0.0:9080 diff --git a/scripts/aws/proxies.nitro.yaml b/scripts/aws/proxies.nitro.yaml index e9b5e0cfc..0f459b150 100644 --- a/scripts/aws/proxies.nitro.yaml +++ b/scripts/aws/proxies.nitro.yaml @@ -5,11 +5,6 @@ uid-operator-in: listen: vsock://-1:8080 connect: tcp://127.0.0.1:8080 -uid-operator-debug-in: - service: direct - listen: vsock://-1:8000 - connect: tcp://127.0.0.1:8000 - prometheus-server: service: direct listen: vsock://-1:9080 diff --git a/scripts/aws/sockd.conf b/scripts/aws/sockd.conf index d5e039c20..6e8814445 100644 --- a/scripts/aws/sockd.conf +++ b/scripts/aws/sockd.conf @@ -1,6 +1,3 @@ -logoutput: stdout -errorlog: stdout -debug: 2 internal: 127.0.0.1 port = 3306 external: ens5 user.notprivileged: ec2-user @@ -9,12 +6,12 @@ socksmethod: none client pass { from: 127.0.0.1/32 to: 127.0.0.1/32 - log: connect disconnect tcpinfo # connect disconnect iooperation + log: error # connect disconnect iooperation } socks pass { from: 127.0.0.1/32 to: 0.0.0.0/0 command: bind connect protocol: tcp - log: connect disconnect tcpinfo + log: error } \ No newline at end of file From 3c28951f9c6e2f387cf4b8369891714a78847cf8 Mon Sep 17 00:00:00 2001 From: Ian Nara Date: Thu, 17 Oct 2024 13:21:26 -0600 Subject: [PATCH 069/431] safe debugging enclave --- scripts/aws/eks-pod/entrypoint.sh | 3 +-- scripts/aws/eks-pod/proxies.host.yaml | 5 ----- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/scripts/aws/eks-pod/entrypoint.sh b/scripts/aws/eks-pod/entrypoint.sh index c51f57cc9..9ec4ea009 100644 --- a/scripts/aws/eks-pod/entrypoint.sh +++ b/scripts/aws/eks-pod/entrypoint.sh @@ -14,7 +14,6 @@ function terminate_old_enclave() { nitro-cli terminate-enclave --enclave-id ${ENCLAVE_ID} echo "Terminated enclave with ID ${ENCLAVE_ID}" else - nitro-cli describe-enclaves echo "No running enclaves to terminate." fi } @@ -29,7 +28,7 @@ function setup_vsockproxy() { VSOCK_PROXY=${VSOCK_PROXY:-/home/vsockpx} VSOCK_CONFIG=${VSOCK_CONFIG:-/home/proxies.host.yaml} VSOCK_THREADS=${VSOCK_THREADS:-$(( $(nproc) * 2 )) } - VSOCK_LOG_LEVEL=1 + VSOCK_LOG_LEVEL=${VSOCK_LOG_LEVEL:-3} echo "starting vsock proxy at $VSOCK_PROXY with $VSOCK_THREADS worker threads..." $VSOCK_PROXY -c $VSOCK_CONFIG --workers $VSOCK_THREADS --log-level $VSOCK_LOG_LEVEL --daemon echo "vsock proxy now running in background." diff --git a/scripts/aws/eks-pod/proxies.host.yaml b/scripts/aws/eks-pod/proxies.host.yaml index da2e40b57..5a2ae0623 100644 --- a/scripts/aws/eks-pod/proxies.host.yaml +++ b/scripts/aws/eks-pod/proxies.host.yaml @@ -10,11 +10,6 @@ operator-service: listen: tcp://0.0.0.0:80 connect: vsock://42:8080 -operator-debug: - service: direct - listen: tcp://0.0.0.0:8000 - connect: vsock://42:8000 - operator-prometheus: service: direct listen: tcp://0.0.0.0:9080 From e8916e5b73f49f9320c52bd042a0b648f4911d51 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Thu, 17 Oct 2024 19:41:04 +0000 Subject: [PATCH 070/431] [CI Pipeline] Released Snapshot version: 5.40.87-alpha-44-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 082c67876..c9ee335f7 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.40.86 + 5.40.87-alpha-44-SNAPSHOT UTF-8 From 78e61d066089261120a605b35478f6093edc3a4b Mon Sep 17 00:00:00 2001 From: asloob qureshi Date: Thu, 17 Oct 2024 13:31:55 -0700 Subject: [PATCH 071/431] Add comment for warning --- src/main/java/com/uid2/operator/store/CloudSyncOptOutStore.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/uid2/operator/store/CloudSyncOptOutStore.java b/src/main/java/com/uid2/operator/store/CloudSyncOptOutStore.java index 663abb821..f7e3a495d 100644 --- a/src/main/java/com/uid2/operator/store/CloudSyncOptOutStore.java +++ b/src/main/java/com/uid2/operator/store/CloudSyncOptOutStore.java @@ -612,6 +612,7 @@ private OptOutStoreSnapshot updateIndexInternal(IndexUpdateContext iuc) { } else if (numPartitions > 1) { if (this.iteration != 0) { final String errorMsg = "Should not load more than 1 partition at a time, unless during service bootstrap. Current iteration " + this.iteration; + // Leaving this as a warning as this condition is true in production LOGGER.warn(errorMsg); } return this.processPartitions(iuc); From aff43f27dd9b7966b5739e65565f118d883ac45f Mon Sep 17 00:00:00 2001 From: Ian Nara Date: Thu, 17 Oct 2024 15:57:57 -0600 Subject: [PATCH 072/431] fix if --- scripts/aws/eks-pod/entrypoint.sh | 2 +- scripts/aws/entrypoint.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/aws/eks-pod/entrypoint.sh b/scripts/aws/eks-pod/entrypoint.sh index 9ec4ea009..adc8a4634 100644 --- a/scripts/aws/eks-pod/entrypoint.sh +++ b/scripts/aws/eks-pod/entrypoint.sh @@ -95,7 +95,7 @@ function update_config() { } function run_enclave() { - if [ "$DEBUG_MODE" = "true" ]; then + if [ "$DEBUG_MODE" == "true" ]; then echo "starting enclave... --cpu-count $CPU_COUNT --memory $MEMORY_MB --eif-path $EIF_PATH --enclave-cid $CID --debug-mode --attach-console" nitro-cli run-enclave --cpu-count $CPU_COUNT --memory $MEMORY_MB --eif-path $EIF_PATH --enclave-cid $CID --enclave-name uid2-operator --debug-mode --attach-console else diff --git a/scripts/aws/entrypoint.sh b/scripts/aws/entrypoint.sh index e5063b148..0e7d259ce 100755 --- a/scripts/aws/entrypoint.sh +++ b/scripts/aws/entrypoint.sh @@ -39,7 +39,7 @@ done DEBUG_MODE=$(jq -r ".debug_mode" < "${OVERRIDES_CONFIG}") -if [[ ! "$DEBUG_MODE" = "true" ]]; then +if [[ ! "$DEBUG_MODE" == "true" ]]; then # -- setup syslog-ng echo "Starting syslog-ng..." /usr/sbin/syslog-ng --verbose @@ -98,7 +98,7 @@ cd /app # -- start operator echo "Starting Java application..." -if [[ "$DEBUG_MODE" = "true" ]]; then +if [[ "$DEBUG_MODE" == "true" ]]; then java \ -XX:MaxRAMPercentage=95 -XX:-UseCompressedOops -XX:+PrintFlagsFinal \ -Djava.security.egd=file:/dev/./urandom \ From 3ce1b215b6ab8bff55b6487e44cbf1099f990011 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Thu, 17 Oct 2024 22:01:07 +0000 Subject: [PATCH 073/431] [CI Pipeline] Released Snapshot version: 5.40.88-alpha-45-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c9ee335f7..bb79a2b5d 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.40.87-alpha-44-SNAPSHOT + 5.40.88-alpha-45-SNAPSHOT UTF-8 From 1519f45d07ac7f465e41092fc0c37b4efe7e6e80 Mon Sep 17 00:00:00 2001 From: Ian Nara Date: Fri, 18 Oct 2024 10:24:34 -0600 Subject: [PATCH 074/431] syslog --- scripts/aws/entrypoint.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/aws/entrypoint.sh b/scripts/aws/entrypoint.sh index 0e7d259ce..98de428d7 100755 --- a/scripts/aws/entrypoint.sh +++ b/scripts/aws/entrypoint.sh @@ -39,6 +39,11 @@ done DEBUG_MODE=$(jq -r ".debug_mode" < "${OVERRIDES_CONFIG}") + +# -- setup syslog-ng +echo "Starting syslog-ng..." +/usr/sbin/syslog-ng --verbose + if [[ ! "$DEBUG_MODE" == "true" ]]; then # -- setup syslog-ng echo "Starting syslog-ng..." From 21b407eccf8483c793ec9e51760474d52ea67acc Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Fri, 18 Oct 2024 16:27:33 +0000 Subject: [PATCH 075/431] [CI Pipeline] Released Snapshot version: 5.40.89-alpha-46-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index bb79a2b5d..379cf7758 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.40.88-alpha-45-SNAPSHOT + 5.40.89-alpha-46-SNAPSHOT UTF-8 From 3cb8e47f4dbe788b1db44d2bf782e63342330108 Mon Sep 17 00:00:00 2001 From: Ian Nara Date: Fri, 18 Oct 2024 11:12:40 -0600 Subject: [PATCH 076/431] syslog --- scripts/aws/entrypoint.sh | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/scripts/aws/entrypoint.sh b/scripts/aws/entrypoint.sh index 98de428d7..75c05ccbd 100755 --- a/scripts/aws/entrypoint.sh +++ b/scripts/aws/entrypoint.sh @@ -5,7 +5,9 @@ LOG_FILE="/home/start.txt" set -x -exec &> >(tee -a "$LOG_FILE") +exec > $LOG_FILE +exec 2>&1 +#exec &> >(tee -a "$LOG_FILE") set -o pipefail ulimit -n 65536 @@ -39,11 +41,6 @@ done DEBUG_MODE=$(jq -r ".debug_mode" < "${OVERRIDES_CONFIG}") - -# -- setup syslog-ng -echo "Starting syslog-ng..." -/usr/sbin/syslog-ng --verbose - if [[ ! "$DEBUG_MODE" == "true" ]]; then # -- setup syslog-ng echo "Starting syslog-ng..." From 7eb0424550982bc9fe0cb5b08089915fea125033 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Fri, 18 Oct 2024 17:13:33 +0000 Subject: [PATCH 077/431] [CI Pipeline] Released Snapshot version: 5.40.90-alpha-47-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 379cf7758..400d972b7 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.40.89-alpha-46-SNAPSHOT + 5.40.90-alpha-47-SNAPSHOT UTF-8 From 1d701e40ef3c113982a487e29ab99a7c9c7fbbc6 Mon Sep 17 00:00:00 2001 From: Ian Nara Date: Fri, 18 Oct 2024 11:43:34 -0600 Subject: [PATCH 078/431] syslog --- scripts/aws/entrypoint.sh | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/scripts/aws/entrypoint.sh b/scripts/aws/entrypoint.sh index 75c05ccbd..13583b8b5 100755 --- a/scripts/aws/entrypoint.sh +++ b/scripts/aws/entrypoint.sh @@ -20,6 +20,10 @@ ifconfig lo 127.0.0.1 echo "Starting vsock proxy..." /app/vsockpx --config /app/proxies.nitro.yaml --daemon --workers $(( $(nproc) * 2 )) --log-level 3 +# -- setup syslog-ng +echo "Starting syslog-ng..." +/usr/sbin/syslog-ng --verbose + # -- load config from identity service echo "Loading config from identity service via proxy..." @@ -41,11 +45,11 @@ done DEBUG_MODE=$(jq -r ".debug_mode" < "${OVERRIDES_CONFIG}") -if [[ ! "$DEBUG_MODE" == "true" ]]; then - # -- setup syslog-ng - echo "Starting syslog-ng..." - /usr/sbin/syslog-ng --verbose -fi +#if [[ ! "$DEBUG_MODE" == "true" ]]; then +# # -- setup syslog-ng +# echo "Starting syslog-ng..." +# /usr/sbin/syslog-ng --verbose +#fi # check the config is valid. Querying for a known missing element (empty) makes jq parse the file, but does not echo the results if jq empty "${OVERRIDES_CONFIG}"; then From b5ac1027007ed2c48aae02046132225e768baf2a Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Fri, 18 Oct 2024 17:44:42 +0000 Subject: [PATCH 079/431] [CI Pipeline] Released Snapshot version: 5.40.91-alpha-48-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 400d972b7..60141f961 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.40.90-alpha-47-SNAPSHOT + 5.40.91-alpha-48-SNAPSHOT UTF-8 From 4772242c13ee195e1e0dc6f257bf7eda91d3f701 Mon Sep 17 00:00:00 2001 From: Ian Nara Date: Fri, 18 Oct 2024 12:08:25 -0600 Subject: [PATCH 080/431] syslog --- scripts/aws/entrypoint.sh | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/scripts/aws/entrypoint.sh b/scripts/aws/entrypoint.sh index 13583b8b5..0e7d259ce 100755 --- a/scripts/aws/entrypoint.sh +++ b/scripts/aws/entrypoint.sh @@ -5,9 +5,7 @@ LOG_FILE="/home/start.txt" set -x -exec > $LOG_FILE -exec 2>&1 -#exec &> >(tee -a "$LOG_FILE") +exec &> >(tee -a "$LOG_FILE") set -o pipefail ulimit -n 65536 @@ -20,10 +18,6 @@ ifconfig lo 127.0.0.1 echo "Starting vsock proxy..." /app/vsockpx --config /app/proxies.nitro.yaml --daemon --workers $(( $(nproc) * 2 )) --log-level 3 -# -- setup syslog-ng -echo "Starting syslog-ng..." -/usr/sbin/syslog-ng --verbose - # -- load config from identity service echo "Loading config from identity service via proxy..." @@ -45,11 +39,11 @@ done DEBUG_MODE=$(jq -r ".debug_mode" < "${OVERRIDES_CONFIG}") -#if [[ ! "$DEBUG_MODE" == "true" ]]; then -# # -- setup syslog-ng -# echo "Starting syslog-ng..." -# /usr/sbin/syslog-ng --verbose -#fi +if [[ ! "$DEBUG_MODE" == "true" ]]; then + # -- setup syslog-ng + echo "Starting syslog-ng..." + /usr/sbin/syslog-ng --verbose +fi # check the config is valid. Querying for a known missing element (empty) makes jq parse the file, but does not echo the results if jq empty "${OVERRIDES_CONFIG}"; then From beacc48d501e7e6b306de505a6bf745b3bf46850 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Fri, 18 Oct 2024 18:09:32 +0000 Subject: [PATCH 081/431] [CI Pipeline] Released Snapshot version: 5.40.92-alpha-49-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 60141f961..36fd4fba2 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.40.91-alpha-48-SNAPSHOT + 5.40.92-alpha-49-SNAPSHOT UTF-8 From 5aa2004a65d0e90321e03849e943fb1d8ff6206c Mon Sep 17 00:00:00 2001 From: Ian Nara Date: Fri, 18 Oct 2024 12:20:11 -0600 Subject: [PATCH 082/431] syslog --- scripts/aws/entrypoint.sh | 33 +++++++++++++-------------------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/scripts/aws/entrypoint.sh b/scripts/aws/entrypoint.sh index 0e7d259ce..78181dd85 100755 --- a/scripts/aws/entrypoint.sh +++ b/scripts/aws/entrypoint.sh @@ -97,25 +97,18 @@ fi cd /app # -- start operator -echo "Starting Java application..." if [[ "$DEBUG_MODE" == "true" ]]; then - java \ - -XX:MaxRAMPercentage=95 -XX:-UseCompressedOops -XX:+PrintFlagsFinal \ - -Djava.security.egd=file:/dev/./urandom \ - -Djava.library.path=/app/lib \ - -Dvertx-config-path="${FINAL_CONFIG}" \ - -Dvertx.logger-delegate-factory-class-name=io.vertx.core.logging.SLF4JLogDelegateFactory \ - -Dlogback.configurationFile=./conf/logback-debug.xml \ - -Dhttp_proxy=socks5://127.0.0.1:3305 \ - -jar /app/"${JAR_NAME}"-"${JAR_VERSION}".jar + LOGBACK_CONF="./conf/logback-debug.xml" else - java \ - -XX:MaxRAMPercentage=95 -XX:-UseCompressedOops -XX:+PrintFlagsFinal \ - -Djava.security.egd=file:/dev/./urandom \ - -Djava.library.path=/app/lib \ - -Dvertx-config-path="${FINAL_CONFIG}" \ - -Dvertx.logger-delegate-factory-class-name=io.vertx.core.logging.SLF4JLogDelegateFactory \ - -Dlogback.configurationFile=./conf/logback.xml \ - -Dhttp_proxy=socks5://127.0.0.1:3305 \ - -jar /app/"${JAR_NAME}"-"${JAR_VERSION}".jar -fi \ No newline at end of file + LOGBACK_CONF="./conf/logback.xml" +fi +echo "Starting Java application..." +java \ + -XX:MaxRAMPercentage=95 -XX:-UseCompressedOops -XX:+PrintFlagsFinal \ + -Djava.security.egd=file:/dev/./urandom \ + -Djava.library.path=/app/lib \ + -Dvertx-config-path="${FINAL_CONFIG}" \ + -Dvertx.logger-delegate-factory-class-name=io.vertx.core.logging.SLF4JLogDelegateFactory \ + -Dlogback.configurationFile=${LOGBACK_CONF} \ + -Dhttp_proxy=socks5://127.0.0.1:3305 \ + -jar /app/"${JAR_NAME}"-"${JAR_VERSION}".jar \ No newline at end of file From 95a826c13ebe8bd4e08a950c420a188e71c2fca2 Mon Sep 17 00:00:00 2001 From: Ian Nara Date: Fri, 18 Oct 2024 12:21:02 -0600 Subject: [PATCH 083/431] syslog --- scripts/aws/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/aws/entrypoint.sh b/scripts/aws/entrypoint.sh index 78181dd85..e1ab1a5bc 100755 --- a/scripts/aws/entrypoint.sh +++ b/scripts/aws/entrypoint.sh @@ -111,4 +111,4 @@ java \ -Dvertx.logger-delegate-factory-class-name=io.vertx.core.logging.SLF4JLogDelegateFactory \ -Dlogback.configurationFile=${LOGBACK_CONF} \ -Dhttp_proxy=socks5://127.0.0.1:3305 \ - -jar /app/"${JAR_NAME}"-"${JAR_VERSION}".jar \ No newline at end of file + -jar /app/"${JAR_NAME}"-"${JAR_VERSION}".jar From 0e63e5abd9cfd4e4399dcbafcd093921b0501052 Mon Sep 17 00:00:00 2001 From: Ian Nara Date: Fri, 18 Oct 2024 12:22:42 -0600 Subject: [PATCH 084/431] debug --- scripts/aws/entrypoint.sh | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/scripts/aws/entrypoint.sh b/scripts/aws/entrypoint.sh index e1ab1a5bc..ef622ff80 100755 --- a/scripts/aws/entrypoint.sh +++ b/scripts/aws/entrypoint.sh @@ -39,7 +39,10 @@ done DEBUG_MODE=$(jq -r ".debug_mode" < "${OVERRIDES_CONFIG}") -if [[ ! "$DEBUG_MODE" == "true" ]]; then +if [[ "$DEBUG_MODE" == "true" ]]; then + LOGBACK_CONF="./conf/logback-debug.xml" +else + LOGBACK_CONF="./conf/logback.xml" # -- setup syslog-ng echo "Starting syslog-ng..." /usr/sbin/syslog-ng --verbose @@ -97,11 +100,6 @@ fi cd /app # -- start operator -if [[ "$DEBUG_MODE" == "true" ]]; then - LOGBACK_CONF="./conf/logback-debug.xml" -else - LOGBACK_CONF="./conf/logback.xml" -fi echo "Starting Java application..." java \ -XX:MaxRAMPercentage=95 -XX:-UseCompressedOops -XX:+PrintFlagsFinal \ From aa8c29267be3c76ac7137d16199890df28365f44 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Fri, 18 Oct 2024 18:24:57 +0000 Subject: [PATCH 085/431] [CI Pipeline] Released Snapshot version: 5.40.93-alpha-50-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 36fd4fba2..8bad5daf9 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.40.92-alpha-49-SNAPSHOT + 5.40.93-alpha-50-SNAPSHOT UTF-8 From c24187036bbc31cd874c1283d357185cfc033601 Mon Sep 17 00:00:00 2001 From: Ian Nara Date: Fri, 18 Oct 2024 15:05:21 -0600 Subject: [PATCH 086/431] debug --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 8bad5daf9..082c67876 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.40.93-alpha-50-SNAPSHOT + 5.40.86 UTF-8 From 3fc5dc35f33db86c6c9e60c4ffe03c98093d4f76 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Fri, 18 Oct 2024 21:11:55 +0000 Subject: [PATCH 087/431] [CI Pipeline] Released Patch version: 5.40.106 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 082c67876..5a8363e44 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.40.86 + 5.40.106 UTF-8 From 24ab08e9c5f6e49889266b9348e06fb1c7eaa411 Mon Sep 17 00:00:00 2001 From: asloob qureshi Date: Fri, 18 Oct 2024 16:14:27 -0700 Subject: [PATCH 088/431] Remove unwanted assertions. Address review comments --- pom.xml | 2 +- .../operator/monitoring/StatsCollectorVerticle.java | 4 ---- .../uid2/operator/store/CloudSyncOptOutStore.java | 3 ++- src/main/java/com/uid2/operator/util/Tuple.java | 13 +++++++------ .../java/com/uid2/operator/V2RequestUtilTest.java | 12 +++++------- 5 files changed, 15 insertions(+), 19 deletions(-) diff --git a/pom.xml b/pom.xml index 65bac8849..082c67876 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.40.87-alpha-110-SNAPSHOT + 5.40.86 UTF-8 diff --git a/src/main/java/com/uid2/operator/monitoring/StatsCollectorVerticle.java b/src/main/java/com/uid2/operator/monitoring/StatsCollectorVerticle.java index e6742356a..77e179e77 100644 --- a/src/main/java/com/uid2/operator/monitoring/StatsCollectorVerticle.java +++ b/src/main/java/com/uid2/operator/monitoring/StatsCollectorVerticle.java @@ -88,10 +88,6 @@ public void handleMessage(Message message) { return; } - if (messageItem == null) { - throw new NullPointerException("Message could not be deserialized"); - } - String path = messageItem.getPath(); String apiVersion = "v0"; String endpoint = path.substring(1); diff --git a/src/main/java/com/uid2/operator/store/CloudSyncOptOutStore.java b/src/main/java/com/uid2/operator/store/CloudSyncOptOutStore.java index f7e3a495d..49f7d0385 100644 --- a/src/main/java/com/uid2/operator/store/CloudSyncOptOutStore.java +++ b/src/main/java/com/uid2/operator/store/CloudSyncOptOutStore.java @@ -604,7 +604,8 @@ private OptOutStoreSnapshot updateIndexInternal(IndexUpdateContext iuc) { if (numPartitions == 0) { // if update doesn't have a new partition, simply update heap with new log data if (!iuc.getDeltasToRemove().isEmpty()) { - final String errorMsg = "Invalid number of Deltas to remove=" + iuc.getDeltasToRemove().size(); + final String errorMsg = "Invalid number of Deltas to remove=" + iuc.getDeltasToRemove().size() + + " when there are 0 new partitions to index"; LOGGER.error(errorMsg); throw new IllegalStateException(errorMsg); } diff --git a/src/main/java/com/uid2/operator/util/Tuple.java b/src/main/java/com/uid2/operator/util/Tuple.java index 979e79b4e..f2cf39ff0 100644 --- a/src/main/java/com/uid2/operator/util/Tuple.java +++ b/src/main/java/com/uid2/operator/util/Tuple.java @@ -1,14 +1,15 @@ package com.uid2.operator.util; +import java.util.Objects; + public class Tuple { public static class Tuple2 { private final T1 item1; private final T2 item2; public Tuple2(T1 item1, T2 item2) { - if (item1 == null || item2 == null) { - throw new NullPointerException(); - } + Objects.requireNonNull(item1); + Objects.requireNonNull(item2); this.item1 = item1; this.item2 = item2; @@ -35,9 +36,9 @@ public static class Tuple3 { private final T3 item3; public Tuple3(T1 item1, T2 item2, T3 item3) { - if (item1 == null || item2 == null || item3 == null) { - throw new NullPointerException(); - } + Objects.requireNonNull(item1); + Objects.requireNonNull(item2); + Objects.requireNonNull(item3); this.item1 = item1; this.item2 = item2; diff --git a/src/test/java/com/uid2/operator/V2RequestUtilTest.java b/src/test/java/com/uid2/operator/V2RequestUtilTest.java index dcf4fa38b..f296411e0 100644 --- a/src/test/java/com/uid2/operator/V2RequestUtilTest.java +++ b/src/test/java/com/uid2/operator/V2RequestUtilTest.java @@ -23,7 +23,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.fail; +import static org.junit.jupiter.api.Assertions.assertThrowsExactly; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -153,11 +153,9 @@ public void testHandleRefreshTokenInResponseBody() { when(keyManager.getRefreshKey()).thenReturn(refreshKey); when(refreshKey.getId()).thenReturn(Integer.MAX_VALUE); when(refreshKey.getKeyBytes()).thenReturn(Random.getRandomKeyBytes()); - try { - V2RequestUtil.handleRefreshTokenInResponseBody(jsonBody, keyManager, IdentityScope.UID2); - fail("IllegalArgumentException not thrown"); - } catch (IllegalArgumentException e) { - assertEquals("Generated refresh token's length=168 is not equal to=388", e.getMessage()); - } + IllegalArgumentException e = assertThrowsExactly( + IllegalArgumentException.class, + () -> V2RequestUtil.handleRefreshTokenInResponseBody(jsonBody, keyManager, IdentityScope.UID2)); + assertEquals("Generated refresh token's length=168 is not equal to=388", e.getMessage()); } } From 9457a9435898f7c09a773979958e89ab918e0f9c Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Sun, 20 Oct 2024 23:19:16 +0000 Subject: [PATCH 089/431] [CI Pipeline] Released Snapshot version: 5.40.107-alpha-111-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9828d16b6..40fa919ae 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.40.106 + 5.40.107-alpha-111-SNAPSHOT UTF-8 From 1afe2eead371a57a3fe5269a71cd2ecede776e45 Mon Sep 17 00:00:00 2001 From: Thomas Manson Date: Mon, 21 Oct 2024 11:20:32 +1100 Subject: [PATCH 090/431] Update version of shared --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9828d16b6..da620668b 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ 2.1.0 2.1.0 2.1.0 - 7.19.2-alpha-151-SNAPSHOT + 7.20.0 ${project.version} 21 21 From 05e63775d4319c2f2ad5bb544b60ccb6bd09a826 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Mon, 21 Oct 2024 00:54:53 +0000 Subject: [PATCH 091/431] [CI Pipeline] Released Minor version: 5.41.0 --- pom.xml | 2 +- version.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 59daeb5fe..4dddd15aa 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.40.107-alpha-111-SNAPSHOT + 5.41.0 UTF-8 diff --git a/version.json b/version.json index 16e102dfa..a43cbb357 100644 --- a/version.json +++ b/version.json @@ -1 +1 @@ -{ "$schema": "https://raw.githubusercontent.com/dotnet/Nerdbank.GitVersioning/master/src/NerdBank.GitVersioning/version.schema.json", "version": "5.40", "publicReleaseRefSpec": [ "^refs/heads/master$", "^refs/heads/v\\d+(?:\\.\\d+)?$" ], "cloudBuild": { "setVersionVariables": true, "buildNumber": { "enabled": true, "includeCommitId": { "when": "always" } } } } +{ "$schema": "https://raw.githubusercontent.com/dotnet/Nerdbank.GitVersioning/master/src/NerdBank.GitVersioning/version.schema.json", "version": "5.41", "publicReleaseRefSpec": [ "^refs/heads/master$", "^refs/heads/v\\d+(?:\\.\\d+)?$" ], "cloudBuild": { "setVersionVariables": true, "buildNumber": { "enabled": true, "includeCommitId": { "when": "always" } } } } From 4c1de75cb033397ba94965d21d630e985163a529 Mon Sep 17 00:00:00 2001 From: Sunny Wu Date: Mon, 21 Oct 2024 12:25:10 +1100 Subject: [PATCH 092/431] Moved privacyBits out of FirstLevelHashIdentity and treat it as a separate input --- .../operator/model/AdvertisingTokenInput.java | 1 - .../operator/model/RefreshTokenInput.java | 6 +++- .../userIdentity/FirstLevelHashIdentity.java | 7 +---- .../service/EncryptedTokenEncoder.java | 18 +++++++----- .../operator/service/UIDOperatorService.java | 29 ++++++++++++------- .../com/uid2/operator/TokenEncodingTest.java | 5 ++-- .../operator/UIDOperatorVerticleTest.java | 2 +- 7 files changed, 39 insertions(+), 29 deletions(-) diff --git a/src/main/java/com/uid2/operator/model/AdvertisingTokenInput.java b/src/main/java/com/uid2/operator/model/AdvertisingTokenInput.java index 2d32dc4f4..a9bcbc11f 100644 --- a/src/main/java/com/uid2/operator/model/AdvertisingTokenInput.java +++ b/src/main/java/com/uid2/operator/model/AdvertisingTokenInput.java @@ -9,7 +9,6 @@ public class AdvertisingTokenInput extends VersionedToken { public final OperatorIdentity operatorIdentity; public final SourcePublisher sourcePublisher; public final RawUidIdentity rawUidIdentity; - public final int privacyBits; public final Instant establishedAt; diff --git a/src/main/java/com/uid2/operator/model/RefreshTokenInput.java b/src/main/java/com/uid2/operator/model/RefreshTokenInput.java index 15bef6a0c..2d306bbfb 100644 --- a/src/main/java/com/uid2/operator/model/RefreshTokenInput.java +++ b/src/main/java/com/uid2/operator/model/RefreshTokenInput.java @@ -9,12 +9,16 @@ public class RefreshTokenInput extends VersionedToken { public final OperatorIdentity operatorIdentity; public final SourcePublisher sourcePublisher; public final FirstLevelHashIdentity firstLevelHashIdentity; + // inherited from the previous refresh token's privacy bits + public final int privacyBits; + public RefreshTokenInput(TokenVersion version, Instant createdAt, Instant expiresAt, OperatorIdentity operatorIdentity, - SourcePublisher sourcePublisher, FirstLevelHashIdentity firstLevelHashIdentity) { + SourcePublisher sourcePublisher, FirstLevelHashIdentity firstLevelHashIdentity, int privacyBits) { super(version, createdAt, expiresAt); this.operatorIdentity = operatorIdentity; this.sourcePublisher = sourcePublisher; this.firstLevelHashIdentity = firstLevelHashIdentity; + this.privacyBits = privacyBits; } } diff --git a/src/main/java/com/uid2/operator/model/userIdentity/FirstLevelHashIdentity.java b/src/main/java/com/uid2/operator/model/userIdentity/FirstLevelHashIdentity.java index 5eb13526a..b549833ed 100644 --- a/src/main/java/com/uid2/operator/model/userIdentity/FirstLevelHashIdentity.java +++ b/src/main/java/com/uid2/operator/model/userIdentity/FirstLevelHashIdentity.java @@ -10,19 +10,14 @@ public class FirstLevelHashIdentity extends UserIdentity { public final byte[] firstLevelHash; - // for brand new token generation, it should contain 1 - // if the first level hash is from token/refresh call, it will inherit from the refresh token - public final int privacyBits; - // for brand new token generation, it should be the time it is generated // if the first level hash is from token/refresh call, it will be when the raw UID was originally created in the earliest token generation public final Instant establishedAt; - public FirstLevelHashIdentity(IdentityScope identityScope, IdentityType identityType, byte[] firstLevelHash, int privacyBits, + public FirstLevelHashIdentity(IdentityScope identityScope, IdentityType identityType, byte[] firstLevelHash, Instant establishedAt) { super(identityScope, identityType); this.firstLevelHash = firstLevelHash; - this.privacyBits = privacyBits; this.establishedAt = establishedAt; } diff --git a/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java b/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java index f3e22c5b5..c60d3f96a 100644 --- a/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java +++ b/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java @@ -127,8 +127,9 @@ private RefreshTokenInput decodeRefreshTokenV2(Buffer b) { TokenVersion.V2, createdAt, validTill, new OperatorIdentity(0, OperatorType.Service, 0, 0), new SourcePublisher(siteId, 0, 0), - new FirstLevelHashIdentity(IdentityScope.UID2, IdentityType.Email, identity, privacyBits, - Instant.ofEpochMilli(establishedMillis))); + new FirstLevelHashIdentity(IdentityScope.UID2, IdentityType.Email, identity, + Instant.ofEpochMilli(establishedMillis)), + privacyBits); } private RefreshTokenInput decodeRefreshTokenV3(Buffer b, byte[] bytes) { @@ -161,7 +162,8 @@ private RefreshTokenInput decodeRefreshTokenV3(Buffer b, byte[] bytes) { return new RefreshTokenInput( TokenVersion.V3, createdAt, expiresAt, operatorIdentity, sourcePublisher, - new FirstLevelHashIdentity(identityScope, identityType, firstLevelHash, privacyBits, establishedAt)); + new FirstLevelHashIdentity(identityScope, identityType, firstLevelHash, establishedAt), + privacyBits); } @Override @@ -308,7 +310,8 @@ public byte[] encodeIntoRefreshTokenV2(RefreshTokenInput t, KeysetKey serviceKey // give an extra minute for clients which are trying to refresh tokens close to or at the refresh expiry timestamp b.appendLong(t.expiresAt.plusSeconds(60).toEpochMilli()); b.appendInt(serviceKey.getId()); - final byte[] encryptedIdentity = encryptIdentityV2(t.sourcePublisher, t.firstLevelHashIdentity, serviceKey); + final byte[] encryptedIdentity = encryptIdentityV2(t.sourcePublisher, t.firstLevelHashIdentity, serviceKey, + t.privacyBits); b.appendBytes(encryptedIdentity); return b.getBytes(); } @@ -319,7 +322,7 @@ public byte[] encodeIntoRefreshTokenV3(RefreshTokenInput t, KeysetKey serviceKey refreshPayload.appendLong(t.createdAt.toEpochMilli()); encodeOperatorIdentityV3(refreshPayload, t.operatorIdentity); encodePublisherRequesterV3(refreshPayload, t.sourcePublisher); - refreshPayload.appendInt(t.firstLevelHashIdentity.privacyBits); + refreshPayload.appendInt(t.privacyBits); refreshPayload.appendLong(t.firstLevelHashIdentity.establishedAt.toEpochMilli()); refreshPayload.appendByte(encodeIdentityTypeV3(t.firstLevelHashIdentity.identityScope, t.firstLevelHashIdentity.identityType)); refreshPayload.appendBytes(t.firstLevelHashIdentity.firstLevelHash); @@ -368,8 +371,9 @@ private String generateAdvertisingTokenString(AdvertisingTokenInput advertisingT return bytesToBase64Token(advertisingTokenBytes, advertisingTokenInput.version); } - private byte[] encryptIdentityV2(SourcePublisher sourcePublisher, FirstLevelHashIdentity firstLevelHashIdentity, KeysetKey key) { - return encryptIdentityV2(sourcePublisher, firstLevelHashIdentity.firstLevelHash, firstLevelHashIdentity.privacyBits, + private byte[] encryptIdentityV2(SourcePublisher sourcePublisher, FirstLevelHashIdentity firstLevelHashIdentity, + KeysetKey key, int privacyBits) { + return encryptIdentityV2(sourcePublisher, firstLevelHashIdentity.firstLevelHash, privacyBits, firstLevelHashIdentity.establishedAt, key); } diff --git a/src/main/java/com/uid2/operator/service/UIDOperatorService.java b/src/main/java/com/uid2/operator/service/UIDOperatorService.java index 2b4288151..b76261f87 100644 --- a/src/main/java/com/uid2/operator/service/UIDOperatorService.java +++ b/src/main/java/com/uid2/operator/service/UIDOperatorService.java @@ -109,13 +109,13 @@ public IdentityResponse generateIdentity(IdentityRequest request) { final Instant now = EncodingUtils.NowUTCMillis(this.clock); final byte[] firstLevelHash = getFirstLevelHash(request.hashedDiiIdentity.hashedDii, now); final FirstLevelHashIdentity firstLevelHashIdentity = new FirstLevelHashIdentity( - request.hashedDiiIdentity.identityScope, request.hashedDiiIdentity.identityType, firstLevelHash, request.privacyBits, + request.hashedDiiIdentity.identityScope, request.hashedDiiIdentity.identityType, firstLevelHash, request.establishedAt); if (request.shouldCheckOptOut() && getGlobalOptOutResult(firstLevelHashIdentity, false).isOptedOut()) { return IdentityResponse.OptOutIdentityResponse; } else { - return generateIdentity(request.sourcePublisher, firstLevelHashIdentity); + return generateIdentity(request.sourcePublisher, firstLevelHashIdentity, request.privacyBits); } } @@ -136,7 +136,7 @@ public RefreshResponse refreshIdentity(RefreshTokenInput token) { return RefreshResponse.Expired; } - final PrivacyBits privacyBits = PrivacyBits.fromInt(token.firstLevelHashIdentity.privacyBits); + final PrivacyBits privacyBits = PrivacyBits.fromInt(token.privacyBits); final boolean isCstg = privacyBits.isClientSideTokenGenerated(); try { @@ -146,7 +146,9 @@ public RefreshResponse refreshIdentity(RefreshTokenInput token) { final Duration durationSinceLastRefresh = Duration.between(token.createdAt, now); if (!optedOut) { - IdentityResponse identityResponse = this.generateIdentity(token.sourcePublisher, token.firstLevelHashIdentity); + IdentityResponse identityResponse = this.generateIdentity(token.sourcePublisher, + token.firstLevelHashIdentity, + token.privacyBits); return RefreshResponse.createRefreshedResponse(identityResponse, durationSinceLastRefresh, isCstg); } else { @@ -231,7 +233,7 @@ private FirstLevelHashIdentity getFirstLevelHashIdentity(HashedDiiIdentity hashe private FirstLevelHashIdentity getFirstLevelHashIdentity(IdentityScope identityScope, IdentityType identityType, byte[] identityHash, Instant asOf) { final byte[] firstLevelHash = getFirstLevelHash(identityHash, asOf); - return new FirstLevelHashIdentity(identityScope, identityType, firstLevelHash, 0, null); + return new FirstLevelHashIdentity(identityScope, identityType, firstLevelHash, null); } private byte[] getFirstLevelHash(byte[] identityHash, Instant asOf) { @@ -249,7 +251,8 @@ private RawUidResponse generateRawUid(FirstLevelHashIdentity firstLevelHashIdent rotatingSalt.getHashedId()); } - private IdentityResponse generateIdentity(SourcePublisher sourcePublisher, FirstLevelHashIdentity firstLevelHashIdentity) { + private IdentityResponse generateIdentity(SourcePublisher sourcePublisher, + FirstLevelHashIdentity firstLevelHashIdentity, int privacyBits) { final Instant nowUtc = EncodingUtils.NowUTCMillis(this.clock); final RawUidResponse rawUidResponse = generateRawUid(firstLevelHashIdentity, nowUtc); @@ -258,22 +261,26 @@ private IdentityResponse generateIdentity(SourcePublisher sourcePublisher, First rawUidResponse.rawUid); return this.encoder.encodeIntoIdentityResponse( - this.createAdvertisingTokenInput(sourcePublisher, rawUidIdentity, nowUtc, firstLevelHashIdentity.privacyBits, firstLevelHashIdentity.establishedAt), - this.createRefreshTokenInput(sourcePublisher, firstLevelHashIdentity, nowUtc), + this.createAdvertisingTokenInput(sourcePublisher, rawUidIdentity, nowUtc, privacyBits, + firstLevelHashIdentity.establishedAt), + this.createRefreshTokenInput(sourcePublisher, firstLevelHashIdentity, nowUtc, privacyBits), nowUtc.plusMillis(refreshIdentityAfter.toMillis()), nowUtc ); } - private RefreshTokenInput createRefreshTokenInput(SourcePublisher sourcePublisher, FirstLevelHashIdentity firstLevelHashIdentity, - Instant now) { + private RefreshTokenInput createRefreshTokenInput(SourcePublisher sourcePublisher, + FirstLevelHashIdentity firstLevelHashIdentity, + Instant now, + int privacyBits) { return new RefreshTokenInput( this.refreshTokenVersion, now, now.plusMillis(refreshExpiresAfter.toMillis()), this.operatorIdentity, sourcePublisher, - firstLevelHashIdentity); + firstLevelHashIdentity, + privacyBits); } private AdvertisingTokenInput createAdvertisingTokenInput(SourcePublisher sourcePublisher, RawUidIdentity rawUidIdentity, diff --git a/src/test/java/com/uid2/operator/TokenEncodingTest.java b/src/test/java/com/uid2/operator/TokenEncodingTest.java index b2255e1d0..2057b8891 100644 --- a/src/test/java/com/uid2/operator/TokenEncodingTest.java +++ b/src/test/java/com/uid2/operator/TokenEncodingTest.java @@ -59,7 +59,8 @@ public void testRefreshTokenEncoding(TokenVersion tokenVersion) { now.plusSeconds(360), new OperatorIdentity(101, OperatorType.Service, 102, 103), new SourcePublisher(111, 112, 113), - new FirstLevelHashIdentity(IdentityScope.UID2, IdentityType.Email, firstLevelHash, 121, now) + new FirstLevelHashIdentity(IdentityScope.UID2, IdentityType.Email, firstLevelHash, now), + 121 ); if (tokenVersion == TokenVersion.V4) { @@ -74,7 +75,7 @@ public void testRefreshTokenEncoding(TokenVersion tokenVersion) { int addSeconds = (tokenVersion == TokenVersion.V2) ? 60 : 0; //todo: why is there a 60 second buffer in encodeV2() but not in encodeV3()? assertEquals(refreshTokenInput.expiresAt.plusSeconds(addSeconds), decoded.expiresAt); assertTrue(refreshTokenInput.firstLevelHashIdentity.matches(decoded.firstLevelHashIdentity)); - assertEquals(refreshTokenInput.firstLevelHashIdentity.privacyBits, decoded.firstLevelHashIdentity.privacyBits); + assertEquals(refreshTokenInput.privacyBits, decoded.privacyBits); assertEquals(refreshTokenInput.firstLevelHashIdentity.establishedAt, decoded.firstLevelHashIdentity.establishedAt); assertEquals(refreshTokenInput.sourcePublisher.siteId, decoded.sourcePublisher.siteId); diff --git a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java index 1e09c6ea7..c0afecbc6 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java @@ -4203,7 +4203,7 @@ private void assertAreClientSideGeneratedTokens(AdvertisingTokenInput advertisin private void assertAreClientSideGeneratedTokens(AdvertisingTokenInput advertisingTokenInput, RefreshTokenInput refreshTokenInput, int siteId, IdentityType identityType, String identity, boolean expectedOptOut) { final PrivacyBits advertisingTokenPrivacyBits = PrivacyBits.fromInt(advertisingTokenInput.privacyBits); - final PrivacyBits refreshTokenPrivacyBits = PrivacyBits.fromInt(refreshTokenInput.firstLevelHashIdentity.privacyBits); + final PrivacyBits refreshTokenPrivacyBits = PrivacyBits.fromInt(refreshTokenInput.privacyBits); final byte[] rawUid = getRawUidFromIdentity(identityType, identity, From 062f90887422c78af1803411eeb32ab1e23890fb Mon Sep 17 00:00:00 2001 From: Sunny Wu Date: Mon, 21 Oct 2024 12:37:29 +1100 Subject: [PATCH 093/431] Added a new IdentityRequest constructor to set default values for privacyBits and establishedAt fields --- .../com/uid2/operator/model/IdentityRequest.java | 13 +++++++++++++ .../com/uid2/operator/model/RefreshTokenInput.java | 2 +- .../java/com/uid2/operator/util/PrivacyBits.java | 3 +++ .../uid2/operator/vertx/UIDOperatorVerticle.java | 6 +++--- .../com/uid2/operator/UIDOperatorServiceTest.java | 2 +- .../operator/benchmark/TokenEndecBenchmark.java | 4 ++-- 6 files changed, 23 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/uid2/operator/model/IdentityRequest.java b/src/main/java/com/uid2/operator/model/IdentityRequest.java index a4f7c9b1d..a6c7bfc9c 100644 --- a/src/main/java/com/uid2/operator/model/IdentityRequest.java +++ b/src/main/java/com/uid2/operator/model/IdentityRequest.java @@ -1,6 +1,7 @@ package com.uid2.operator.model; import com.uid2.operator.model.userIdentity.HashedDiiIdentity; +import com.uid2.operator.util.PrivacyBits; import java.time.Instant; @@ -26,6 +27,18 @@ public IdentityRequest( this.establishedAt = establishedAt; } + public IdentityRequest( + SourcePublisher sourcePublisher, + HashedDiiIdentity hashedDiiIdentity, + OptoutCheckPolicy tokenGeneratePolicy) + { + this.sourcePublisher = sourcePublisher; + this.hashedDiiIdentity = hashedDiiIdentity; + this.optoutCheckPolicy = tokenGeneratePolicy; + this.privacyBits = PrivacyBits.DEFAULT_PRIVACY_BIT_VALUE; + this.establishedAt = Instant.now(); + } + public boolean shouldCheckOptOut() { return optoutCheckPolicy.equals(OptoutCheckPolicy.RespectOptOut); } diff --git a/src/main/java/com/uid2/operator/model/RefreshTokenInput.java b/src/main/java/com/uid2/operator/model/RefreshTokenInput.java index 2d306bbfb..9b45ca3e9 100644 --- a/src/main/java/com/uid2/operator/model/RefreshTokenInput.java +++ b/src/main/java/com/uid2/operator/model/RefreshTokenInput.java @@ -9,7 +9,7 @@ public class RefreshTokenInput extends VersionedToken { public final OperatorIdentity operatorIdentity; public final SourcePublisher sourcePublisher; public final FirstLevelHashIdentity firstLevelHashIdentity; - // inherited from the previous refresh token's privacy bits + // by default, inherited from the previous refresh token's privacy bits public final int privacyBits; diff --git a/src/main/java/com/uid2/operator/util/PrivacyBits.java b/src/main/java/com/uid2/operator/util/PrivacyBits.java index f8b7edf6a..11edde226 100644 --- a/src/main/java/com/uid2/operator/util/PrivacyBits.java +++ b/src/main/java/com/uid2/operator/util/PrivacyBits.java @@ -3,6 +3,9 @@ public class PrivacyBits { + // For historical reason this bit is set + public static final int DEFAULT_PRIVACY_BIT_VALUE = 1; + private static final int BIT_LEGACY = 0; private static final int BIT_CSTG = 1; private static final int BIT_CSTG_OPTOUT = 2; diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index 143de537d..828dd2c14 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -941,7 +941,7 @@ private void handleTokenGenerateV1(RoutingContext rc) { new IdentityRequest( new SourcePublisher(siteId, 0, 0), input.toHashedDiiIdentity(this.identityScope), - OptoutCheckPolicy.defaultPolicy(), 1, Instant.now())); + OptoutCheckPolicy.defaultPolicy())); //Integer.parseInt(rc.queryParam("privacy_bits").get(0)))); @@ -997,7 +997,7 @@ private void handleTokenGenerateV2(RoutingContext rc) { new IdentityRequest( new SourcePublisher(siteId, 0, 0), input.toHashedDiiIdentity(this.identityScope), - OptoutCheckPolicy.respectOptOut(),1, Instant.now())); + OptoutCheckPolicy.respectOptOut())); if (t.isOptedOut()) { if (optoutCheckPolicy.getItem1() == OptoutCheckPolicy.DoNotRespect) { // only legacy can use this policy @@ -1053,7 +1053,7 @@ else if (!input.isValid()) { new IdentityRequest( new SourcePublisher(siteId, 0, 0), input.toHashedDiiIdentity(this.identityScope), - OptoutCheckPolicy.defaultPolicy(), 1, Instant.now())); + OptoutCheckPolicy.defaultPolicy())); //Integer.parseInt(rc.queryParam("privacy_bits").get(0)))); diff --git a/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java b/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java index 5c3b23433..4d749f4d7 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java @@ -648,7 +648,7 @@ void testNormalIdentityOptIn(TestIdentityInputType type, String id, IdentityScop final IdentityRequest identityRequest = new IdentityRequest( new SourcePublisher(123, 124, 125), inputVal.toHashedDiiIdentity(scope), - OptoutCheckPolicy.DoNotRespect, 1, Instant.now() + OptoutCheckPolicy.DoNotRespect ); IdentityResponse identityResponse; if(scope == IdentityScope.EUID) { diff --git a/src/test/java/com/uid2/operator/benchmark/TokenEndecBenchmark.java b/src/test/java/com/uid2/operator/benchmark/TokenEndecBenchmark.java index bc8fddb27..14a650047 100644 --- a/src/test/java/com/uid2/operator/benchmark/TokenEndecBenchmark.java +++ b/src/test/java/com/uid2/operator/benchmark/TokenEndecBenchmark.java @@ -43,7 +43,7 @@ static IdentityResponse[] createAdvertisingTokens() { uidService.generateIdentity(new IdentityRequest( publisher, hashedDiiIdentities[i], - OptoutCheckPolicy.DoNotRespect, 1, Instant.now()))); + OptoutCheckPolicy.DoNotRespect))); } return tokens.toArray(new IdentityResponse[tokens.size()]); } @@ -54,7 +54,7 @@ public IdentityResponse TokenGenerationBenchmark() { return uidService.generateIdentity(new IdentityRequest( publisher, hashedDiiIdentities[(idx++) & 65535], - OptoutCheckPolicy.DoNotRespect, 1, Instant.now())); + OptoutCheckPolicy.DoNotRespect)); } @Benchmark From 76a134579c312e1ddcf0ef092242e50577759a35 Mon Sep 17 00:00:00 2001 From: Sunny Wu Date: Mon, 28 Oct 2024 12:35:47 +1100 Subject: [PATCH 094/431] Update src/main/java/com/uid2/operator/model/IdentityRequest.java Co-authored-by: mcollins-ttd <118872455+mcollins-ttd@users.noreply.github.com> --- src/main/java/com/uid2/operator/model/IdentityRequest.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/uid2/operator/model/IdentityRequest.java b/src/main/java/com/uid2/operator/model/IdentityRequest.java index a6c7bfc9c..877a95d72 100644 --- a/src/main/java/com/uid2/operator/model/IdentityRequest.java +++ b/src/main/java/com/uid2/operator/model/IdentityRequest.java @@ -32,11 +32,8 @@ public IdentityRequest( HashedDiiIdentity hashedDiiIdentity, OptoutCheckPolicy tokenGeneratePolicy) { - this.sourcePublisher = sourcePublisher; - this.hashedDiiIdentity = hashedDiiIdentity; - this.optoutCheckPolicy = tokenGeneratePolicy; - this.privacyBits = PrivacyBits.DEFAULT_PRIVACY_BIT_VALUE; - this.establishedAt = Instant.now(); + this(sourcePublisher, hashedDiiIdentity, tokenGeneratePolicy, PrivacyBits.DEFAULT_PRIVACY_BIT_VALUE, Instant.now()); + } public boolean shouldCheckOptOut() { From 5db6e4f29cec6c19ac9e6512df9713a9a5ecdb71 Mon Sep 17 00:00:00 2001 From: Sunny Wu Date: Mon, 28 Oct 2024 13:47:28 +1100 Subject: [PATCH 095/431] Replace privacy bits type to PrivacyBits class instance --- .../operator/model/AdvertisingTokenInput.java | 5 ++- .../uid2/operator/model/IdentityRequest.java | 6 +-- .../operator/model/RefreshTokenInput.java | 5 ++- .../service/EncryptedTokenEncoder.java | 23 ++++++----- .../operator/service/UIDOperatorService.java | 9 ++--- .../com/uid2/operator/util/PrivacyBits.java | 20 +++++++++- .../operator/vertx/UIDOperatorVerticle.java | 4 +- .../com/uid2/operator/TokenEncodingTest.java | 5 ++- .../uid2/operator/UIDOperatorServiceTest.java | 23 ++++++----- .../operator/UIDOperatorVerticleTest.java | 40 +++++++++---------- 10 files changed, 81 insertions(+), 59 deletions(-) diff --git a/src/main/java/com/uid2/operator/model/AdvertisingTokenInput.java b/src/main/java/com/uid2/operator/model/AdvertisingTokenInput.java index a9bcbc11f..0458498e9 100644 --- a/src/main/java/com/uid2/operator/model/AdvertisingTokenInput.java +++ b/src/main/java/com/uid2/operator/model/AdvertisingTokenInput.java @@ -3,17 +3,18 @@ import java.time.Instant; import com.uid2.operator.model.userIdentity.RawUidIdentity; +import com.uid2.operator.util.PrivacyBits; import com.uid2.shared.model.TokenVersion; public class AdvertisingTokenInput extends VersionedToken { public final OperatorIdentity operatorIdentity; public final SourcePublisher sourcePublisher; public final RawUidIdentity rawUidIdentity; - public final int privacyBits; + public final PrivacyBits privacyBits; public final Instant establishedAt; public AdvertisingTokenInput(TokenVersion version, Instant createdAt, Instant expiresAt, OperatorIdentity operatorIdentity, - SourcePublisher sourcePublisher, RawUidIdentity rawUidIdentity, int privacyBits, + SourcePublisher sourcePublisher, RawUidIdentity rawUidIdentity, PrivacyBits privacyBits, Instant establishedAt) { super(version, createdAt, expiresAt); this.operatorIdentity = operatorIdentity; diff --git a/src/main/java/com/uid2/operator/model/IdentityRequest.java b/src/main/java/com/uid2/operator/model/IdentityRequest.java index 877a95d72..84a16c698 100644 --- a/src/main/java/com/uid2/operator/model/IdentityRequest.java +++ b/src/main/java/com/uid2/operator/model/IdentityRequest.java @@ -10,14 +10,14 @@ public final class IdentityRequest { public final HashedDiiIdentity hashedDiiIdentity; public final OptoutCheckPolicy optoutCheckPolicy; - public final int privacyBits; + public final PrivacyBits privacyBits; public final Instant establishedAt; public IdentityRequest( SourcePublisher sourcePublisher, HashedDiiIdentity hashedDiiIdentity, OptoutCheckPolicy tokenGeneratePolicy, - int privacyBits, + PrivacyBits privacyBits, Instant establishedAt) { this.sourcePublisher = sourcePublisher; @@ -32,7 +32,7 @@ public IdentityRequest( HashedDiiIdentity hashedDiiIdentity, OptoutCheckPolicy tokenGeneratePolicy) { - this(sourcePublisher, hashedDiiIdentity, tokenGeneratePolicy, PrivacyBits.DEFAULT_PRIVACY_BIT_VALUE, Instant.now()); + this(sourcePublisher, hashedDiiIdentity, tokenGeneratePolicy, PrivacyBits.DEFAULT, Instant.now()); } diff --git a/src/main/java/com/uid2/operator/model/RefreshTokenInput.java b/src/main/java/com/uid2/operator/model/RefreshTokenInput.java index 9b45ca3e9..8abcf98bc 100644 --- a/src/main/java/com/uid2/operator/model/RefreshTokenInput.java +++ b/src/main/java/com/uid2/operator/model/RefreshTokenInput.java @@ -3,6 +3,7 @@ import java.time.Instant; import com.uid2.operator.model.userIdentity.FirstLevelHashIdentity; +import com.uid2.operator.util.PrivacyBits; import com.uid2.shared.model.TokenVersion; public class RefreshTokenInput extends VersionedToken { @@ -10,11 +11,11 @@ public class RefreshTokenInput extends VersionedToken { public final SourcePublisher sourcePublisher; public final FirstLevelHashIdentity firstLevelHashIdentity; // by default, inherited from the previous refresh token's privacy bits - public final int privacyBits; + public final PrivacyBits privacyBits; public RefreshTokenInput(TokenVersion version, Instant createdAt, Instant expiresAt, OperatorIdentity operatorIdentity, - SourcePublisher sourcePublisher, FirstLevelHashIdentity firstLevelHashIdentity, int privacyBits) { + SourcePublisher sourcePublisher, FirstLevelHashIdentity firstLevelHashIdentity, PrivacyBits privacyBits) { super(version, createdAt, expiresAt); this.operatorIdentity = operatorIdentity; this.sourcePublisher = sourcePublisher; diff --git a/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java b/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java index c60d3f96a..eb13b485d 100644 --- a/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java +++ b/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java @@ -3,6 +3,7 @@ import com.uid2.operator.model.*; import com.uid2.operator.model.userIdentity.FirstLevelHashIdentity; import com.uid2.operator.model.userIdentity.RawUidIdentity; +import com.uid2.operator.util.PrivacyBits; import com.uid2.operator.vertx.ClientInputValidationException; import com.uid2.shared.Const.Data; import com.uid2.shared.encryption.AesCbc; @@ -53,7 +54,7 @@ private byte[] encodeIntoAdvertisingTokenV2(AdvertisingTokenInput t, KeysetKey m private byte[] encodeIntoAdvertisingTokenV3(AdvertisingTokenInput t, KeysetKey masterKey, KeysetKey siteKey) { final Buffer sitePayload = Buffer.buffer(69); encodePublisherRequesterV3(sitePayload, t.sourcePublisher); - sitePayload.appendInt(t.privacyBits); + sitePayload.appendInt(t.privacyBits.getAsInt()); sitePayload.appendLong(t.establishedAt.toEpochMilli()); // this is the refreshedAt field in the spec - but effectively it is the time this advertising token is generated sitePayload.appendLong(t.createdAt.toEpochMilli()); @@ -120,7 +121,7 @@ private RefreshTokenInput decodeRefreshTokenV2(Buffer b) { throw new ClientInputValidationException("Failed to decode refreshTokenV2: Identity segment is not valid base64.", e); } - final int privacyBits = b2.getInt(8 + length); + final PrivacyBits privacyBits = PrivacyBits.fromInt(b2.getInt(8 + length)); final long establishedMillis = b2.getLong(8 + length + 4); return new RefreshTokenInput( @@ -147,7 +148,7 @@ private RefreshTokenInput decodeRefreshTokenV3(Buffer b, byte[] bytes) { final Instant createdAt = Instant.ofEpochMilli(b2.getLong(8)); final OperatorIdentity operatorIdentity = decodeOperatorIdentityV3(b2, 16); final SourcePublisher sourcePublisher = decodeSourcePublisherV3(b2, 29); - final int privacyBits = b2.getInt(45); + final PrivacyBits privacyBits = PrivacyBits.fromInt(b2.getInt(45)); final Instant establishedAt = Instant.ofEpochMilli(b2.getLong(49)); final IdentityScope identityScope = decodeIdentityScopeV3(b2.getByte(57)); final IdentityType identityType = decodeIdentityTypeV3(b2.getByte(57)); @@ -222,7 +223,7 @@ public AdvertisingTokenInput decodeAdvertisingTokenV2(Buffer b) { final byte[] rawUid = EncodingUtils.fromBase64(b3.slice(8, 8 + length).getBytes()); - final int privacyBits = b3.getInt(8 + length); + final PrivacyBits privacyBits = PrivacyBits.fromInt(b3.getInt(8 + length)); final long establishedMillis = b3.getLong(8 + length + 4); return new AdvertisingTokenInput( @@ -254,7 +255,7 @@ public AdvertisingTokenInput decodeAdvertisingTokenV3orV4(Buffer b, byte[] bytes final Buffer sitePayload = Buffer.buffer(AesGcm.decrypt(masterPayloadBytes, 33, this.keyManager.getKey(siteKeyId))); final SourcePublisher sourcePublisher = decodeSourcePublisherV3(sitePayload, 0); - final int privacyBits = sitePayload.getInt(16); + final PrivacyBits privacyBits = PrivacyBits.fromInt(sitePayload.getInt(16)); final Instant establishedAt = Instant.ofEpochMilli(sitePayload.getLong(20)); // refreshedAt is currently not used final Instant refreshedAt = Instant.ofEpochMilli(sitePayload.getLong(28)); @@ -322,7 +323,7 @@ public byte[] encodeIntoRefreshTokenV3(RefreshTokenInput t, KeysetKey serviceKey refreshPayload.appendLong(t.createdAt.toEpochMilli()); encodeOperatorIdentityV3(refreshPayload, t.operatorIdentity); encodePublisherRequesterV3(refreshPayload, t.sourcePublisher); - refreshPayload.appendInt(t.privacyBits); + refreshPayload.appendInt(t.privacyBits.getAsInt()); refreshPayload.appendLong(t.firstLevelHashIdentity.establishedAt.toEpochMilli()); refreshPayload.appendByte(encodeIdentityTypeV3(t.firstLevelHashIdentity.identityScope, t.firstLevelHashIdentity.identityType)); refreshPayload.appendBytes(t.firstLevelHashIdentity.firstLevelHash); @@ -337,7 +338,7 @@ public byte[] encodeIntoRefreshTokenV3(RefreshTokenInput t, KeysetKey serviceKey } private void encodeSiteIdentityV2(Buffer b, SourcePublisher sourcePublisher, RawUidIdentity rawUidIdentity, - KeysetKey siteEncryptionKey, int privacyBits, Instant establishedAt) { + KeysetKey siteEncryptionKey, PrivacyBits privacyBits, Instant establishedAt) { b.appendInt(siteEncryptionKey.getId()); final byte[] encryptedIdentity = encryptIdentityV2(sourcePublisher, rawUidIdentity, siteEncryptionKey, privacyBits, establishedAt); b.appendBytes(encryptedIdentity); @@ -372,19 +373,19 @@ private String generateAdvertisingTokenString(AdvertisingTokenInput advertisingT } private byte[] encryptIdentityV2(SourcePublisher sourcePublisher, FirstLevelHashIdentity firstLevelHashIdentity, - KeysetKey key, int privacyBits) { + KeysetKey key, PrivacyBits privacyBits) { return encryptIdentityV2(sourcePublisher, firstLevelHashIdentity.firstLevelHash, privacyBits, firstLevelHashIdentity.establishedAt, key); } private byte[] encryptIdentityV2(SourcePublisher sourcePublisher, RawUidIdentity rawUidIdentity, - KeysetKey key, int privacyBits, Instant establishedAt) { + KeysetKey key, PrivacyBits privacyBits, Instant establishedAt) { return encryptIdentityV2(sourcePublisher, rawUidIdentity.rawUid, privacyBits, establishedAt, key); } - private byte[] encryptIdentityV2(SourcePublisher sourcePublisher, byte[] id, int privacyBits, + private byte[] encryptIdentityV2(SourcePublisher sourcePublisher, byte[] id, PrivacyBits privacyBits, Instant establishedAt, KeysetKey key) { Buffer b = Buffer.buffer(); try { @@ -392,7 +393,7 @@ private byte[] encryptIdentityV2(SourcePublisher sourcePublisher, byte[] id, int final byte[] identityBytes = EncodingUtils.toBase64(id); b.appendInt(identityBytes.length); b.appendBytes(identityBytes); - b.appendInt(privacyBits); + b.appendInt(privacyBits.getAsInt()); b.appendLong(establishedAt.toEpochMilli()); return AesCbc.encrypt(b.getBytes(), key).getPayload(); } catch (Exception e) { diff --git a/src/main/java/com/uid2/operator/service/UIDOperatorService.java b/src/main/java/com/uid2/operator/service/UIDOperatorService.java index b76261f87..d1ac83cb6 100644 --- a/src/main/java/com/uid2/operator/service/UIDOperatorService.java +++ b/src/main/java/com/uid2/operator/service/UIDOperatorService.java @@ -136,8 +136,7 @@ public RefreshResponse refreshIdentity(RefreshTokenInput token) { return RefreshResponse.Expired; } - final PrivacyBits privacyBits = PrivacyBits.fromInt(token.privacyBits); - final boolean isCstg = privacyBits.isClientSideTokenGenerated(); + final boolean isCstg = token.privacyBits.isClientSideTokenGenerated(); try { final GlobalOptoutResult logoutEntry = getGlobalOptOutResult(token.firstLevelHashIdentity, true); @@ -252,7 +251,7 @@ private RawUidResponse generateRawUid(FirstLevelHashIdentity firstLevelHashIdent } private IdentityResponse generateIdentity(SourcePublisher sourcePublisher, - FirstLevelHashIdentity firstLevelHashIdentity, int privacyBits) { + FirstLevelHashIdentity firstLevelHashIdentity, PrivacyBits privacyBits) { final Instant nowUtc = EncodingUtils.NowUTCMillis(this.clock); final RawUidResponse rawUidResponse = generateRawUid(firstLevelHashIdentity, nowUtc); @@ -272,7 +271,7 @@ private IdentityResponse generateIdentity(SourcePublisher sourcePublisher, private RefreshTokenInput createRefreshTokenInput(SourcePublisher sourcePublisher, FirstLevelHashIdentity firstLevelHashIdentity, Instant now, - int privacyBits) { + PrivacyBits privacyBits) { return new RefreshTokenInput( this.refreshTokenVersion, now, @@ -284,7 +283,7 @@ private RefreshTokenInput createRefreshTokenInput(SourcePublisher sourcePublishe } private AdvertisingTokenInput createAdvertisingTokenInput(SourcePublisher sourcePublisher, RawUidIdentity rawUidIdentity, - Instant now, int privacyBits, Instant establishedAt) { + Instant now, PrivacyBits privacyBits, Instant establishedAt) { TokenVersion tokenVersion; if (siteIdsUsingV4Tokens.contains(sourcePublisher.siteId)) { tokenVersion = TokenVersion.V4; diff --git a/src/main/java/com/uid2/operator/util/PrivacyBits.java b/src/main/java/com/uid2/operator/util/PrivacyBits.java index 11edde226..41e3159ee 100644 --- a/src/main/java/com/uid2/operator/util/PrivacyBits.java +++ b/src/main/java/com/uid2/operator/util/PrivacyBits.java @@ -4,7 +4,7 @@ public class PrivacyBits { // For historical reason this bit is set - public static final int DEFAULT_PRIVACY_BIT_VALUE = 1; + public static final PrivacyBits DEFAULT = PrivacyBits.fromInt(1); private static final int BIT_LEGACY = 0; private static final int BIT_CSTG = 1; @@ -19,6 +19,24 @@ public class PrivacyBits { public PrivacyBits() { } + public PrivacyBits(PrivacyBits pb) { + bits = pb.bits; + } + + @Override + public boolean equals(Object obj) { + if (obj == null || !obj.getClass().equals(this.getClass())) { + return false; + } + PrivacyBits other = (PrivacyBits)obj; + return this.bits == other.bits; + } + + @Override + public int hashCode() { + return this.bits; + } + public PrivacyBits(int bits) { this.bits = bits; } diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index 828dd2c14..b68f4d04b 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -466,7 +466,7 @@ else if(emailHash != null) { new IdentityRequest( new SourcePublisher(clientSideKeypair.getSiteId(), 0, 0), input.toHashedDiiIdentity(this.identityScope), - OptoutCheckPolicy.RespectOptOut, privacyBits.getAsInt(), Instant.now())); + OptoutCheckPolicy.RespectOptOut, privacyBits, Instant.now())); } catch (KeyManager.NoActiveKeyException e){ SendServerErrorResponseAndRecordStats(rc, "No active encryption key available", clientSideKeypair.getSiteId(), TokenResponseStatsCollector.Endpoint.ClientSideTokenGenerateV2, TokenResponseStatsCollector.ResponseStatus.NoActiveKey, siteProvider, e, platformType); return; @@ -1013,7 +1013,7 @@ private void handleTokenGenerateV2(RoutingContext rc) { new IdentityRequest( new SourcePublisher(siteId, 0, 0), optOutTokenInput.toHashedDiiIdentity(this.identityScope), - OptoutCheckPolicy.DoNotRespect, pb.getAsInt(), Instant.now())); + OptoutCheckPolicy.DoNotRespect, pb, Instant.now())); ResponseUtil.SuccessV2(rc, optOutTokens.toJsonV1()); recordTokenResponseStats(siteId, TokenResponseStatsCollector.Endpoint.GenerateV2, TokenResponseStatsCollector.ResponseStatus.Success, siteProvider, optOutTokens.getAdvertisingTokenVersion(), platformType); diff --git a/src/test/java/com/uid2/operator/TokenEncodingTest.java b/src/test/java/com/uid2/operator/TokenEncodingTest.java index 2057b8891..484caae54 100644 --- a/src/test/java/com/uid2/operator/TokenEncodingTest.java +++ b/src/test/java/com/uid2/operator/TokenEncodingTest.java @@ -6,6 +6,7 @@ import com.uid2.operator.service.EncodingUtils; import com.uid2.operator.service.EncryptedTokenEncoder; import com.uid2.operator.service.TokenUtils; +import com.uid2.operator.util.PrivacyBits; import com.uid2.shared.Const.Data; import com.uid2.shared.model.TokenVersion; import com.uid2.shared.store.CloudPath; @@ -60,7 +61,7 @@ public void testRefreshTokenEncoding(TokenVersion tokenVersion) { new OperatorIdentity(101, OperatorType.Service, 102, 103), new SourcePublisher(111, 112, 113), new FirstLevelHashIdentity(IdentityScope.UID2, IdentityType.Email, firstLevelHash, now), - 121 + PrivacyBits.fromInt(121) ); if (tokenVersion == TokenVersion.V4) { @@ -103,7 +104,7 @@ public void testAdvertisingTokenEncodings(TokenVersion tokenVersion) { new OperatorIdentity(101, OperatorType.Service, 102, 103), new SourcePublisher(111, 112, 113), new RawUidIdentity(IdentityScope.UID2, IdentityType.Email, rawUid), - 121, + PrivacyBits.fromInt(121), now ); diff --git a/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java b/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java index 4d749f4d7..18b72e24c 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java @@ -6,6 +6,7 @@ import com.uid2.operator.model.userIdentity.UserIdentity; import com.uid2.operator.service.*; import com.uid2.operator.store.IOptOutStore; +import com.uid2.operator.util.PrivacyBits; import com.uid2.operator.vertx.OperatorShutdownHandler; import com.uid2.shared.store.CloudPath; import com.uid2.shared.store.RotatingSaltProvider; @@ -149,7 +150,7 @@ public void testGenerateAndRefresh(int siteId, TokenVersion tokenVersion) { final IdentityRequest identityRequest = new IdentityRequest( new SourcePublisher(siteId, 124, 125), createHashedDiiIdentity("test-email-hash", IdentityScope.UID2, IdentityType.Email), - OptoutCheckPolicy.DoNotRespect, 0, + OptoutCheckPolicy.DoNotRespect, PrivacyBits.fromInt(0), this.now.minusSeconds(234) ); final IdentityResponse identityResponse = uid2Service.generateIdentity(identityRequest); @@ -206,7 +207,7 @@ public void testTestOptOutKey_DoNotRespectOptout() { final IdentityRequest identityRequest = new IdentityRequest( new SourcePublisher(123, 124, 125), inputVal.toHashedDiiIdentity(IdentityScope.UID2), - OptoutCheckPolicy.DoNotRespect, 0, this.now + OptoutCheckPolicy.DoNotRespect, PrivacyBits.fromInt(0), this.now ); final IdentityResponse identityResponse = uid2Service.generateIdentity(identityRequest); verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); @@ -225,7 +226,7 @@ public void testTestOptOutKey_RespectOptout() { final IdentityRequest identityRequest = new IdentityRequest( new SourcePublisher(123, 124, 125), inputVal.toHashedDiiIdentity(IdentityScope.UID2), - OptoutCheckPolicy.RespectOptOut, 0, this.now + OptoutCheckPolicy.RespectOptOut, PrivacyBits.fromInt(0), this.now ); final IdentityResponse identityResponse = uid2Service.generateIdentity(identityRequest); assertTrue(identityResponse.isOptedOut()); @@ -241,7 +242,7 @@ public void testTestOptOutKeyIdentityScopeMismatch() { final IdentityRequest identityRequest = new IdentityRequest( new SourcePublisher(123, 124, 125), inputVal.toHashedDiiIdentity(IdentityScope.EUID), - OptoutCheckPolicy.DoNotRespect, 0, this.now + OptoutCheckPolicy.DoNotRespect, PrivacyBits.fromInt(0), this.now ); final IdentityResponse identityResponse = euidService.generateIdentity(identityRequest); verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); @@ -266,13 +267,13 @@ public void testGenerateTokenForOptOutUser(IdentityType type, String id, Identit final IdentityRequest identityRequestForceGenerate = new IdentityRequest( new SourcePublisher(123, 124, 125), hashedDiiIdentity, - OptoutCheckPolicy.DoNotRespect,0, + OptoutCheckPolicy.DoNotRespect, PrivacyBits.fromInt(0), this.now.minusSeconds(234)); final IdentityRequest identityRequestRespectOptOut = new IdentityRequest( new SourcePublisher(123, 124, 125), hashedDiiIdentity, - OptoutCheckPolicy.RespectOptOut, 0, + OptoutCheckPolicy.RespectOptOut, PrivacyBits.fromInt(0), this.now.minusSeconds(234)); // the clock value shouldn't matter here @@ -399,7 +400,7 @@ void testSpecialIdentityOptOutTokenGenerate(TestIdentityInputType type, String i final IdentityRequest identityRequest = new IdentityRequest( new SourcePublisher(123, 124, 125), inputVal.toHashedDiiIdentity(scope), - OptoutCheckPolicy.RespectOptOut, 0, this.now + OptoutCheckPolicy.RespectOptOut, PrivacyBits.fromInt(0), this.now ); // identity has no optout record, ensure generate still returns optout @@ -466,7 +467,7 @@ void testSpecialIdentityOptOutTokenRefresh(TestIdentityInputType type, String id final IdentityRequest identityRequest = new IdentityRequest( new SourcePublisher(123, 124, 125), inputVal.toHashedDiiIdentity(scope), - OptoutCheckPolicy.DoNotRespect, 0, this.now + OptoutCheckPolicy.DoNotRespect, PrivacyBits.fromInt(0), this.now ); IdentityResponse identityResponse; @@ -505,7 +506,7 @@ void testSpecialIdentityRefreshOptOutGenerate(TestIdentityInputType type, String final IdentityRequest identityRequest = new IdentityRequest( new SourcePublisher(123, 124, 125), inputVal.toHashedDiiIdentity(scope), - OptoutCheckPolicy.RespectOptOut, 0, this.now + OptoutCheckPolicy.RespectOptOut, PrivacyBits.fromInt(0), this.now ); // identity has optout record, ensure still generates @@ -580,7 +581,7 @@ void testSpecialIdentityValidateGenerate(TestIdentityInputType type, String id, final IdentityRequest identityRequest = new IdentityRequest( new SourcePublisher(123, 124, 125), inputVal.toHashedDiiIdentity(scope), - OptoutCheckPolicy.RespectOptOut, 0, this.now + OptoutCheckPolicy.RespectOptOut, PrivacyBits.fromInt(0), this.now ); // all identities have optout records, ensure validate identities still get generated @@ -709,7 +710,7 @@ void testExpiredSaltsNotifiesShutdownHandler(TestIdentityInputType type, String final IdentityRequest identityRequest = new IdentityRequest( new SourcePublisher(123, 124, 125), inputVal.toHashedDiiIdentity(scope), - OptoutCheckPolicy.RespectOptOut, 0, this.now); + OptoutCheckPolicy.RespectOptOut, PrivacyBits.fromInt(0), this.now); IdentityResponse identityResponse; AdvertisingTokenInput advertisingTokenInput; diff --git a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java index c0afecbc6..b936bcde7 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java @@ -1182,8 +1182,8 @@ void tokenGenerateOptOutToken(String policyParameterKey, String identity, Identi final String token = advertisingTokenString; final boolean matchedOptedOutIdentity = this.uidOperatorVerticle.getIdService().advertisingTokenMatches(token, optOutTokenInput.toHashedDiiIdentity(getIdentityScope()), now); assertTrue(matchedOptedOutIdentity); - assertFalse(PrivacyBits.fromInt(advertisingTokenInput.privacyBits).isClientSideTokenGenerated()); - assertTrue(PrivacyBits.fromInt(advertisingTokenInput.privacyBits).isClientSideTokenOptedOut()); + assertFalse(advertisingTokenInput.privacyBits.isClientSideTokenGenerated()); + assertTrue(advertisingTokenInput.privacyBits.isClientSideTokenOptedOut()); assertTokenStatusMetrics( 201, @@ -1229,8 +1229,8 @@ void tokenGenerateForEmail(String apiVersion, Vertx vertx, VertxTestContext test AdvertisingTokenInput advertisingTokenInput = validateAndGetToken(encoder, body, IdentityType.Email); - assertFalse(PrivacyBits.fromInt(advertisingTokenInput.privacyBits).isClientSideTokenGenerated()); - assertFalse(PrivacyBits.fromInt(advertisingTokenInput.privacyBits).isClientSideTokenOptedOut()); + assertFalse(advertisingTokenInput.privacyBits.isClientSideTokenGenerated()); + assertFalse(advertisingTokenInput.privacyBits.isClientSideTokenOptedOut()); assertEquals(clientSiteId, advertisingTokenInput.sourcePublisher.siteId); assertArrayEquals(getRawUidFromIdentity(IdentityType.Email, emailAddress, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenInput.rawUidIdentity.rawUid); @@ -1271,8 +1271,8 @@ void tokenGenerateForEmailHash(String apiVersion, Vertx vertx, VertxTestContext AdvertisingTokenInput advertisingTokenInput = validateAndGetToken(encoder, body, IdentityType.Email); - assertFalse(PrivacyBits.fromInt(advertisingTokenInput.privacyBits).isClientSideTokenGenerated()); - assertFalse(PrivacyBits.fromInt(advertisingTokenInput.privacyBits).isClientSideTokenOptedOut()); + assertFalse(advertisingTokenInput.privacyBits.isClientSideTokenGenerated()); + assertFalse(advertisingTokenInput.privacyBits.isClientSideTokenOptedOut()); assertEquals(clientSiteId, advertisingTokenInput.sourcePublisher.siteId); assertArrayEquals(getRawUidFromIdentityHash(IdentityType.Email, emailHash, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenInput.rawUidIdentity.rawUid); @@ -1315,8 +1315,8 @@ void tokenGenerateThenRefresh(String apiVersion, Vertx vertx, VertxTestContext t AdvertisingTokenInput advertisingTokenInput = validateAndGetToken(encoder, refreshBody, IdentityType.Email); - assertFalse(PrivacyBits.fromInt(advertisingTokenInput.privacyBits).isClientSideTokenGenerated()); - assertFalse(PrivacyBits.fromInt(advertisingTokenInput.privacyBits).isClientSideTokenOptedOut()); + assertFalse(advertisingTokenInput.privacyBits.isClientSideTokenGenerated()); + assertFalse(advertisingTokenInput.privacyBits.isClientSideTokenOptedOut()); assertEquals(clientSiteId, advertisingTokenInput.sourcePublisher.siteId); assertArrayEquals(getRawUidFromIdentity(IdentityType.Email, emailAddress, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenInput.rawUidIdentity.rawUid); @@ -1374,8 +1374,8 @@ void tokenGenerateThenRefreshSaltsExpired(String apiVersion, Vertx vertx, VertxT AdvertisingTokenInput advertisingTokenInput = validateAndGetToken(encoder, refreshBody, IdentityType.Email); - assertFalse(PrivacyBits.fromInt(advertisingTokenInput.privacyBits).isClientSideTokenGenerated()); - assertFalse(PrivacyBits.fromInt(advertisingTokenInput.privacyBits).isClientSideTokenOptedOut()); + assertFalse(advertisingTokenInput.privacyBits.isClientSideTokenGenerated()); + assertFalse(advertisingTokenInput.privacyBits.isClientSideTokenOptedOut()); assertEquals(clientSiteId, advertisingTokenInput.sourcePublisher.siteId); assertArrayEquals(getRawUidFromIdentity(IdentityType.Email, emailAddress, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenInput.rawUidIdentity.rawUid); @@ -1586,8 +1586,8 @@ void tokenGenerateSaltsExpired(String apiVersion, Vertx vertx, VertxTestContext AdvertisingTokenInput advertisingTokenInput = validateAndGetToken(encoder, body, IdentityType.Email); - assertFalse(PrivacyBits.fromInt(advertisingTokenInput.privacyBits).isClientSideTokenGenerated()); - assertFalse(PrivacyBits.fromInt(advertisingTokenInput.privacyBits).isClientSideTokenOptedOut()); + assertFalse(advertisingTokenInput.privacyBits.isClientSideTokenGenerated()); + assertFalse(advertisingTokenInput.privacyBits.isClientSideTokenOptedOut()); assertEquals(clientSiteId, advertisingTokenInput.sourcePublisher.siteId); assertArrayEquals(getRawUidFromIdentity(IdentityType.Email, emailAddress, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenInput.rawUidIdentity.rawUid); @@ -2498,8 +2498,8 @@ void tokenGenerateForPhone(String apiVersion, Vertx vertx, VertxTestContext test AdvertisingTokenInput advertisingTokenInput = validateAndGetToken(encoder, body, IdentityType.Phone); - assertFalse(PrivacyBits.fromInt(advertisingTokenInput.privacyBits).isClientSideTokenGenerated()); - assertFalse(PrivacyBits.fromInt(advertisingTokenInput.privacyBits).isClientSideTokenOptedOut()); + assertFalse(advertisingTokenInput.privacyBits.isClientSideTokenGenerated()); + assertFalse(advertisingTokenInput.privacyBits.isClientSideTokenOptedOut()); assertEquals(clientSiteId, advertisingTokenInput.sourcePublisher.siteId); assertArrayEquals(getRawUidFromIdentity(IdentityType.Phone, phone, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenInput.rawUidIdentity.rawUid); @@ -2537,8 +2537,8 @@ void tokenGenerateForPhoneHash(String apiVersion, Vertx vertx, VertxTestContext AdvertisingTokenInput advertisingTokenInput = validateAndGetToken(encoder, body, IdentityType.Phone); - assertFalse(PrivacyBits.fromInt(advertisingTokenInput.privacyBits).isClientSideTokenGenerated()); - assertFalse(PrivacyBits.fromInt(advertisingTokenInput.privacyBits).isClientSideTokenOptedOut()); + assertFalse(advertisingTokenInput.privacyBits.isClientSideTokenGenerated()); + assertFalse(advertisingTokenInput.privacyBits.isClientSideTokenOptedOut()); assertEquals(clientSiteId, advertisingTokenInput.sourcePublisher.siteId); assertArrayEquals(getRawUidFromIdentity(IdentityType.Phone, phone, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenInput.rawUidIdentity.rawUid); @@ -2581,8 +2581,8 @@ void tokenGenerateThenRefreshForPhone(String apiVersion, Vertx vertx, VertxTestC AdvertisingTokenInput advertisingTokenInput = validateAndGetToken(encoder, refreshBody, IdentityType.Phone); - assertFalse(PrivacyBits.fromInt(advertisingTokenInput.privacyBits).isClientSideTokenGenerated()); - assertFalse(PrivacyBits.fromInt(advertisingTokenInput.privacyBits).isClientSideTokenOptedOut()); + assertFalse(advertisingTokenInput.privacyBits.isClientSideTokenGenerated()); + assertFalse(advertisingTokenInput.privacyBits.isClientSideTokenOptedOut()); assertEquals(clientSiteId, advertisingTokenInput.sourcePublisher.siteId); assertArrayEquals(getRawUidFromIdentity(IdentityType.Phone, phone, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenInput.rawUidIdentity.rawUid); @@ -4202,8 +4202,8 @@ private void assertAreClientSideGeneratedTokens(AdvertisingTokenInput advertisin } private void assertAreClientSideGeneratedTokens(AdvertisingTokenInput advertisingTokenInput, RefreshTokenInput refreshTokenInput, int siteId, IdentityType identityType, String identity, boolean expectedOptOut) { - final PrivacyBits advertisingTokenPrivacyBits = PrivacyBits.fromInt(advertisingTokenInput.privacyBits); - final PrivacyBits refreshTokenPrivacyBits = PrivacyBits.fromInt(refreshTokenInput.privacyBits); + final PrivacyBits advertisingTokenPrivacyBits = advertisingTokenInput.privacyBits; + final PrivacyBits refreshTokenPrivacyBits = refreshTokenInput.privacyBits; final byte[] rawUid = getRawUidFromIdentity(identityType, identity, From 741237d8e5b2448c066ffef3f78b97535e8287a2 Mon Sep 17 00:00:00 2001 From: Sunny Wu Date: Mon, 28 Oct 2024 13:56:55 +1100 Subject: [PATCH 096/431] Replace privacy bits type to PrivacyBits class instance --- .../com/uid2/operator/PrivacyBitsTest.java | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 src/test/java/com/uid2/operator/PrivacyBitsTest.java diff --git a/src/test/java/com/uid2/operator/PrivacyBitsTest.java b/src/test/java/com/uid2/operator/PrivacyBitsTest.java new file mode 100644 index 000000000..15cea9f9a --- /dev/null +++ b/src/test/java/com/uid2/operator/PrivacyBitsTest.java @@ -0,0 +1,64 @@ +package com.uid2.operator; + +import com.uid2.operator.service.EncodingUtils; +import com.uid2.operator.util.PrivacyBits; +import com.uid2.shared.cloud.EmbeddedResourceStorage; +import com.uid2.shared.store.CloudPath; +import com.uid2.shared.store.reader.RotatingClientKeyProvider; +import com.uid2.shared.store.scope.GlobalScope; +import io.vertx.core.json.JsonObject; +import org.junit.Test; + +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; + +import static org.junit.Assert.*; + + +public class PrivacyBitsTest { + @Test + public void generateNewClientKeys() throws NoSuchAlgorithmException { + assertEquals(PrivacyBits.DEFAULT.getAsInt(), 1); + PrivacyBits pb1 = new PrivacyBits(); + assertEquals(pb1.getAsInt(), 0); + assertEquals(pb1.hashCode(), 0); + assertNotEquals(pb1, PrivacyBits.fromInt(1)); + assertNotEquals(pb1, PrivacyBits.fromInt(121)); + assertFalse(pb1.isClientSideTokenGenerated()); + assertFalse(pb1.isClientSideTokenOptedOut()); + + pb1.setLegacyBit(); + assertEquals(pb1.getAsInt(), 0b1); + assertEquals(pb1.hashCode(), 0b1); + assertEquals(pb1, PrivacyBits.fromInt(1)); + assertNotEquals(pb1, PrivacyBits.fromInt(121)); + assertFalse(pb1.isClientSideTokenGenerated()); + assertFalse(pb1.isClientSideTokenOptedOut()); + + + pb1.setClientSideTokenGenerate(); + assertEquals(pb1.getAsInt(), 0b11); + assertEquals(pb1.hashCode(), 0b11); + assertEquals(pb1, PrivacyBits.fromInt(3)); + assertNotEquals(pb1, PrivacyBits.fromInt(121)); + assertTrue(pb1.isClientSideTokenGenerated()); + assertFalse(pb1.isClientSideTokenOptedOut()); + + + pb1.setClientSideTokenGenerateOptout(); + assertEquals(pb1.getAsInt(), 0b111); + assertEquals(pb1.hashCode(), 0b111); + assertEquals(pb1, PrivacyBits.fromInt(7)); + assertNotEquals(pb1, PrivacyBits.fromInt(121)); + assertTrue(pb1.isClientSideTokenGenerated()); + assertTrue(pb1.isClientSideTokenOptedOut()); + + PrivacyBits pb2 = new PrivacyBits(pb1); + assertEquals(pb2.getAsInt(), 0b111); + + PrivacyBits pb3 = PrivacyBits.fromInt(0b10110); + assertEquals(pb3.getAsInt(), 0b10110); + pb3.setLegacyBit(); + assertEquals(pb3.getAsInt(), 0b10111); + } +} From 39000173ab7d7d9254e945fe4ba79faaa9cbfade Mon Sep 17 00:00:00 2001 From: Sunny Wu Date: Mon, 28 Oct 2024 13:57:27 +1100 Subject: [PATCH 097/431] Replace privacy bits type to PrivacyBits class instance --- src/test/java/com/uid2/operator/PrivacyBitsTest.java | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/test/java/com/uid2/operator/PrivacyBitsTest.java b/src/test/java/com/uid2/operator/PrivacyBitsTest.java index 15cea9f9a..e22ac6845 100644 --- a/src/test/java/com/uid2/operator/PrivacyBitsTest.java +++ b/src/test/java/com/uid2/operator/PrivacyBitsTest.java @@ -1,17 +1,8 @@ package com.uid2.operator; -import com.uid2.operator.service.EncodingUtils; import com.uid2.operator.util.PrivacyBits; -import com.uid2.shared.cloud.EmbeddedResourceStorage; -import com.uid2.shared.store.CloudPath; -import com.uid2.shared.store.reader.RotatingClientKeyProvider; -import com.uid2.shared.store.scope.GlobalScope; -import io.vertx.core.json.JsonObject; import org.junit.Test; - import java.security.NoSuchAlgorithmException; -import java.security.SecureRandom; - import static org.junit.Assert.*; From e5b104dbfe68a667ba2fca8d63f96e35d71369f0 Mon Sep 17 00:00:00 2001 From: Sunny Wu Date: Mon, 28 Oct 2024 14:49:04 +1100 Subject: [PATCH 098/431] missing a newline --- src/test/java/com/uid2/operator/UIDOperatorServiceTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java b/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java index 18b72e24c..223b0625c 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java @@ -159,7 +159,8 @@ public void testGenerateAndRefresh(int siteId, TokenVersion tokenVersion) { assertNotNull(identityResponse); UIDOperatorVerticleTest.validateAdvertisingToken(identityResponse.getAdvertisingToken(), tokenVersion, IdentityScope.UID2, IdentityType.Email); - AdvertisingTokenInput advertisingTokenInput = tokenEncoder.decodeAdvertisingToken(identityResponse.getAdvertisingToken());assertEquals(this.now.plusSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS), advertisingTokenInput.expiresAt); + AdvertisingTokenInput advertisingTokenInput = tokenEncoder.decodeAdvertisingToken(identityResponse.getAdvertisingToken()); + assertEquals(this.now.plusSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS), advertisingTokenInput.expiresAt); assertEquals(identityRequest.sourcePublisher.siteId, advertisingTokenInput.sourcePublisher.siteId); assertIdentityScopeIdentityType(identityRequest.hashedDiiIdentity, advertisingTokenInput.rawUidIdentity); assertEquals(identityRequest.establishedAt, advertisingTokenInput.establishedAt); From e25cd19d6446115cbac3714d2288b905b7c531e9 Mon Sep 17 00:00:00 2001 From: Sunny Wu Date: Mon, 28 Oct 2024 16:13:58 +1100 Subject: [PATCH 099/431] 1. Added PrivacyBitsTest/IdentityResponseTest/RawUidResponseTest classes 2. added some extra check for firstLevelHash in existing tests --- .../uid2/operator/UIDOperatorServiceTest.java | 14 +++++- .../utilTests/IdentityResponseTest.java | 48 +++++++++++++++++++ .../{ => utilTests}/PrivacyBitsTest.java | 4 +- .../utilTests/RawUidResponseTest.java | 30 ++++++++++++ 4 files changed, 92 insertions(+), 4 deletions(-) create mode 100644 src/test/java/com/uid2/operator/utilTests/IdentityResponseTest.java rename src/test/java/com/uid2/operator/{ => utilTests}/PrivacyBitsTest.java (94%) create mode 100644 src/test/java/com/uid2/operator/utilTests/RawUidResponseTest.java diff --git a/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java b/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java index 223b0625c..0f2a19714 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java @@ -19,6 +19,8 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; + +import static com.uid2.operator.service.TokenUtils.getFirstLevelHash; import static org.junit.jupiter.api.Assertions.*; import org.junit.jupiter.params.ParameterizedTest; @@ -46,7 +48,7 @@ public class UIDOperatorServiceTest { UIDOperatorService uid2Service; UIDOperatorService euidService; Instant now; - + RotatingSaltProvider saltProvider; final int IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS = 600; final int REFRESH_TOKEN_EXPIRES_AFTER_SECONDS = 900; final int REFRESH_IDENTITY_TOKEN_AFTER_SECONDS = 300; @@ -67,7 +69,7 @@ void setup() throws Exception { new GlobalScope(new CloudPath("/com.uid2.core/test/keysets/metadata.json"))); keysetProvider.loadContent(); - RotatingSaltProvider saltProvider = new RotatingSaltProvider( + saltProvider = new RotatingSaltProvider( new EmbeddedResourceStorage(Main.class), "/com.uid2.core/test/salts/metadata.json"); saltProvider.loadContent(); @@ -164,6 +166,7 @@ public void testGenerateAndRefresh(int siteId, TokenVersion tokenVersion) { assertEquals(identityRequest.sourcePublisher.siteId, advertisingTokenInput.sourcePublisher.siteId); assertIdentityScopeIdentityType(identityRequest.hashedDiiIdentity, advertisingTokenInput.rawUidIdentity); assertEquals(identityRequest.establishedAt, advertisingTokenInput.establishedAt); + assertEquals(identityRequest.privacyBits, advertisingTokenInput.privacyBits); RefreshTokenInput refreshTokenInput = tokenEncoder.decodeRefreshToken(identityResponse.getRefreshToken()); assertEquals(this.now, refreshTokenInput.createdAt); @@ -172,6 +175,11 @@ public void testGenerateAndRefresh(int siteId, TokenVersion tokenVersion) { assertIdentityScopeIdentityType(identityRequest.hashedDiiIdentity, refreshTokenInput.firstLevelHashIdentity); assertEquals(identityRequest.establishedAt, refreshTokenInput.firstLevelHashIdentity.establishedAt); + final byte[] firstLevelHash = getFirstLevelHash(identityRequest.hashedDiiIdentity.hashedDii, + saltProvider.getSnapshot(this.now).getFirstLevelSalt() ); + assertArrayEquals(firstLevelHash, refreshTokenInput.firstLevelHashIdentity.firstLevelHash); + + setNow(Instant.now().plusSeconds(200)); reset(shutdownHandler); @@ -191,6 +199,7 @@ public void testGenerateAndRefresh(int siteId, TokenVersion tokenVersion) { assertEquals(advertisingTokenInput.establishedAt, advertisingTokenInput2.establishedAt); assertArrayEquals(advertisingTokenInput.rawUidIdentity.rawUid, advertisingTokenInput2.rawUidIdentity.rawUid); + assertEquals(identityRequest.privacyBits, advertisingTokenInput2.privacyBits); RefreshTokenInput refreshTokenInput2 = tokenEncoder.decodeRefreshToken(refreshResponse.getIdentityResponse().getRefreshToken()); assertEquals(this.now, refreshTokenInput2.createdAt); @@ -199,6 +208,7 @@ public void testGenerateAndRefresh(int siteId, TokenVersion tokenVersion) { assertIdentityScopeIdentityType(refreshTokenInput.firstLevelHashIdentity, refreshTokenInput2.firstLevelHashIdentity); assertEquals(refreshTokenInput.firstLevelHashIdentity.establishedAt, refreshTokenInput2.firstLevelHashIdentity.establishedAt); assertArrayEquals(refreshTokenInput.firstLevelHashIdentity.firstLevelHash, refreshTokenInput2.firstLevelHashIdentity.firstLevelHash); + assertArrayEquals(firstLevelHash, refreshTokenInput2.firstLevelHashIdentity.firstLevelHash); } @Test diff --git a/src/test/java/com/uid2/operator/utilTests/IdentityResponseTest.java b/src/test/java/com/uid2/operator/utilTests/IdentityResponseTest.java new file mode 100644 index 000000000..5a9b4023c --- /dev/null +++ b/src/test/java/com/uid2/operator/utilTests/IdentityResponseTest.java @@ -0,0 +1,48 @@ +package com.uid2.operator.utilTests; + +import com.uid2.operator.model.IdentityResponse; +import com.uid2.shared.model.TokenVersion; +import io.vertx.core.json.JsonObject; +import org.junit.Test; + +import java.security.NoSuchAlgorithmException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; + +import static org.junit.Assert.*; + + +public class IdentityResponseTest { + @Test + public void doIdentityResponseTest() throws NoSuchAlgorithmException { + assertEquals(IdentityResponse.OptOutIdentityResponse.getAdvertisingToken(), ""); + assertTrue(IdentityResponse.OptOutIdentityResponse.isOptedOut()); + + IdentityResponse nullAdTokenValue = new IdentityResponse(null, TokenVersion.V4, "refreshToken", null,null,null); + assertTrue(nullAdTokenValue.isOptedOut()); + + Instant identityExpires = Instant.now(); + Instant refreshFrom = identityExpires.plus(5, ChronoUnit.MINUTES); + Instant refreshExpires = identityExpires.plus(10, ChronoUnit.MINUTES); + IdentityResponse response1 = new IdentityResponse("adToken", TokenVersion.V3, "refreshToken", identityExpires + , refreshExpires, refreshFrom); + + assertEquals(response1.getAdvertisingToken(), "adToken"); + assertEquals(response1.getAdvertisingTokenVersion(), TokenVersion.V3); + assertEquals(response1.getRefreshToken(), "refreshToken"); + assertEquals(response1.getIdentityExpires(), identityExpires); + assertEquals(response1.getRefreshExpires(), refreshExpires); + assertEquals(response1.getRefreshFrom(), refreshFrom); + + JsonObject jsonV1 = response1.toJsonV1(); + assertEquals(jsonV1.getString("advertising_token"), response1.getAdvertisingToken()); + assertEquals(jsonV1.getString("refresh_token"), response1.getRefreshToken()); + assertEquals(jsonV1.getLong("refresh_expires").longValue(), response1.getRefreshExpires().toEpochMilli()); + assertEquals(jsonV1.getLong("refresh_from").longValue(), response1.getRefreshFrom().toEpochMilli()); + + JsonObject jsonV0 = response1.toJsonV0(); + assertEquals(jsonV0.getString("advertisement_token"), response1.getAdvertisingToken()); + assertEquals(jsonV0.getString("advertising_token"), response1.getAdvertisingToken()); + assertEquals(jsonV0.getString("refresh_token"), response1.getRefreshToken()); + } +} diff --git a/src/test/java/com/uid2/operator/PrivacyBitsTest.java b/src/test/java/com/uid2/operator/utilTests/PrivacyBitsTest.java similarity index 94% rename from src/test/java/com/uid2/operator/PrivacyBitsTest.java rename to src/test/java/com/uid2/operator/utilTests/PrivacyBitsTest.java index e22ac6845..91750bf87 100644 --- a/src/test/java/com/uid2/operator/PrivacyBitsTest.java +++ b/src/test/java/com/uid2/operator/utilTests/PrivacyBitsTest.java @@ -1,4 +1,4 @@ -package com.uid2.operator; +package com.uid2.operator.utilTests; import com.uid2.operator.util.PrivacyBits; import org.junit.Test; @@ -8,7 +8,7 @@ public class PrivacyBitsTest { @Test - public void generateNewClientKeys() throws NoSuchAlgorithmException { + public void doPrivacyBitsTest() throws NoSuchAlgorithmException { assertEquals(PrivacyBits.DEFAULT.getAsInt(), 1); PrivacyBits pb1 = new PrivacyBits(); assertEquals(pb1.getAsInt(), 0); diff --git a/src/test/java/com/uid2/operator/utilTests/RawUidResponseTest.java b/src/test/java/com/uid2/operator/utilTests/RawUidResponseTest.java new file mode 100644 index 000000000..4e759b853 --- /dev/null +++ b/src/test/java/com/uid2/operator/utilTests/RawUidResponseTest.java @@ -0,0 +1,30 @@ +package com.uid2.operator.utilTests; + +import com.uid2.operator.model.RawUidResponse; +import org.junit.Test; + +import java.security.NoSuchAlgorithmException; +import java.util.Arrays; + +import static org.junit.Assert.*; + + +public class RawUidResponseTest { + @Test + public void doRawUidResponseTest() throws NoSuchAlgorithmException { + assertEquals(RawUidResponse.OptoutIdentity.bucketId, ""); + assertTrue(RawUidResponse.OptoutIdentity.isOptedOut()); + + RawUidResponse optoutResponse = new RawUidResponse(new byte[33], null); + assertTrue(optoutResponse.isOptedOut()); + + byte[] rawUid = new byte[33]; + for(int i = 0; i < 33; i++) { + rawUid[i] = (byte) i; + } + + RawUidResponse generatedUid = new RawUidResponse(rawUid, "12345"); + assertFalse(generatedUid.isOptedOut()); + assertTrue(Arrays.equals(rawUid, generatedUid.rawUid)); + } +} From 986cf2e03c816626f876e4a30e162f6b88b4ea20 Mon Sep 17 00:00:00 2001 From: Sunny Wu Date: Mon, 28 Oct 2024 18:01:19 +1100 Subject: [PATCH 100/431] Created verifyFirstLevelHashIdentityAndEstablishedAt method for verifying FirstLevelHashIdentity and added more checks for FirstLevel and check the correctness of raw Uid/First level hash in various unit tests --- .../operator/UIDOperatorVerticleTest.java | 38 +++++++++++++++---- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java index b936bcde7..0049a0d7b 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java @@ -1236,7 +1236,7 @@ void tokenGenerateForEmail(String apiVersion, Vertx vertx, VertxTestContext test RefreshTokenInput refreshTokenInput = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token")); assertEquals(clientSiteId, refreshTokenInput.sourcePublisher.siteId); - assertArrayEquals(TokenUtils.getFirstLevelHashFromIdentity(emailAddress, firstLevelSalt), refreshTokenInput.firstLevelHashIdentity.firstLevelHash); + verifyFirstLevelHashIdentityAndEstablishedAt(TokenUtils.getFirstLevelHashFromIdentity(emailAddress, firstLevelSalt), refreshTokenInput, body, now); assertEqualsClose(now.plusMillis(identityExpiresAfter.toMillis()), Instant.ofEpochMilli(body.getLong("identity_expires")), 10); assertEqualsClose(now.plusMillis(refreshExpiresAfter.toMillis()), Instant.ofEpochMilli(body.getLong("refresh_expires")), 10); @@ -1278,7 +1278,7 @@ void tokenGenerateForEmailHash(String apiVersion, Vertx vertx, VertxTestContext RefreshTokenInput refreshTokenInput = decodeRefreshToken(encoder, apiVersion.equals("v2") ? body.getString("decrypted_refresh_token") : body.getString("refresh_token")); assertEquals(clientSiteId, refreshTokenInput.sourcePublisher.siteId); - assertArrayEquals(TokenUtils.getFirstLevelHashFromIdentityHash(emailHash, firstLevelSalt), refreshTokenInput.firstLevelHashIdentity.firstLevelHash); + verifyFirstLevelHashIdentityAndEstablishedAt(TokenUtils.getFirstLevelHashFromIdentityHash(emailHash, firstLevelSalt), refreshTokenInput, body, now); assertEqualsClose(now.plusMillis(identityExpiresAfter.toMillis()), Instant.ofEpochMilli(body.getLong("identity_expires")), 10); assertEqualsClose(now.plusMillis(refreshExpiresAfter.toMillis()), Instant.ofEpochMilli(body.getLong("refresh_expires")), 10); @@ -1303,6 +1303,8 @@ void tokenGenerateThenRefresh(String apiVersion, Vertx vertx, VertxTestContext t assertNotNull(bodyJson); String genRefreshToken = bodyJson.getString("refresh_token"); + EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); + RefreshTokenInput firstRefreshTokenInput = decodeRefreshToken(encoder, bodyJson.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token")); when(this.optOutStore.getLatestEntry(any())).thenReturn(null); @@ -1311,7 +1313,7 @@ void tokenGenerateThenRefresh(String apiVersion, Vertx vertx, VertxTestContext t assertEquals("success", refreshRespJson.getString("status")); JsonObject refreshBody = refreshRespJson.getJsonObject("body"); assertNotNull(refreshBody); - EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); + AdvertisingTokenInput advertisingTokenInput = validateAndGetToken(encoder, refreshBody, IdentityType.Email); @@ -1323,6 +1325,11 @@ void tokenGenerateThenRefresh(String apiVersion, Vertx vertx, VertxTestContext t String refreshTokenStringNew = refreshBody.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"); assertNotEquals(genRefreshToken, refreshTokenStringNew); RefreshTokenInput refreshTokenInput = decodeRefreshToken(encoder, refreshTokenStringNew); + + assertEquals(firstRefreshTokenInput.firstLevelHashIdentity.establishedAt, + refreshTokenInput.firstLevelHashIdentity.establishedAt); + assertTrue(firstRefreshTokenInput.firstLevelHashIdentity.matches(refreshTokenInput.firstLevelHashIdentity)); + assertEquals(clientSiteId, refreshTokenInput.sourcePublisher.siteId); assertArrayEquals(TokenUtils.getFirstLevelHashFromIdentity(emailAddress, firstLevelSalt), refreshTokenInput.firstLevelHashIdentity.firstLevelHash); @@ -2504,8 +2511,10 @@ void tokenGenerateForPhone(String apiVersion, Vertx vertx, VertxTestContext test assertArrayEquals(getRawUidFromIdentity(IdentityType.Phone, phone, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenInput.rawUidIdentity.rawUid); RefreshTokenInput refreshTokenInput = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), IdentityType.Phone); + assertEquals(clientSiteId, refreshTokenInput.sourcePublisher.siteId); - assertArrayEquals(TokenUtils.getFirstLevelHashFromIdentity(phone, firstLevelSalt), refreshTokenInput.firstLevelHashIdentity.firstLevelHash); + verifyFirstLevelHashIdentityAndEstablishedAt(TokenUtils.getFirstLevelHashFromIdentity(phone, + firstLevelSalt), refreshTokenInput, body, now); assertEqualsClose(now.plusMillis(identityExpiresAfter.toMillis()), Instant.ofEpochMilli(body.getLong("identity_expires")), 10); assertEqualsClose(now.plusMillis(refreshExpiresAfter.toMillis()), Instant.ofEpochMilli(body.getLong("refresh_expires")), 10); @@ -2515,6 +2524,18 @@ void tokenGenerateForPhone(String apiVersion, Vertx vertx, VertxTestContext test }); } + void verifyFirstLevelHashIdentityAndEstablishedAt(byte[] expectedFirstLevelHash, + RefreshTokenInput refreshTokenInput, + JsonObject receivedJsonBody, + Instant establishedTime) { + + assertArrayEquals(expectedFirstLevelHash, refreshTokenInput.firstLevelHashIdentity.firstLevelHash); + assertEqualsClose(establishedTime, refreshTokenInput.firstLevelHashIdentity.establishedAt, 10); + assertTrue(refreshTokenInput.firstLevelHashIdentity.establishedAt.toEpochMilli() < receivedJsonBody.getLong("identity_expires") ); + assertTrue(refreshTokenInput.firstLevelHashIdentity.establishedAt.toEpochMilli() < receivedJsonBody.getLong("refresh_expires") ); + assertTrue(refreshTokenInput.firstLevelHashIdentity.establishedAt.toEpochMilli() < receivedJsonBody.getLong("refresh_from") ); + } + @ParameterizedTest @ValueSource(strings = {"v1", "v2"}) void tokenGenerateForPhoneHash(String apiVersion, Vertx vertx, VertxTestContext testContext) { @@ -2544,7 +2565,7 @@ void tokenGenerateForPhoneHash(String apiVersion, Vertx vertx, VertxTestContext RefreshTokenInput refreshTokenInput = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), IdentityType.Phone); assertEquals(clientSiteId, refreshTokenInput.sourcePublisher.siteId); - assertArrayEquals(TokenUtils.getFirstLevelHashFromIdentity(phone, firstLevelSalt), refreshTokenInput.firstLevelHashIdentity.firstLevelHash); + verifyFirstLevelHashIdentityAndEstablishedAt(TokenUtils.getFirstLevelHashFromIdentity(phone, firstLevelSalt), refreshTokenInput, body, now); assertEqualsClose(now.plusMillis(identityExpiresAfter.toMillis()), Instant.ofEpochMilli(body.getLong("identity_expires")), 10); assertEqualsClose(now.plusMillis(refreshExpiresAfter.toMillis()), Instant.ofEpochMilli(body.getLong("refresh_expires")), 10); @@ -2590,7 +2611,8 @@ void tokenGenerateThenRefreshForPhone(String apiVersion, Vertx vertx, VertxTestC assertNotEquals(genRefreshToken, refreshTokenStringNew); RefreshTokenInput refreshTokenInput = decodeRefreshToken(encoder, refreshTokenStringNew, IdentityType.Phone); assertEquals(clientSiteId, refreshTokenInput.sourcePublisher.siteId); - assertArrayEquals(TokenUtils.getFirstLevelHashFromIdentity(phone, firstLevelSalt), refreshTokenInput.firstLevelHashIdentity.firstLevelHash); + verifyFirstLevelHashIdentityAndEstablishedAt(TokenUtils.getFirstLevelHashFromIdentity(phone, + firstLevelSalt), refreshTokenInput, refreshBody, now); assertEqualsClose(now.plusMillis(identityExpiresAfter.toMillis()), Instant.ofEpochMilli(refreshBody.getLong("identity_expires")), 10); assertEqualsClose(now.plusMillis(refreshExpiresAfter.toMillis()), Instant.ofEpochMilli(refreshBody.getLong("refresh_expires")), 10); @@ -4070,7 +4092,7 @@ void cstgSuccessForBothOptedAndNonOptedOutTest(boolean optOutExpected, String id AdvertisingTokenInput advertisingTokenInput = validateAndGetToken(encoder, genBody, identityType); RefreshTokenInput refreshTokenInput = decodeRefreshToken(encoder, genBody.getString("decrypted_refresh_token"), identityType); - + verifyFirstLevelHashIdentityAndEstablishedAt(TokenUtils.getFirstLevelHashFromIdentity(id, firstLevelSalt), refreshTokenInput, genBody, now); assertAreClientSideGeneratedTokens(advertisingTokenInput, refreshTokenInput, clientSideTokenGenerateSiteId, identityType, id); assertEqualsClose(Instant.now().plusMillis(identityExpiresAfter.toMillis()), Instant.ofEpochMilli(genBody.getLong("identity_expires")), 10); assertEqualsClose(Instant.now().plusMillis(refreshExpiresAfter.toMillis()), Instant.ofEpochMilli(genBody.getLong("refresh_expires")), 10); @@ -4098,6 +4120,8 @@ void cstgSuccessForBothOptedAndNonOptedOutTest(boolean optOutExpected, String id assertNotEquals(genRefreshToken, refreshTokenStringNew); RefreshTokenInput refreshTokenAfterRefreshSource = decodeRefreshToken(encoder, refreshTokenStringNew, identityType); + verifyFirstLevelHashIdentityAndEstablishedAt(TokenUtils.getFirstLevelHashFromIdentity(id, firstLevelSalt), refreshTokenAfterRefreshSource, refreshBody, now); + assertAreClientSideGeneratedTokens(adTokenFromRefresh, refreshTokenAfterRefreshSource, clientSideTokenGenerateSiteId, identityType, id); assertEqualsClose(Instant.now().plusMillis(identityExpiresAfter.toMillis()), Instant.ofEpochMilli(refreshBody.getLong("identity_expires")), 10); assertEqualsClose(Instant.now().plusMillis(refreshExpiresAfter.toMillis()), Instant.ofEpochMilli(refreshBody.getLong("refresh_expires")), 10); From f445180b4b471152013fa29050c0110b95f2fa25 Mon Sep 17 00:00:00 2001 From: Sunny Wu Date: Mon, 28 Oct 2024 18:02:16 +1100 Subject: [PATCH 101/431] Created verifyFirstLevelHashIdentityAndEstablishedAt method for verifying FirstLevelHashIdentity and added more checks for FirstLevel and check the correctness of raw Uid/First level hash in various unit tests --- .../uid2/operator/UIDOperatorVerticleTest.java | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java index 0049a0d7b..b28655600 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java @@ -1304,6 +1304,10 @@ void tokenGenerateThenRefresh(String apiVersion, Vertx vertx, VertxTestContext t String genRefreshToken = bodyJson.getString("refresh_token"); EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); + + AdvertisingTokenInput firstAdvertisingTokenInput = validateAndGetToken(encoder, bodyJson, + IdentityType.Email); + RefreshTokenInput firstRefreshTokenInput = decodeRefreshToken(encoder, bodyJson.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token")); when(this.optOutStore.getLatestEntry(any())).thenReturn(null); @@ -1316,19 +1320,29 @@ void tokenGenerateThenRefresh(String apiVersion, Vertx vertx, VertxTestContext t AdvertisingTokenInput advertisingTokenInput = validateAndGetToken(encoder, refreshBody, IdentityType.Email); - assertFalse(advertisingTokenInput.privacyBits.isClientSideTokenGenerated()); assertFalse(advertisingTokenInput.privacyBits.isClientSideTokenOptedOut()); assertEquals(clientSiteId, advertisingTokenInput.sourcePublisher.siteId); + + // checks that the raw Uid from raw input, from original tokgen/generate response and the subsequent + // token/refresh response are identical assertArrayEquals(getRawUidFromIdentity(IdentityType.Email, emailAddress, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenInput.rawUidIdentity.rawUid); + assertArrayEquals(firstAdvertisingTokenInput.rawUidIdentity.rawUid, + advertisingTokenInput.rawUidIdentity.rawUid); + String refreshTokenStringNew = refreshBody.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"); assertNotEquals(genRefreshToken, refreshTokenStringNew); RefreshTokenInput refreshTokenInput = decodeRefreshToken(encoder, refreshTokenStringNew); + // check that first level hash from token/generate's refresh token and token/refresh's refresh token + // are identical assertEquals(firstRefreshTokenInput.firstLevelHashIdentity.establishedAt, refreshTokenInput.firstLevelHashIdentity.establishedAt); assertTrue(firstRefreshTokenInput.firstLevelHashIdentity.matches(refreshTokenInput.firstLevelHashIdentity)); + + verifyFirstLevelHashIdentityAndEstablishedAt(TokenUtils.getFirstLevelHashFromIdentity(emailAddress, + firstLevelSalt), refreshTokenInput, refreshBody, now); assertEquals(clientSiteId, refreshTokenInput.sourcePublisher.siteId); assertArrayEquals(TokenUtils.getFirstLevelHashFromIdentity(emailAddress, firstLevelSalt), refreshTokenInput.firstLevelHashIdentity.firstLevelHash); From fa1c0f198ad1305270ff22b7d1e66a296e3fd3f5 Mon Sep 17 00:00:00 2001 From: Sunny Wu Date: Mon, 28 Oct 2024 18:46:04 +1100 Subject: [PATCH 102/431] Checking privacy bits in more tests in UIDOperatorVerticleTest --- .../operator/UIDOperatorVerticleTest.java | 36 +++++++++++++++++-- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java index b28655600..cb30b0ffd 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java @@ -1228,7 +1228,6 @@ void tokenGenerateForEmail(String apiVersion, Vertx vertx, VertxTestContext test EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); AdvertisingTokenInput advertisingTokenInput = validateAndGetToken(encoder, body, IdentityType.Email); - assertFalse(advertisingTokenInput.privacyBits.isClientSideTokenGenerated()); assertFalse(advertisingTokenInput.privacyBits.isClientSideTokenOptedOut()); assertEquals(clientSiteId, advertisingTokenInput.sourcePublisher.siteId); @@ -1236,6 +1235,9 @@ void tokenGenerateForEmail(String apiVersion, Vertx vertx, VertxTestContext test RefreshTokenInput refreshTokenInput = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token")); assertEquals(clientSiteId, refreshTokenInput.sourcePublisher.siteId); + + verifyPrivacyBits(PrivacyBits.DEFAULT, advertisingTokenInput, refreshTokenInput); + verifyFirstLevelHashIdentityAndEstablishedAt(TokenUtils.getFirstLevelHashFromIdentity(emailAddress, firstLevelSalt), refreshTokenInput, body, now); assertEqualsClose(now.plusMillis(identityExpiresAfter.toMillis()), Instant.ofEpochMilli(body.getLong("identity_expires")), 10); @@ -1248,6 +1250,12 @@ void tokenGenerateForEmail(String apiVersion, Vertx vertx, VertxTestContext test }); } + public void verifyPrivacyBits(PrivacyBits expectedValue, AdvertisingTokenInput advertisingTokenInput, + RefreshTokenInput refreshTokenInput) { + assertEquals(advertisingTokenInput.privacyBits, expectedValue); + assertEquals(advertisingTokenInput.privacyBits, refreshTokenInput.privacyBits); + } + @ParameterizedTest @ValueSource(strings = {"v1", "v2"}) void tokenGenerateForEmailHash(String apiVersion, Vertx vertx, VertxTestContext testContext) { @@ -1278,6 +1286,8 @@ void tokenGenerateForEmailHash(String apiVersion, Vertx vertx, VertxTestContext RefreshTokenInput refreshTokenInput = decodeRefreshToken(encoder, apiVersion.equals("v2") ? body.getString("decrypted_refresh_token") : body.getString("refresh_token")); assertEquals(clientSiteId, refreshTokenInput.sourcePublisher.siteId); + + verifyPrivacyBits(PrivacyBits.DEFAULT, advertisingTokenInput, refreshTokenInput); verifyFirstLevelHashIdentityAndEstablishedAt(TokenUtils.getFirstLevelHashFromIdentityHash(emailHash, firstLevelSalt), refreshTokenInput, body, now); assertEqualsClose(now.plusMillis(identityExpiresAfter.toMillis()), Instant.ofEpochMilli(body.getLong("identity_expires")), 10); @@ -1340,7 +1350,10 @@ void tokenGenerateThenRefresh(String apiVersion, Vertx vertx, VertxTestContext t assertEquals(firstRefreshTokenInput.firstLevelHashIdentity.establishedAt, refreshTokenInput.firstLevelHashIdentity.establishedAt); assertTrue(firstRefreshTokenInput.firstLevelHashIdentity.matches(refreshTokenInput.firstLevelHashIdentity)); - + + verifyPrivacyBits(PrivacyBits.DEFAULT, firstAdvertisingTokenInput, refreshTokenInput); + verifyPrivacyBits(PrivacyBits.DEFAULT, advertisingTokenInput, refreshTokenInput); + verifyFirstLevelHashIdentityAndEstablishedAt(TokenUtils.getFirstLevelHashFromIdentity(emailAddress, firstLevelSalt), refreshTokenInput, refreshBody, now); @@ -2527,6 +2540,7 @@ void tokenGenerateForPhone(String apiVersion, Vertx vertx, VertxTestContext test RefreshTokenInput refreshTokenInput = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), IdentityType.Phone); assertEquals(clientSiteId, refreshTokenInput.sourcePublisher.siteId); + verifyPrivacyBits(PrivacyBits.DEFAULT, advertisingTokenInput, refreshTokenInput); verifyFirstLevelHashIdentityAndEstablishedAt(TokenUtils.getFirstLevelHashFromIdentity(phone, firstLevelSalt), refreshTokenInput, body, now); @@ -2579,6 +2593,8 @@ void tokenGenerateForPhoneHash(String apiVersion, Vertx vertx, VertxTestContext RefreshTokenInput refreshTokenInput = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), IdentityType.Phone); assertEquals(clientSiteId, refreshTokenInput.sourcePublisher.siteId); + + verifyPrivacyBits(PrivacyBits.DEFAULT, advertisingTokenInput, refreshTokenInput); verifyFirstLevelHashIdentityAndEstablishedAt(TokenUtils.getFirstLevelHashFromIdentity(phone, firstLevelSalt), refreshTokenInput, body, now); assertEqualsClose(now.plusMillis(identityExpiresAfter.toMillis()), Instant.ofEpochMilli(body.getLong("identity_expires")), 10); @@ -2603,6 +2619,9 @@ void tokenGenerateThenRefreshForPhone(String apiVersion, Vertx vertx, VertxTestC JsonObject bodyJson = genRespJson.getJsonObject("body"); assertNotNull(bodyJson); + EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); + AdvertisingTokenInput firstAdvertisingTokenInput = validateAndGetToken(encoder, bodyJson, + IdentityType.Phone); String genRefreshToken = bodyJson.getString("refresh_token"); when(this.optOutStore.getLatestEntry(any())).thenReturn(null); @@ -2612,7 +2631,7 @@ void tokenGenerateThenRefreshForPhone(String apiVersion, Vertx vertx, VertxTestC assertEquals("success", refreshRespJson.getString("status")); JsonObject refreshBody = refreshRespJson.getJsonObject("body"); assertNotNull(refreshBody); - EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); + AdvertisingTokenInput advertisingTokenInput = validateAndGetToken(encoder, refreshBody, IdentityType.Phone); @@ -2625,6 +2644,9 @@ void tokenGenerateThenRefreshForPhone(String apiVersion, Vertx vertx, VertxTestC assertNotEquals(genRefreshToken, refreshTokenStringNew); RefreshTokenInput refreshTokenInput = decodeRefreshToken(encoder, refreshTokenStringNew, IdentityType.Phone); assertEquals(clientSiteId, refreshTokenInput.sourcePublisher.siteId); + + verifyPrivacyBits(PrivacyBits.DEFAULT, firstAdvertisingTokenInput, refreshTokenInput); + verifyPrivacyBits(PrivacyBits.DEFAULT, advertisingTokenInput, refreshTokenInput); verifyFirstLevelHashIdentityAndEstablishedAt(TokenUtils.getFirstLevelHashFromIdentity(phone, firstLevelSalt), refreshTokenInput, refreshBody, now); @@ -4106,7 +4128,13 @@ void cstgSuccessForBothOptedAndNonOptedOutTest(boolean optOutExpected, String id AdvertisingTokenInput advertisingTokenInput = validateAndGetToken(encoder, genBody, identityType); RefreshTokenInput refreshTokenInput = decodeRefreshToken(encoder, genBody.getString("decrypted_refresh_token"), identityType); + + PrivacyBits pb = new PrivacyBits(); + pb.setLegacyBit(); + pb.setClientSideTokenGenerate(); + verifyPrivacyBits(pb, advertisingTokenInput, refreshTokenInput); verifyFirstLevelHashIdentityAndEstablishedAt(TokenUtils.getFirstLevelHashFromIdentity(id, firstLevelSalt), refreshTokenInput, genBody, now); + assertAreClientSideGeneratedTokens(advertisingTokenInput, refreshTokenInput, clientSideTokenGenerateSiteId, identityType, id); assertEqualsClose(Instant.now().plusMillis(identityExpiresAfter.toMillis()), Instant.ofEpochMilli(genBody.getLong("identity_expires")), 10); assertEqualsClose(Instant.now().plusMillis(refreshExpiresAfter.toMillis()), Instant.ofEpochMilli(genBody.getLong("refresh_expires")), 10); @@ -4134,6 +4162,8 @@ void cstgSuccessForBothOptedAndNonOptedOutTest(boolean optOutExpected, String id assertNotEquals(genRefreshToken, refreshTokenStringNew); RefreshTokenInput refreshTokenAfterRefreshSource = decodeRefreshToken(encoder, refreshTokenStringNew, identityType); + verifyPrivacyBits(pb, adTokenFromRefresh, refreshTokenInput); + verifyPrivacyBits(pb, adTokenFromRefresh, refreshTokenAfterRefreshSource); verifyFirstLevelHashIdentityAndEstablishedAt(TokenUtils.getFirstLevelHashFromIdentity(id, firstLevelSalt), refreshTokenAfterRefreshSource, refreshBody, now); assertAreClientSideGeneratedTokens(adTokenFromRefresh, refreshTokenAfterRefreshSource, clientSideTokenGenerateSiteId, identityType, id); From d745cc77e17404281307612077178613366ba493 Mon Sep 17 00:00:00 2001 From: asloob qureshi Date: Mon, 28 Oct 2024 16:45:01 -0700 Subject: [PATCH 103/431] Change optout loading exception to a warning temporarily --- .../java/com/uid2/operator/store/CloudSyncOptOutStore.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/uid2/operator/store/CloudSyncOptOutStore.java b/src/main/java/com/uid2/operator/store/CloudSyncOptOutStore.java index 49f7d0385..1bc882c73 100644 --- a/src/main/java/com/uid2/operator/store/CloudSyncOptOutStore.java +++ b/src/main/java/com/uid2/operator/store/CloudSyncOptOutStore.java @@ -545,8 +545,8 @@ else if (OptOutUtils.isPartitionFile(f)) if (tsOld != Instant.EPOCH && tsNew != Instant.EPOCH && !tsOld.isBefore(tsNew)) { final String errorMsg = "Last partition timestamp of indexed files " + tsOld.getEpochSecond() + " is after last partition of non-indexed files " + tsNew.getEpochSecond(); - LOGGER.error(errorMsg); - throw new IllegalStateException(errorMsg); + // Leaving this as a warning until issue is fixed permanently + LOGGER.warn(errorMsg); } // if there are new partitions in this update, let index delete some in-mem delta caches that is old if (tsNew != Instant.EPOCH) { From 20d695c8c72f6da4d1d45d0abab6ee0fe63a2a19 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Tue, 29 Oct 2024 02:02:10 +0000 Subject: [PATCH 104/431] [CI Pipeline] Released Snapshot version: 5.41.1-alpha-116-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 4dddd15aa..428f9c4cb 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.41.0 + 5.41.1-alpha-116-SNAPSHOT UTF-8 From c950c6d9b1c0426db343e16532b315a6fdab2bdb Mon Sep 17 00:00:00 2001 From: Sunny Wu Date: Wed, 30 Oct 2024 14:04:37 +1100 Subject: [PATCH 105/431] revered version in pom.xml --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 428f9c4cb..4dddd15aa 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.41.1-alpha-116-SNAPSHOT + 5.41.0 UTF-8 From a908e1f6b0e8b0f6897d7971051a73737d635b83 Mon Sep 17 00:00:00 2001 From: Sunny Wu Date: Wed, 30 Oct 2024 16:25:45 +1100 Subject: [PATCH 106/431] added some comments and renamed refreshIdentity method param to input instead of oken --- .../uid2/operator/model/IdentityResponse.java | 3 +++ .../userIdentity/FirstLevelHashIdentity.java | 1 + .../service/EncryptedTokenEncoder.java | 1 + .../operator/service/IUIDOperatorService.java | 2 +- .../operator/service/UIDOperatorService.java | 20 +++++++++---------- 5 files changed, 16 insertions(+), 11 deletions(-) diff --git a/src/main/java/com/uid2/operator/model/IdentityResponse.java b/src/main/java/com/uid2/operator/model/IdentityResponse.java index 8b3ad4a43..45d8b4a59 100644 --- a/src/main/java/com/uid2/operator/model/IdentityResponse.java +++ b/src/main/java/com/uid2/operator/model/IdentityResponse.java @@ -9,9 +9,12 @@ // jsonified public class IdentityResponse { public static IdentityResponse OptOutIdentityResponse = new IdentityResponse("", null, "", Instant.EPOCH, Instant.EPOCH, Instant.EPOCH); + + //aka UID token private final String advertisingToken; private final TokenVersion advertisingTokenVersion; private final String refreshToken; + // when the advertising token/uid token expires private final Instant identityExpires; private final Instant refreshExpires; private final Instant refreshFrom; diff --git a/src/main/java/com/uid2/operator/model/userIdentity/FirstLevelHashIdentity.java b/src/main/java/com/uid2/operator/model/userIdentity/FirstLevelHashIdentity.java index b549833ed..4df2b1c6c 100644 --- a/src/main/java/com/uid2/operator/model/userIdentity/FirstLevelHashIdentity.java +++ b/src/main/java/com/uid2/operator/model/userIdentity/FirstLevelHashIdentity.java @@ -21,6 +21,7 @@ public FirstLevelHashIdentity(IdentityScope identityScope, IdentityType identity this.establishedAt = establishedAt; } + // explicitly not checking establishedAt - this is only for making sure the first level hash matches a new input public boolean matches(FirstLevelHashIdentity that) { return this.identityScope.equals(that.identityScope) && this.identityType.equals(that.identityType) && diff --git a/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java b/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java index eb13b485d..85961981d 100644 --- a/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java +++ b/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java @@ -57,6 +57,7 @@ private byte[] encodeIntoAdvertisingTokenV3(AdvertisingTokenInput t, KeysetKey m sitePayload.appendInt(t.privacyBits.getAsInt()); sitePayload.appendLong(t.establishedAt.toEpochMilli()); // this is the refreshedAt field in the spec - but effectively it is the time this advertising token is generated + // this is a redundant field as it is stored in master payload again, can consider dropping this field in future token version sitePayload.appendLong(t.createdAt.toEpochMilli()); sitePayload.appendBytes(t.rawUidIdentity.rawUid); // 32 or 33 bytes diff --git a/src/main/java/com/uid2/operator/service/IUIDOperatorService.java b/src/main/java/com/uid2/operator/service/IUIDOperatorService.java index 38624848d..dc6671cef 100644 --- a/src/main/java/com/uid2/operator/service/IUIDOperatorService.java +++ b/src/main/java/com/uid2/operator/service/IUIDOperatorService.java @@ -14,7 +14,7 @@ public interface IUIDOperatorService { IdentityResponse generateIdentity(IdentityRequest request); - RefreshResponse refreshIdentity(RefreshTokenInput refreshTokenInput); + RefreshResponse refreshIdentity(RefreshTokenInput input); RawUidResponse mapIdentity(MapRequest request); diff --git a/src/main/java/com/uid2/operator/service/UIDOperatorService.java b/src/main/java/com/uid2/operator/service/UIDOperatorService.java index 2b36b1b3c..f5ba469b9 100644 --- a/src/main/java/com/uid2/operator/service/UIDOperatorService.java +++ b/src/main/java/com/uid2/operator/service/UIDOperatorService.java @@ -120,34 +120,34 @@ public IdentityResponse generateIdentity(IdentityRequest request) { } @Override - public RefreshResponse refreshIdentity(RefreshTokenInput token) { + public RefreshResponse refreshIdentity(RefreshTokenInput input) { // should not be possible as different scopes should be using different keys, but just in case - if (token.firstLevelHashIdentity.identityScope != this.identityScope) { + if (input.firstLevelHashIdentity.identityScope != this.identityScope) { return RefreshResponse.Invalid; } - if (token.firstLevelHashIdentity.establishedAt.isBefore(RefreshCutoff)) { + if (input.firstLevelHashIdentity.establishedAt.isBefore(RefreshCutoff)) { return RefreshResponse.Deprecated; } final Instant now = clock.instant(); - if (token.expiresAt.isBefore(now)) { + if (input.expiresAt.isBefore(now)) { return RefreshResponse.Expired; } - final boolean isCstg = token.privacyBits.isClientSideTokenGenerated(); + final boolean isCstg = input.privacyBits.isClientSideTokenGenerated(); try { - final GlobalOptoutResult logoutEntry = getGlobalOptOutResult(token.firstLevelHashIdentity, true); + final GlobalOptoutResult logoutEntry = getGlobalOptOutResult(input.firstLevelHashIdentity, true); final boolean optedOut = logoutEntry.isOptedOut(); - final Duration durationSinceLastRefresh = Duration.between(token.createdAt, now); + final Duration durationSinceLastRefresh = Duration.between(input.createdAt, now); if (!optedOut) { - IdentityResponse identityResponse = this.generateIdentity(token.sourcePublisher, - token.firstLevelHashIdentity, - token.privacyBits); + IdentityResponse identityResponse = this.generateIdentity(input.sourcePublisher, + input.firstLevelHashIdentity, + input.privacyBits); return RefreshResponse.createRefreshedResponse(identityResponse, durationSinceLastRefresh, isCstg); } else { From 04dfc147100a71e171829e82ae9fbb182a89f36d Mon Sep 17 00:00:00 2001 From: Sunny Wu Date: Wed, 30 Oct 2024 17:30:09 +1100 Subject: [PATCH 107/431] fixed unit tests checking establishedAt between advertisingTokenInput & refreshTokenInput's --- .../com/uid2/operator/util/PrivacyBits.java | 3 +++ .../uid2/operator/UIDOperatorServiceTest.java | 4 +--- .../operator/UIDOperatorVerticleTest.java | 24 +++++++++++-------- .../utilTests/IdentityResponseTest.java | 4 +++- 4 files changed, 21 insertions(+), 14 deletions(-) diff --git a/src/main/java/com/uid2/operator/util/PrivacyBits.java b/src/main/java/com/uid2/operator/util/PrivacyBits.java index 41e3159ee..0df69d7fd 100644 --- a/src/main/java/com/uid2/operator/util/PrivacyBits.java +++ b/src/main/java/com/uid2/operator/util/PrivacyBits.java @@ -58,6 +58,9 @@ public boolean isClientSideTokenOptedOut() { public void setLegacyBit() { setBit(BIT_LEGACY);//unknown why this bit is set in https://github.com/IABTechLab/uid2-operator/blob/dbab58346e367c9d4122ad541ff9632dc37bd410/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java#L534 } + public boolean isLegacyBitSet() { + return isBitSet(BIT_LEGACY); + } private void setBit(int position) { bits |= (1 << position); diff --git a/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java b/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java index 644ed3d07..794228db8 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java @@ -7,7 +7,6 @@ import com.uid2.operator.service.*; import com.uid2.operator.service.EncodingUtils; import com.uid2.operator.service.EncryptedTokenEncoder; -import com.uid2.operator.service.ITokenEncoder; import com.uid2.operator.service.InputUtil; import com.uid2.operator.service.UIDOperatorService; import com.uid2.operator.store.IOptOutStore; @@ -337,7 +336,7 @@ public void testGenerateTokenForOptOutUser(IdentityType type, String id, Identit assertNotNull(advertisingTokenInput.rawUidIdentity); assertNotNull(identityResponseAfterOptOut); assertTrue(identityResponseAfterOptOut.getAdvertisingToken() == null || identityResponseAfterOptOut.getAdvertisingToken().isEmpty()); - + assertTrue(identityResponseAfterOptOut.isOptedOut()); } @ParameterizedTest @@ -464,7 +463,6 @@ void testSpecialIdentityOptOutIdentityMap(TestIdentityInputType type, String id, final MapRequest mapRequestRespectOptOut = new MapRequest( inputVal.toHashedDiiIdentity(scope), -// inputVal.toHashedDiiIdentity(scope, 0, this.now), OptoutCheckPolicy.RespectOptOut, now); diff --git a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java index cb30b0ffd..a8bddccfc 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java @@ -1238,7 +1238,7 @@ void tokenGenerateForEmail(String apiVersion, Vertx vertx, VertxTestContext test verifyPrivacyBits(PrivacyBits.DEFAULT, advertisingTokenInput, refreshTokenInput); - verifyFirstLevelHashIdentityAndEstablishedAt(TokenUtils.getFirstLevelHashFromIdentity(emailAddress, firstLevelSalt), refreshTokenInput, body, now); + verifyFirstLevelHashIdentityAndEstablishedAt(TokenUtils.getFirstLevelHashFromIdentity(emailAddress, firstLevelSalt), refreshTokenInput, body, advertisingTokenInput.establishedAt); assertEqualsClose(now.plusMillis(identityExpiresAfter.toMillis()), Instant.ofEpochMilli(body.getLong("identity_expires")), 10); assertEqualsClose(now.plusMillis(refreshExpiresAfter.toMillis()), Instant.ofEpochMilli(body.getLong("refresh_expires")), 10); @@ -1288,7 +1288,7 @@ void tokenGenerateForEmailHash(String apiVersion, Vertx vertx, VertxTestContext assertEquals(clientSiteId, refreshTokenInput.sourcePublisher.siteId); verifyPrivacyBits(PrivacyBits.DEFAULT, advertisingTokenInput, refreshTokenInput); - verifyFirstLevelHashIdentityAndEstablishedAt(TokenUtils.getFirstLevelHashFromIdentityHash(emailHash, firstLevelSalt), refreshTokenInput, body, now); + verifyFirstLevelHashIdentityAndEstablishedAt(TokenUtils.getFirstLevelHashFromIdentityHash(emailHash, firstLevelSalt), refreshTokenInput, body, advertisingTokenInput.establishedAt); assertEqualsClose(now.plusMillis(identityExpiresAfter.toMillis()), Instant.ofEpochMilli(body.getLong("identity_expires")), 10); assertEqualsClose(now.plusMillis(refreshExpiresAfter.toMillis()), Instant.ofEpochMilli(body.getLong("refresh_expires")), 10); @@ -1320,6 +1320,8 @@ void tokenGenerateThenRefresh(String apiVersion, Vertx vertx, VertxTestContext t RefreshTokenInput firstRefreshTokenInput = decodeRefreshToken(encoder, bodyJson.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token")); + assertEquals(firstAdvertisingTokenInput.establishedAt, firstRefreshTokenInput.firstLevelHashIdentity.establishedAt); + when(this.optOutStore.getLatestEntry(any())).thenReturn(null); sendTokenRefresh(apiVersion, vertx, ClientVersionHeader, iosClientVersionHeaderValue, testContext, genRefreshToken, bodyJson.getString("refresh_response_key"), 200, refreshRespJson -> @@ -1355,7 +1357,7 @@ void tokenGenerateThenRefresh(String apiVersion, Vertx vertx, VertxTestContext t verifyPrivacyBits(PrivacyBits.DEFAULT, advertisingTokenInput, refreshTokenInput); verifyFirstLevelHashIdentityAndEstablishedAt(TokenUtils.getFirstLevelHashFromIdentity(emailAddress, - firstLevelSalt), refreshTokenInput, refreshBody, now); + firstLevelSalt), refreshTokenInput, refreshBody, firstAdvertisingTokenInput.establishedAt); assertEquals(clientSiteId, refreshTokenInput.sourcePublisher.siteId); assertArrayEquals(TokenUtils.getFirstLevelHashFromIdentity(emailAddress, firstLevelSalt), refreshTokenInput.firstLevelHashIdentity.firstLevelHash); @@ -1620,6 +1622,8 @@ void tokenGenerateSaltsExpired(String apiVersion, Vertx vertx, VertxTestContext AdvertisingTokenInput advertisingTokenInput = validateAndGetToken(encoder, body, IdentityType.Email); + assertTrue(advertisingTokenInput.privacyBits.isLegacyBitSet()); + assertEquals(advertisingTokenInput.privacyBits, PrivacyBits.DEFAULT); assertFalse(advertisingTokenInput.privacyBits.isClientSideTokenGenerated()); assertFalse(advertisingTokenInput.privacyBits.isClientSideTokenOptedOut()); assertEquals(clientSiteId, advertisingTokenInput.sourcePublisher.siteId); @@ -2542,7 +2546,7 @@ void tokenGenerateForPhone(String apiVersion, Vertx vertx, VertxTestContext test assertEquals(clientSiteId, refreshTokenInput.sourcePublisher.siteId); verifyPrivacyBits(PrivacyBits.DEFAULT, advertisingTokenInput, refreshTokenInput); verifyFirstLevelHashIdentityAndEstablishedAt(TokenUtils.getFirstLevelHashFromIdentity(phone, - firstLevelSalt), refreshTokenInput, body, now); + firstLevelSalt), refreshTokenInput, body, advertisingTokenInput.establishedAt); assertEqualsClose(now.plusMillis(identityExpiresAfter.toMillis()), Instant.ofEpochMilli(body.getLong("identity_expires")), 10); assertEqualsClose(now.plusMillis(refreshExpiresAfter.toMillis()), Instant.ofEpochMilli(body.getLong("refresh_expires")), 10); @@ -2555,10 +2559,10 @@ void tokenGenerateForPhone(String apiVersion, Vertx vertx, VertxTestContext test void verifyFirstLevelHashIdentityAndEstablishedAt(byte[] expectedFirstLevelHash, RefreshTokenInput refreshTokenInput, JsonObject receivedJsonBody, - Instant establishedTime) { + Instant expectedEstablishedTime) { assertArrayEquals(expectedFirstLevelHash, refreshTokenInput.firstLevelHashIdentity.firstLevelHash); - assertEqualsClose(establishedTime, refreshTokenInput.firstLevelHashIdentity.establishedAt, 10); + assertEquals(expectedEstablishedTime, refreshTokenInput.firstLevelHashIdentity.establishedAt); assertTrue(refreshTokenInput.firstLevelHashIdentity.establishedAt.toEpochMilli() < receivedJsonBody.getLong("identity_expires") ); assertTrue(refreshTokenInput.firstLevelHashIdentity.establishedAt.toEpochMilli() < receivedJsonBody.getLong("refresh_expires") ); assertTrue(refreshTokenInput.firstLevelHashIdentity.establishedAt.toEpochMilli() < receivedJsonBody.getLong("refresh_from") ); @@ -2595,7 +2599,7 @@ void tokenGenerateForPhoneHash(String apiVersion, Vertx vertx, VertxTestContext assertEquals(clientSiteId, refreshTokenInput.sourcePublisher.siteId); verifyPrivacyBits(PrivacyBits.DEFAULT, advertisingTokenInput, refreshTokenInput); - verifyFirstLevelHashIdentityAndEstablishedAt(TokenUtils.getFirstLevelHashFromIdentity(phone, firstLevelSalt), refreshTokenInput, body, now); + verifyFirstLevelHashIdentityAndEstablishedAt(TokenUtils.getFirstLevelHashFromIdentity(phone, firstLevelSalt), refreshTokenInput, body, advertisingTokenInput.establishedAt); assertEqualsClose(now.plusMillis(identityExpiresAfter.toMillis()), Instant.ofEpochMilli(body.getLong("identity_expires")), 10); assertEqualsClose(now.plusMillis(refreshExpiresAfter.toMillis()), Instant.ofEpochMilli(body.getLong("refresh_expires")), 10); @@ -2648,7 +2652,7 @@ void tokenGenerateThenRefreshForPhone(String apiVersion, Vertx vertx, VertxTestC verifyPrivacyBits(PrivacyBits.DEFAULT, firstAdvertisingTokenInput, refreshTokenInput); verifyPrivacyBits(PrivacyBits.DEFAULT, advertisingTokenInput, refreshTokenInput); verifyFirstLevelHashIdentityAndEstablishedAt(TokenUtils.getFirstLevelHashFromIdentity(phone, - firstLevelSalt), refreshTokenInput, refreshBody, now); + firstLevelSalt), refreshTokenInput, refreshBody, advertisingTokenInput.establishedAt); assertEqualsClose(now.plusMillis(identityExpiresAfter.toMillis()), Instant.ofEpochMilli(refreshBody.getLong("identity_expires")), 10); assertEqualsClose(now.plusMillis(refreshExpiresAfter.toMillis()), Instant.ofEpochMilli(refreshBody.getLong("refresh_expires")), 10); @@ -4133,7 +4137,7 @@ void cstgSuccessForBothOptedAndNonOptedOutTest(boolean optOutExpected, String id pb.setLegacyBit(); pb.setClientSideTokenGenerate(); verifyPrivacyBits(pb, advertisingTokenInput, refreshTokenInput); - verifyFirstLevelHashIdentityAndEstablishedAt(TokenUtils.getFirstLevelHashFromIdentity(id, firstLevelSalt), refreshTokenInput, genBody, now); + verifyFirstLevelHashIdentityAndEstablishedAt(TokenUtils.getFirstLevelHashFromIdentity(id, firstLevelSalt), refreshTokenInput, genBody, advertisingTokenInput.establishedAt); assertAreClientSideGeneratedTokens(advertisingTokenInput, refreshTokenInput, clientSideTokenGenerateSiteId, identityType, id); assertEqualsClose(Instant.now().plusMillis(identityExpiresAfter.toMillis()), Instant.ofEpochMilli(genBody.getLong("identity_expires")), 10); @@ -4164,7 +4168,7 @@ void cstgSuccessForBothOptedAndNonOptedOutTest(boolean optOutExpected, String id verifyPrivacyBits(pb, adTokenFromRefresh, refreshTokenInput); verifyPrivacyBits(pb, adTokenFromRefresh, refreshTokenAfterRefreshSource); - verifyFirstLevelHashIdentityAndEstablishedAt(TokenUtils.getFirstLevelHashFromIdentity(id, firstLevelSalt), refreshTokenAfterRefreshSource, refreshBody, now); + verifyFirstLevelHashIdentityAndEstablishedAt(TokenUtils.getFirstLevelHashFromIdentity(id, firstLevelSalt), refreshTokenAfterRefreshSource, refreshBody, advertisingTokenInput.establishedAt); assertAreClientSideGeneratedTokens(adTokenFromRefresh, refreshTokenAfterRefreshSource, clientSideTokenGenerateSiteId, identityType, id); assertEqualsClose(Instant.now().plusMillis(identityExpiresAfter.toMillis()), Instant.ofEpochMilli(refreshBody.getLong("identity_expires")), 10); diff --git a/src/test/java/com/uid2/operator/utilTests/IdentityResponseTest.java b/src/test/java/com/uid2/operator/utilTests/IdentityResponseTest.java index 5a9b4023c..c8ff6feed 100644 --- a/src/test/java/com/uid2/operator/utilTests/IdentityResponseTest.java +++ b/src/test/java/com/uid2/operator/utilTests/IdentityResponseTest.java @@ -24,9 +24,11 @@ public void doIdentityResponseTest() throws NoSuchAlgorithmException { Instant identityExpires = Instant.now(); Instant refreshFrom = identityExpires.plus(5, ChronoUnit.MINUTES); Instant refreshExpires = identityExpires.plus(10, ChronoUnit.MINUTES); + + + IdentityResponse response1 = new IdentityResponse("adToken", TokenVersion.V3, "refreshToken", identityExpires , refreshExpires, refreshFrom); - assertEquals(response1.getAdvertisingToken(), "adToken"); assertEquals(response1.getAdvertisingTokenVersion(), TokenVersion.V3); assertEquals(response1.getRefreshToken(), "refreshToken"); From 2edcb05239c63ca64051a03627d6ae7b868a607b Mon Sep 17 00:00:00 2001 From: Sunny Wu Date: Wed, 30 Oct 2024 18:19:01 +1100 Subject: [PATCH 108/431] Refactored unit test codes to standardise the advertisingTokenInput/refreshTokenInput verification --- .../operator/UIDOperatorVerticleTest.java | 230 ++++++++++-------- 1 file changed, 125 insertions(+), 105 deletions(-) diff --git a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java index a8bddccfc..98c64b685 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java @@ -854,9 +854,6 @@ RefreshTokenInput decodeRefreshToken(EncryptedTokenEncoder encoder, String refre assertEquals(identityType, refreshTokenInput.firstLevelHashIdentity.identityType); return refreshTokenInput; } - RefreshTokenInput decodeRefreshToken(EncryptedTokenEncoder encoder, String refreshTokenString) { - return decodeRefreshToken(encoder, refreshTokenString, IdentityType.Email); - } @ParameterizedTest @ValueSource(strings = {"v1", "v2"}) @@ -1228,21 +1225,13 @@ void tokenGenerateForEmail(String apiVersion, Vertx vertx, VertxTestContext test EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); AdvertisingTokenInput advertisingTokenInput = validateAndGetToken(encoder, body, IdentityType.Email); - assertFalse(advertisingTokenInput.privacyBits.isClientSideTokenGenerated()); - assertFalse(advertisingTokenInput.privacyBits.isClientSideTokenOptedOut()); - assertEquals(clientSiteId, advertisingTokenInput.sourcePublisher.siteId); - assertArrayEquals(getRawUidFromIdentity(IdentityType.Email, emailAddress, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenInput.rawUidIdentity.rawUid); + RefreshTokenInput refreshTokenInput = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), IdentityType.Email); - RefreshTokenInput refreshTokenInput = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token")); - assertEquals(clientSiteId, refreshTokenInput.sourcePublisher.siteId); - - verifyPrivacyBits(PrivacyBits.DEFAULT, advertisingTokenInput, refreshTokenInput); - - verifyFirstLevelHashIdentityAndEstablishedAt(TokenUtils.getFirstLevelHashFromIdentity(emailAddress, firstLevelSalt), refreshTokenInput, body, advertisingTokenInput.establishedAt); - - assertEqualsClose(now.plusMillis(identityExpiresAfter.toMillis()), Instant.ofEpochMilli(body.getLong("identity_expires")), 10); - assertEqualsClose(now.plusMillis(refreshExpiresAfter.toMillis()), Instant.ofEpochMilli(body.getLong("refresh_expires")), 10); - assertEqualsClose(now.plusMillis(refreshIdentityAfter.toMillis()), Instant.ofEpochMilli(body.getLong("refresh_from")), 10); + assertAdvertisingTokenRefreshTokenInputs(advertisingTokenInput, refreshTokenInput, clientSiteId, + getRawUidFromIdentity(IdentityType.Email, emailAddress, firstLevelSalt, rotatingSalt123.getSalt()), + PrivacyBits.DEFAULT, + body, + TokenUtils.getFirstLevelHashFromIdentity(emailAddress, firstLevelSalt)); assertStatsCollector("/" + apiVersion + "/token/generate", null, "test-contact", clientSiteId); @@ -1250,6 +1239,21 @@ void tokenGenerateForEmail(String apiVersion, Vertx vertx, VertxTestContext test }); } + public void assertAdvertisingTokenRefreshTokenInputs(AdvertisingTokenInput advertisingTokenInput, RefreshTokenInput refreshTokenInput, + int expectedClientSiteId, byte[] expectedRawUidIdentity, PrivacyBits expectedPrivacyBits, JsonObject identityResponse, byte[] firstLevelHashIdentity) { + + assertEquals(expectedClientSiteId, advertisingTokenInput.sourcePublisher.siteId); + assertEquals(expectedClientSiteId, refreshTokenInput.sourcePublisher.siteId); + assertArrayEquals(expectedRawUidIdentity, advertisingTokenInput.rawUidIdentity.rawUid); + + verifyPrivacyBits(expectedPrivacyBits, advertisingTokenInput, refreshTokenInput); + verifyFirstLevelHashIdentityAndEstablishedAt(firstLevelHashIdentity, refreshTokenInput, identityResponse, advertisingTokenInput.establishedAt); + + assertEqualsClose(now.plusMillis(identityExpiresAfter.toMillis()), Instant.ofEpochMilli(identityResponse.getLong("identity_expires")), 10); + assertEqualsClose(now.plusMillis(refreshExpiresAfter.toMillis()), Instant.ofEpochMilli(identityResponse.getLong("refresh_expires")), 10); + assertEqualsClose(now.plusMillis(refreshIdentityAfter.toMillis()), Instant.ofEpochMilli(identityResponse.getLong("refresh_from")), 10); + } + public void verifyPrivacyBits(PrivacyBits expectedValue, AdvertisingTokenInput advertisingTokenInput, RefreshTokenInput refreshTokenInput) { assertEquals(advertisingTokenInput.privacyBits, expectedValue); @@ -1278,21 +1282,13 @@ void tokenGenerateForEmailHash(String apiVersion, Vertx vertx, VertxTestContext EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); AdvertisingTokenInput advertisingTokenInput = validateAndGetToken(encoder, body, IdentityType.Email); + RefreshTokenInput refreshTokenInput = decodeRefreshToken(encoder, apiVersion.equals("v2") ? body.getString("decrypted_refresh_token") : body.getString("refresh_token"), IdentityType.Email); - assertFalse(advertisingTokenInput.privacyBits.isClientSideTokenGenerated()); - assertFalse(advertisingTokenInput.privacyBits.isClientSideTokenOptedOut()); - assertEquals(clientSiteId, advertisingTokenInput.sourcePublisher.siteId); - assertArrayEquals(getRawUidFromIdentityHash(IdentityType.Email, emailHash, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenInput.rawUidIdentity.rawUid); - - RefreshTokenInput refreshTokenInput = decodeRefreshToken(encoder, apiVersion.equals("v2") ? body.getString("decrypted_refresh_token") : body.getString("refresh_token")); - assertEquals(clientSiteId, refreshTokenInput.sourcePublisher.siteId); - - verifyPrivacyBits(PrivacyBits.DEFAULT, advertisingTokenInput, refreshTokenInput); - verifyFirstLevelHashIdentityAndEstablishedAt(TokenUtils.getFirstLevelHashFromIdentityHash(emailHash, firstLevelSalt), refreshTokenInput, body, advertisingTokenInput.establishedAt); - - assertEqualsClose(now.plusMillis(identityExpiresAfter.toMillis()), Instant.ofEpochMilli(body.getLong("identity_expires")), 10); - assertEqualsClose(now.plusMillis(refreshExpiresAfter.toMillis()), Instant.ofEpochMilli(body.getLong("refresh_expires")), 10); - assertEqualsClose(now.plusMillis(refreshIdentityAfter.toMillis()), Instant.ofEpochMilli(body.getLong("refresh_from")), 10); + assertAdvertisingTokenRefreshTokenInputs(advertisingTokenInput, refreshTokenInput, clientSiteId, + getRawUidFromIdentityHash(IdentityType.Email, emailHash, firstLevelSalt, rotatingSalt123.getSalt()), + PrivacyBits.DEFAULT, + body, + TokenUtils.getFirstLevelHashFromIdentityHash(emailHash, firstLevelSalt)); testContext.completeNow(); }); @@ -1318,12 +1314,23 @@ void tokenGenerateThenRefresh(String apiVersion, Vertx vertx, VertxTestContext t AdvertisingTokenInput firstAdvertisingTokenInput = validateAndGetToken(encoder, bodyJson, IdentityType.Email); - RefreshTokenInput firstRefreshTokenInput = decodeRefreshToken(encoder, bodyJson.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token")); + RefreshTokenInput firstRefreshTokenInput = decodeRefreshToken(encoder, bodyJson.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), IdentityType.Email); assertEquals(firstAdvertisingTokenInput.establishedAt, firstRefreshTokenInput.firstLevelHashIdentity.establishedAt); when(this.optOutStore.getLatestEntry(any())).thenReturn(null); + + byte[] expectedRawUidIdentity = getRawUidFromIdentity(IdentityType.Email, emailAddress, firstLevelSalt, rotatingSalt123.getSalt()); + byte[] expectedFirstLevelHashIdentity = TokenUtils.getFirstLevelHashFromIdentity(emailAddress, firstLevelSalt); + + assertAdvertisingTokenRefreshTokenInputs(firstAdvertisingTokenInput, firstRefreshTokenInput, clientSiteId, + expectedRawUidIdentity, + PrivacyBits.DEFAULT, + bodyJson, + expectedFirstLevelHashIdentity); + + sendTokenRefresh(apiVersion, vertx, ClientVersionHeader, iosClientVersionHeaderValue, testContext, genRefreshToken, bodyJson.getString("refresh_response_key"), 200, refreshRespJson -> { assertEquals("success", refreshRespJson.getString("status")); @@ -1332,39 +1339,35 @@ void tokenGenerateThenRefresh(String apiVersion, Vertx vertx, VertxTestContext t AdvertisingTokenInput advertisingTokenInput = validateAndGetToken(encoder, refreshBody, IdentityType.Email); - assertFalse(advertisingTokenInput.privacyBits.isClientSideTokenGenerated()); - assertFalse(advertisingTokenInput.privacyBits.isClientSideTokenOptedOut()); - assertEquals(clientSiteId, advertisingTokenInput.sourcePublisher.siteId); - - // checks that the raw Uid from raw input, from original tokgen/generate response and the subsequent - // token/refresh response are identical - assertArrayEquals(getRawUidFromIdentity(IdentityType.Email, emailAddress, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenInput.rawUidIdentity.rawUid); - assertArrayEquals(firstAdvertisingTokenInput.rawUidIdentity.rawUid, - advertisingTokenInput.rawUidIdentity.rawUid); - - String refreshTokenStringNew = refreshBody.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"); assertNotEquals(genRefreshToken, refreshTokenStringNew); - RefreshTokenInput refreshTokenInput = decodeRefreshToken(encoder, refreshTokenStringNew); - - // check that first level hash from token/generate's refresh token and token/refresh's refresh token - // are identical - assertEquals(firstRefreshTokenInput.firstLevelHashIdentity.establishedAt, - refreshTokenInput.firstLevelHashIdentity.establishedAt); - assertTrue(firstRefreshTokenInput.firstLevelHashIdentity.matches(refreshTokenInput.firstLevelHashIdentity)); - - verifyPrivacyBits(PrivacyBits.DEFAULT, firstAdvertisingTokenInput, refreshTokenInput); - verifyPrivacyBits(PrivacyBits.DEFAULT, advertisingTokenInput, refreshTokenInput); + RefreshTokenInput refreshTokenInput = decodeRefreshToken(encoder, refreshTokenStringNew, IdentityType.Email); - verifyFirstLevelHashIdentityAndEstablishedAt(TokenUtils.getFirstLevelHashFromIdentity(emailAddress, - firstLevelSalt), refreshTokenInput, refreshBody, firstAdvertisingTokenInput.establishedAt); - - assertEquals(clientSiteId, refreshTokenInput.sourcePublisher.siteId); - assertArrayEquals(TokenUtils.getFirstLevelHashFromIdentity(emailAddress, firstLevelSalt), refreshTokenInput.firstLevelHashIdentity.firstLevelHash); - - assertEqualsClose(now.plusMillis(identityExpiresAfter.toMillis()), Instant.ofEpochMilli(refreshBody.getLong("identity_expires")), 10); - assertEqualsClose(now.plusMillis(refreshExpiresAfter.toMillis()), Instant.ofEpochMilli(refreshBody.getLong("refresh_expires")), 10); - assertEqualsClose(now.plusMillis(refreshIdentityAfter.toMillis()), Instant.ofEpochMilli(refreshBody.getLong("refresh_from")), 10); + // assert if the ad/refresh tokens from original token/generate is same as the ad/refresh tokens from token/refresh + assertAdvertisingTokenRefreshTokenInputs( + advertisingTokenInput, + firstRefreshTokenInput, + clientSiteId, + expectedRawUidIdentity, + PrivacyBits.DEFAULT, + bodyJson, + expectedFirstLevelHashIdentity); + assertAdvertisingTokenRefreshTokenInputs( + firstAdvertisingTokenInput, + refreshTokenInput, + clientSiteId, + expectedRawUidIdentity, + PrivacyBits.DEFAULT, + bodyJson, + expectedFirstLevelHashIdentity); + assertAdvertisingTokenRefreshTokenInputs( + advertisingTokenInput, + refreshTokenInput, + clientSiteId, + expectedRawUidIdentity, + PrivacyBits.DEFAULT, + bodyJson, + expectedFirstLevelHashIdentity); assertTokenStatusMetrics( clientSiteId, @@ -1417,7 +1420,7 @@ void tokenGenerateThenRefreshSaltsExpired(String apiVersion, Vertx vertx, VertxT String refreshTokenStringNew = refreshBody.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"); assertNotEquals(genRefreshToken, refreshTokenStringNew); - RefreshTokenInput refreshTokenInput = decodeRefreshToken(encoder, refreshTokenStringNew); + RefreshTokenInput refreshTokenInput = decodeRefreshToken(encoder, refreshTokenStringNew, IdentityType.Email); assertEquals(clientSiteId, refreshTokenInput.sourcePublisher.siteId); assertArrayEquals(TokenUtils.getFirstLevelHashFromIdentity(emailAddress, firstLevelSalt), refreshTokenInput.firstLevelHashIdentity.firstLevelHash); @@ -1590,7 +1593,7 @@ void tokenGenerateUsingCustomSiteKey(String apiVersion, Vertx vertx, VertxTestCo assertEquals(clientSiteId, advertisingTokenInput.sourcePublisher.siteId); assertArrayEquals(getRawUidFromIdentity(IdentityType.Email, emailAddress, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenInput.rawUidIdentity.rawUid); - RefreshTokenInput refreshTokenInput = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token")); + RefreshTokenInput refreshTokenInput = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), IdentityType.Email); assertEquals(clientSiteId, refreshTokenInput.sourcePublisher.siteId); assertArrayEquals(TokenUtils.getFirstLevelHashFromIdentity(emailAddress, firstLevelSalt), refreshTokenInput.firstLevelHashIdentity.firstLevelHash); @@ -1629,7 +1632,7 @@ void tokenGenerateSaltsExpired(String apiVersion, Vertx vertx, VertxTestContext assertEquals(clientSiteId, advertisingTokenInput.sourcePublisher.siteId); assertArrayEquals(getRawUidFromIdentity(IdentityType.Email, emailAddress, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenInput.rawUidIdentity.rawUid); - RefreshTokenInput refreshTokenInput = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token")); + RefreshTokenInput refreshTokenInput = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), IdentityType.Email); assertEquals(clientSiteId, refreshTokenInput.sourcePublisher.siteId); assertArrayEquals(TokenUtils.getFirstLevelHashFromIdentity(emailAddress, firstLevelSalt), refreshTokenInput.firstLevelHashIdentity.firstLevelHash); @@ -2535,22 +2538,13 @@ void tokenGenerateForPhone(String apiVersion, Vertx vertx, VertxTestContext test EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); AdvertisingTokenInput advertisingTokenInput = validateAndGetToken(encoder, body, IdentityType.Phone); - - assertFalse(advertisingTokenInput.privacyBits.isClientSideTokenGenerated()); - assertFalse(advertisingTokenInput.privacyBits.isClientSideTokenOptedOut()); - assertEquals(clientSiteId, advertisingTokenInput.sourcePublisher.siteId); - assertArrayEquals(getRawUidFromIdentity(IdentityType.Phone, phone, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenInput.rawUidIdentity.rawUid); - RefreshTokenInput refreshTokenInput = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), IdentityType.Phone); - assertEquals(clientSiteId, refreshTokenInput.sourcePublisher.siteId); - verifyPrivacyBits(PrivacyBits.DEFAULT, advertisingTokenInput, refreshTokenInput); - verifyFirstLevelHashIdentityAndEstablishedAt(TokenUtils.getFirstLevelHashFromIdentity(phone, - firstLevelSalt), refreshTokenInput, body, advertisingTokenInput.establishedAt); - - assertEqualsClose(now.plusMillis(identityExpiresAfter.toMillis()), Instant.ofEpochMilli(body.getLong("identity_expires")), 10); - assertEqualsClose(now.plusMillis(refreshExpiresAfter.toMillis()), Instant.ofEpochMilli(body.getLong("refresh_expires")), 10); - assertEqualsClose(now.plusMillis(refreshIdentityAfter.toMillis()), Instant.ofEpochMilli(body.getLong("refresh_from")), 10); + assertAdvertisingTokenRefreshTokenInputs(advertisingTokenInput, refreshTokenInput, clientSiteId, + getRawUidFromIdentity(IdentityType.Phone, phone, firstLevelSalt, rotatingSalt123.getSalt()), + PrivacyBits.DEFAULT, + body, + TokenUtils.getFirstLevelHashFromIdentity(phone, firstLevelSalt)); testContext.completeNow(); }); @@ -2589,21 +2583,14 @@ void tokenGenerateForPhoneHash(String apiVersion, Vertx vertx, VertxTestContext EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); AdvertisingTokenInput advertisingTokenInput = validateAndGetToken(encoder, body, IdentityType.Phone); - - assertFalse(advertisingTokenInput.privacyBits.isClientSideTokenGenerated()); - assertFalse(advertisingTokenInput.privacyBits.isClientSideTokenOptedOut()); - assertEquals(clientSiteId, advertisingTokenInput.sourcePublisher.siteId); - assertArrayEquals(getRawUidFromIdentity(IdentityType.Phone, phone, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenInput.rawUidIdentity.rawUid); - RefreshTokenInput refreshTokenInput = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), IdentityType.Phone); - assertEquals(clientSiteId, refreshTokenInput.sourcePublisher.siteId); - verifyPrivacyBits(PrivacyBits.DEFAULT, advertisingTokenInput, refreshTokenInput); - verifyFirstLevelHashIdentityAndEstablishedAt(TokenUtils.getFirstLevelHashFromIdentity(phone, firstLevelSalt), refreshTokenInput, body, advertisingTokenInput.establishedAt); + assertAdvertisingTokenRefreshTokenInputs(advertisingTokenInput, refreshTokenInput, clientSiteId, + getRawUidFromIdentity(IdentityType.Phone, phone, firstLevelSalt, rotatingSalt123.getSalt()), + PrivacyBits.DEFAULT, + body, + TokenUtils.getFirstLevelHashFromIdentity(phone, firstLevelSalt)); - assertEqualsClose(now.plusMillis(identityExpiresAfter.toMillis()), Instant.ofEpochMilli(body.getLong("identity_expires")), 10); - assertEqualsClose(now.plusMillis(refreshExpiresAfter.toMillis()), Instant.ofEpochMilli(body.getLong("refresh_expires")), 10); - assertEqualsClose(now.plusMillis(refreshIdentityAfter.toMillis()), Instant.ofEpochMilli(body.getLong("refresh_from")), 10); testContext.completeNow(); }); @@ -2627,9 +2614,19 @@ void tokenGenerateThenRefreshForPhone(String apiVersion, Vertx vertx, VertxTestC AdvertisingTokenInput firstAdvertisingTokenInput = validateAndGetToken(encoder, bodyJson, IdentityType.Phone); String genRefreshToken = bodyJson.getString("refresh_token"); + RefreshTokenInput firstRefreshTokenInput = decodeRefreshToken(encoder, bodyJson.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), IdentityType.Phone); when(this.optOutStore.getLatestEntry(any())).thenReturn(null); + byte[] expectedRawUidIdentity = getRawUidFromIdentity(IdentityType.Phone, phone, firstLevelSalt, rotatingSalt123.getSalt()); + byte[] expectedFirstLevelHashIdentity = TokenUtils.getFirstLevelHashFromIdentity(phone, firstLevelSalt); + + assertAdvertisingTokenRefreshTokenInputs(firstAdvertisingTokenInput, firstRefreshTokenInput, clientSiteId, + expectedRawUidIdentity, + PrivacyBits.DEFAULT, + bodyJson, + expectedFirstLevelHashIdentity); + sendTokenRefresh(apiVersion, vertx, testContext, genRefreshToken, bodyJson.getString("refresh_response_key"), 200, refreshRespJson -> { assertEquals("success", refreshRespJson.getString("status")); @@ -2638,25 +2635,48 @@ void tokenGenerateThenRefreshForPhone(String apiVersion, Vertx vertx, VertxTestC AdvertisingTokenInput advertisingTokenInput = validateAndGetToken(encoder, refreshBody, IdentityType.Phone); - - assertFalse(advertisingTokenInput.privacyBits.isClientSideTokenGenerated()); - assertFalse(advertisingTokenInput.privacyBits.isClientSideTokenOptedOut()); - assertEquals(clientSiteId, advertisingTokenInput.sourcePublisher.siteId); - assertArrayEquals(getRawUidFromIdentity(IdentityType.Phone, phone, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenInput.rawUidIdentity.rawUid); - String refreshTokenStringNew = refreshBody.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"); assertNotEquals(genRefreshToken, refreshTokenStringNew); RefreshTokenInput refreshTokenInput = decodeRefreshToken(encoder, refreshTokenStringNew, IdentityType.Phone); - assertEquals(clientSiteId, refreshTokenInput.sourcePublisher.siteId); - verifyPrivacyBits(PrivacyBits.DEFAULT, firstAdvertisingTokenInput, refreshTokenInput); - verifyPrivacyBits(PrivacyBits.DEFAULT, advertisingTokenInput, refreshTokenInput); - verifyFirstLevelHashIdentityAndEstablishedAt(TokenUtils.getFirstLevelHashFromIdentity(phone, - firstLevelSalt), refreshTokenInput, refreshBody, advertisingTokenInput.establishedAt); + // assert if the ad/refresh tokens from original token/generate is same as the ad/refresh tokens from token/refresh + assertAdvertisingTokenRefreshTokenInputs( + advertisingTokenInput, + firstRefreshTokenInput, + clientSiteId, + expectedRawUidIdentity, + PrivacyBits.DEFAULT, + bodyJson, + expectedFirstLevelHashIdentity); + assertAdvertisingTokenRefreshTokenInputs( + firstAdvertisingTokenInput, + refreshTokenInput, + clientSiteId, + expectedRawUidIdentity, + PrivacyBits.DEFAULT, + bodyJson, + expectedFirstLevelHashIdentity); + assertAdvertisingTokenRefreshTokenInputs( + advertisingTokenInput, + refreshTokenInput, + clientSiteId, + expectedRawUidIdentity, + PrivacyBits.DEFAULT, + bodyJson, + expectedFirstLevelHashIdentity); - assertEqualsClose(now.plusMillis(identityExpiresAfter.toMillis()), Instant.ofEpochMilli(refreshBody.getLong("identity_expires")), 10); - assertEqualsClose(now.plusMillis(refreshExpiresAfter.toMillis()), Instant.ofEpochMilli(refreshBody.getLong("refresh_expires")), 10); - assertEqualsClose(now.plusMillis(refreshIdentityAfter.toMillis()), Instant.ofEpochMilli(refreshBody.getLong("refresh_from")), 10); + assertTokenStatusMetrics( + clientSiteId, + apiVersion.equals("v1") ? TokenResponseStatsCollector.Endpoint.GenerateV1 : TokenResponseStatsCollector.Endpoint.GenerateV2, + TokenResponseStatsCollector.ResponseStatus.Success, + //didn't set any specific header + TokenResponseStatsCollector.PlatformType.Other); + assertTokenStatusMetrics( + clientSiteId, + apiVersion.equals("v1") ? TokenResponseStatsCollector.Endpoint.RefreshV1 : TokenResponseStatsCollector.Endpoint.RefreshV2, + TokenResponseStatsCollector.ResponseStatus.Success, + //didn't set any specific header + TokenResponseStatsCollector.PlatformType.Other); testContext.completeNow(); }); From ca4494553e4599e884a72556288df939485aac15 Mon Sep 17 00:00:00 2001 From: Sunny Wu Date: Wed, 30 Oct 2024 18:27:16 +1100 Subject: [PATCH 109/431] Refactored unit test codes to standardise the advertisingTokenInput/refreshTokenInput verification --- .../operator/UIDOperatorVerticleTest.java | 29 +++++++++++++------ 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java index 98c64b685..452b6ecc9 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java @@ -4153,12 +4153,21 @@ void cstgSuccessForBothOptedAndNonOptedOutTest(boolean optOutExpected, String id RefreshTokenInput refreshTokenInput = decodeRefreshToken(encoder, genBody.getString("decrypted_refresh_token"), identityType); - PrivacyBits pb = new PrivacyBits(); - pb.setLegacyBit(); - pb.setClientSideTokenGenerate(); - verifyPrivacyBits(pb, advertisingTokenInput, refreshTokenInput); - verifyFirstLevelHashIdentityAndEstablishedAt(TokenUtils.getFirstLevelHashFromIdentity(id, firstLevelSalt), refreshTokenInput, genBody, advertisingTokenInput.establishedAt); + + byte[] expectedRawUidIdentity = getRawUidFromIdentity(identityType, id, firstLevelSalt, rotatingSalt123.getSalt()); + byte[] expectedFirstLevelHashIdentity = TokenUtils.getFirstLevelHashFromIdentity(id, firstLevelSalt); + + PrivacyBits expectedPrivacyBits = new PrivacyBits(); + expectedPrivacyBits.setLegacyBit(); + expectedPrivacyBits.setClientSideTokenGenerate(); + + assertAdvertisingTokenRefreshTokenInputs(advertisingTokenInput, refreshTokenInput, + clientSideTokenGenerateSiteId, + expectedRawUidIdentity, + expectedPrivacyBits, + genBody, + expectedFirstLevelHashIdentity); assertAreClientSideGeneratedTokens(advertisingTokenInput, refreshTokenInput, clientSideTokenGenerateSiteId, identityType, id); assertEqualsClose(Instant.now().plusMillis(identityExpiresAfter.toMillis()), Instant.ofEpochMilli(genBody.getLong("identity_expires")), 10); assertEqualsClose(Instant.now().plusMillis(refreshExpiresAfter.toMillis()), Instant.ofEpochMilli(genBody.getLong("refresh_expires")), 10); @@ -4186,10 +4195,12 @@ void cstgSuccessForBothOptedAndNonOptedOutTest(boolean optOutExpected, String id assertNotEquals(genRefreshToken, refreshTokenStringNew); RefreshTokenInput refreshTokenAfterRefreshSource = decodeRefreshToken(encoder, refreshTokenStringNew, identityType); - verifyPrivacyBits(pb, adTokenFromRefresh, refreshTokenInput); - verifyPrivacyBits(pb, adTokenFromRefresh, refreshTokenAfterRefreshSource); - verifyFirstLevelHashIdentityAndEstablishedAt(TokenUtils.getFirstLevelHashFromIdentity(id, firstLevelSalt), refreshTokenAfterRefreshSource, refreshBody, advertisingTokenInput.establishedAt); - + assertAdvertisingTokenRefreshTokenInputs(adTokenFromRefresh, refreshTokenAfterRefreshSource, + clientSideTokenGenerateSiteId, + expectedRawUidIdentity, + expectedPrivacyBits, + genBody, + expectedFirstLevelHashIdentity); assertAreClientSideGeneratedTokens(adTokenFromRefresh, refreshTokenAfterRefreshSource, clientSideTokenGenerateSiteId, identityType, id); assertEqualsClose(Instant.now().plusMillis(identityExpiresAfter.toMillis()), Instant.ofEpochMilli(refreshBody.getLong("identity_expires")), 10); assertEqualsClose(Instant.now().plusMillis(refreshExpiresAfter.toMillis()), Instant.ofEpochMilli(refreshBody.getLong("refresh_expires")), 10); From da549b011d1bfdfa864ea72bf5ede0f381af8813 Mon Sep 17 00:00:00 2001 From: Sunny Wu Date: Mon, 4 Nov 2024 12:42:39 +1100 Subject: [PATCH 110/431] Code review feedback --- src/main/java/com/uid2/operator/IdentityConst.java | 2 +- .../java/com/uid2/operator/model/IdentityRequest.java | 6 ++---- .../java/com/uid2/operator/model/IdentityResponse.java | 1 + src/main/java/com/uid2/operator/model/MapRequest.java | 3 +-- .../java/com/uid2/operator/model/RawUidResponse.java | 2 +- .../java/com/uid2/operator/model/SourcePublisher.java | 6 ++++++ .../uid2/operator/model/userIdentity/UserIdentity.java | 4 +--- .../uid2/operator/service/EncryptedTokenEncoder.java | 2 +- .../com/uid2/operator/vertx/UIDOperatorVerticle.java | 10 +++++----- .../com/uid2/operator/benchmark/BenchmarkCommon.java | 2 +- 10 files changed, 20 insertions(+), 18 deletions(-) diff --git a/src/main/java/com/uid2/operator/IdentityConst.java b/src/main/java/com/uid2/operator/IdentityConst.java index 9362ade6e..9dad70a0c 100644 --- a/src/main/java/com/uid2/operator/IdentityConst.java +++ b/src/main/java/com/uid2/operator/IdentityConst.java @@ -7,7 +7,7 @@ public class IdentityConst { public static final String OptOutTokenIdentityForEmail = "optout@unifiedid.com"; public static final String OptOutTokenIdentityForPhone = "+00000000001"; - // DIIs for for testing with token/validate endpoint, see https://unifiedid.com/docs/endpoints/post-token-validate + // DIIs for testing with token/validate endpoint, see https://unifiedid.com/docs/endpoints/post-token-validate public static final String ValidateIdentityForEmail = "validate@example.com"; public static final String ValidateIdentityForPhone = "+12345678901"; public static final byte[] ValidateIdentityForEmailHash = EncodingUtils.getSha256Bytes(IdentityConst.ValidateIdentityForEmail); diff --git a/src/main/java/com/uid2/operator/model/IdentityRequest.java b/src/main/java/com/uid2/operator/model/IdentityRequest.java index 84a16c698..c1cca1d75 100644 --- a/src/main/java/com/uid2/operator/model/IdentityRequest.java +++ b/src/main/java/com/uid2/operator/model/IdentityRequest.java @@ -18,8 +18,7 @@ public IdentityRequest( HashedDiiIdentity hashedDiiIdentity, OptoutCheckPolicy tokenGeneratePolicy, PrivacyBits privacyBits, - Instant establishedAt) - { + Instant establishedAt) { this.sourcePublisher = sourcePublisher; this.hashedDiiIdentity = hashedDiiIdentity; this.optoutCheckPolicy = tokenGeneratePolicy; @@ -30,8 +29,7 @@ public IdentityRequest( public IdentityRequest( SourcePublisher sourcePublisher, HashedDiiIdentity hashedDiiIdentity, - OptoutCheckPolicy tokenGeneratePolicy) - { + OptoutCheckPolicy tokenGeneratePolicy) { this(sourcePublisher, hashedDiiIdentity, tokenGeneratePolicy, PrivacyBits.DEFAULT, Instant.now()); } diff --git a/src/main/java/com/uid2/operator/model/IdentityResponse.java b/src/main/java/com/uid2/operator/model/IdentityResponse.java index 45d8b4a59..11e2f4c6e 100644 --- a/src/main/java/com/uid2/operator/model/IdentityResponse.java +++ b/src/main/java/com/uid2/operator/model/IdentityResponse.java @@ -7,6 +7,7 @@ // this defines all the fields for the response of the /token/generate and /client/generate endpoints before they are // jsonified +// todo: can be converted to record later public class IdentityResponse { public static IdentityResponse OptOutIdentityResponse = new IdentityResponse("", null, "", Instant.EPOCH, Instant.EPOCH, Instant.EPOCH); diff --git a/src/main/java/com/uid2/operator/model/MapRequest.java b/src/main/java/com/uid2/operator/model/MapRequest.java index 925296e44..660503041 100644 --- a/src/main/java/com/uid2/operator/model/MapRequest.java +++ b/src/main/java/com/uid2/operator/model/MapRequest.java @@ -12,8 +12,7 @@ public final class MapRequest { public MapRequest( HashedDiiIdentity hashedDiiIdentity, OptoutCheckPolicy optoutCheckPolicy, - Instant asOf) - { + Instant asOf) { this.hashedDiiIdentity = hashedDiiIdentity; this.optoutCheckPolicy = optoutCheckPolicy; this.asOf = asOf; diff --git a/src/main/java/com/uid2/operator/model/RawUidResponse.java b/src/main/java/com/uid2/operator/model/RawUidResponse.java index 249bef4c5..bbd98931f 100644 --- a/src/main/java/com/uid2/operator/model/RawUidResponse.java +++ b/src/main/java/com/uid2/operator/model/RawUidResponse.java @@ -2,7 +2,7 @@ // Contains the computed raw UID and its bucket ID from identity/map request public class RawUidResponse { - public static RawUidResponse OptoutIdentity = new RawUidResponse(new byte[33], ""); + public static final RawUidResponse OptoutIdentity = new RawUidResponse(new byte[33], ""); // The raw UID is also known as Advertising Id (historically) public final byte[] rawUid; public final String bucketId; diff --git a/src/main/java/com/uid2/operator/model/SourcePublisher.java b/src/main/java/com/uid2/operator/model/SourcePublisher.java index 4f13fd53e..a6b4f49cd 100644 --- a/src/main/java/com/uid2/operator/model/SourcePublisher.java +++ b/src/main/java/com/uid2/operator/model/SourcePublisher.java @@ -11,4 +11,10 @@ public SourcePublisher(int siteId, int clientKeyId, long publisherId) { this.clientKeyId = clientKeyId; this.publisherId = publisherId; } + + public SourcePublisher(int siteId) { + this.siteId = siteId; + this.clientKeyId = 0; + this.publisherId = 0; + } } diff --git a/src/main/java/com/uid2/operator/model/userIdentity/UserIdentity.java b/src/main/java/com/uid2/operator/model/userIdentity/UserIdentity.java index 8221a8c61..8b6c042c5 100644 --- a/src/main/java/com/uid2/operator/model/userIdentity/UserIdentity.java +++ b/src/main/java/com/uid2/operator/model/userIdentity/UserIdentity.java @@ -3,9 +3,7 @@ import com.uid2.operator.model.IdentityScope; import com.uid2.operator.model.IdentityType; -import java.time.Instant; - -//base class for all other HshedDii/FirstLevelHash/RawUIDIdentity class and define the basic common fields +//base class for all other HashedDii/FirstLevelHash/RawUIDIdentity class and define the basic common fields public abstract class UserIdentity { public final IdentityScope identityScope; diff --git a/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java b/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java index 85961981d..4dfb36937 100644 --- a/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java +++ b/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java @@ -128,7 +128,7 @@ private RefreshTokenInput decodeRefreshTokenV2(Buffer b) { return new RefreshTokenInput( TokenVersion.V2, createdAt, validTill, new OperatorIdentity(0, OperatorType.Service, 0, 0), - new SourcePublisher(siteId, 0, 0), + new SourcePublisher(siteId), new FirstLevelHashIdentity(IdentityScope.UID2, IdentityType.Email, identity, Instant.ofEpochMilli(establishedMillis)), privacyBits); diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index dfe2a4dd5..055659924 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -468,7 +468,7 @@ else if(emailHash != null) { try { identityResponse = this.idService.generateIdentity( new IdentityRequest( - new SourcePublisher(clientSideKeypair.getSiteId(), 0, 0), + new SourcePublisher(clientSideKeypair.getSiteId()), input.toHashedDiiIdentity(this.identityScope), OptoutCheckPolicy.RespectOptOut, privacyBits, Instant.now())); } catch (KeyManager.NoActiveKeyException e){ @@ -941,7 +941,7 @@ private void handleTokenGenerateV1(RoutingContext rc) { if (isTokenInputValid(input, rc)) { final IdentityResponse t = this.idService.generateIdentity( new IdentityRequest( - new SourcePublisher(siteId, 0, 0), + new SourcePublisher(siteId), input.toHashedDiiIdentity(this.identityScope), OptoutCheckPolicy.defaultPolicy())); ResponseUtil.Success(rc, t.toJsonV1()); @@ -993,7 +993,7 @@ private void handleTokenGenerateV2(RoutingContext rc) { final IdentityResponse t = this.idService.generateIdentity( new IdentityRequest( - new SourcePublisher(siteId, 0, 0), + new SourcePublisher(siteId), input.toHashedDiiIdentity(this.identityScope), OptoutCheckPolicy.respectOptOut())); @@ -1009,7 +1009,7 @@ private void handleTokenGenerateV2(RoutingContext rc) { final IdentityResponse optOutTokens = this.idService.generateIdentity( new IdentityRequest( - new SourcePublisher(siteId, 0, 0), + new SourcePublisher(siteId), optOutTokenInput.toHashedDiiIdentity(this.identityScope), OptoutCheckPolicy.DoNotRespect, pb, Instant.now())); @@ -1049,7 +1049,7 @@ else if (!input.isValid()) { siteId = AuthMiddleware.getAuthClient(rc).getSiteId(); final IdentityResponse t = this.idService.generateIdentity( new IdentityRequest( - new SourcePublisher(siteId, 0, 0), + new SourcePublisher(siteId), input.toHashedDiiIdentity(this.identityScope), OptoutCheckPolicy.defaultPolicy())); diff --git a/src/test/java/com/uid2/operator/benchmark/BenchmarkCommon.java b/src/test/java/com/uid2/operator/benchmark/BenchmarkCommon.java index 1c6153d22..c5b062ad8 100644 --- a/src/test/java/com/uid2/operator/benchmark/BenchmarkCommon.java +++ b/src/test/java/com/uid2/operator/benchmark/BenchmarkCommon.java @@ -168,7 +168,7 @@ static SourcePublisher createSourcePublisher() throws Exception { for (ClientKey client : clients.getAll()) { if (client.hasRole(Role.GENERATOR)) { - return new SourcePublisher(client.getSiteId(), 0, 0); + return new SourcePublisher(client.getSiteId()); } } throw new IllegalStateException("embedded resource does not include any publisher key"); From c6586a55e3f4226470a3b25269dbbe84c819374c Mon Sep 17 00:00:00 2001 From: Sunny Wu Date: Mon, 4 Nov 2024 13:13:32 +1100 Subject: [PATCH 111/431] renamed IUIDOperatorService#mapIdentity to mapHashedDiiIdentity --- .../operator/service/IUIDOperatorService.java | 2 +- .../operator/service/UIDOperatorService.java | 2 +- .../operator/vertx/UIDOperatorVerticle.java | 2 +- .../uid2/operator/UIDOperatorServiceTest.java | 24 +++++++++---------- .../benchmark/IdentityMapBenchmark.java | 2 +- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/main/java/com/uid2/operator/service/IUIDOperatorService.java b/src/main/java/com/uid2/operator/service/IUIDOperatorService.java index dc6671cef..5aef54f7a 100644 --- a/src/main/java/com/uid2/operator/service/IUIDOperatorService.java +++ b/src/main/java/com/uid2/operator/service/IUIDOperatorService.java @@ -16,7 +16,7 @@ public interface IUIDOperatorService { RefreshResponse refreshIdentity(RefreshTokenInput input); - RawUidResponse mapIdentity(MapRequest request); + RawUidResponse mapHashedDiiIdentity(MapRequest request); @Deprecated RawUidResponse map(HashedDiiIdentity hashedDiiIdentity, Instant asOf); diff --git a/src/main/java/com/uid2/operator/service/UIDOperatorService.java b/src/main/java/com/uid2/operator/service/UIDOperatorService.java index f5ba469b9..965afdb4b 100644 --- a/src/main/java/com/uid2/operator/service/UIDOperatorService.java +++ b/src/main/java/com/uid2/operator/service/UIDOperatorService.java @@ -161,7 +161,7 @@ public RefreshResponse refreshIdentity(RefreshTokenInput input) { } @Override - public RawUidResponse mapIdentity(MapRequest request) { + public RawUidResponse mapHashedDiiIdentity(MapRequest request) { final FirstLevelHashIdentity firstLevelHashIdentity = getFirstLevelHashIdentity(request.hashedDiiIdentity, request.asOf); if (request.shouldCheckOptOut() && getGlobalOptOutResult(firstLevelHashIdentity, false).isOptedOut()) { diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index 055659924..52a4d7e8d 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -1450,7 +1450,7 @@ private JsonObject handleIdentityMapCommon(RoutingContext rc, InputUtil.InputVal for (int i = 0; i < count; ++i) { final InputUtil.InputVal input = inputList[i]; if (input != null && input.isValid()) { - final RawUidResponse rawUidResponse = idService.mapIdentity( + final RawUidResponse rawUidResponse = idService.mapHashedDiiIdentity( new MapRequest( input.toHashedDiiIdentity(this.identityScope), OptoutCheckPolicy.respectOptOut(), diff --git a/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java b/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java index 794228db8..0a4b56299 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java @@ -367,15 +367,15 @@ public void testIdentityMapForOptOutUser(IdentityType type, String identity, Ide if (scope == IdentityScope.UID2) { verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); - rawUidResponse = uid2Service.mapIdentity(mapRequestForceMap); + rawUidResponse = uid2Service.mapHashedDiiIdentity(mapRequestForceMap); reset(shutdownHandler); - rawUidResponseShouldBeOptOut = uid2Service.mapIdentity(mapRequestRespectOptOut); + rawUidResponseShouldBeOptOut = uid2Service.mapHashedDiiIdentity(mapRequestRespectOptOut); } else { verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); - rawUidResponse = euidService.mapIdentity(mapRequestForceMap); + rawUidResponse = euidService.mapHashedDiiIdentity(mapRequestForceMap); reset(shutdownHandler); - rawUidResponseShouldBeOptOut = euidService.mapIdentity(mapRequestRespectOptOut); + rawUidResponseShouldBeOptOut = euidService.mapHashedDiiIdentity(mapRequestRespectOptOut); } verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); @@ -471,10 +471,10 @@ void testSpecialIdentityOptOutIdentityMap(TestIdentityInputType type, String id, final RawUidResponse rawUidResponse; if(scope == IdentityScope.EUID) { - rawUidResponse = euidService.mapIdentity(mapRequestRespectOptOut); + rawUidResponse = euidService.mapHashedDiiIdentity(mapRequestRespectOptOut); } else { - rawUidResponse = uid2Service.mapIdentity(mapRequestRespectOptOut); + rawUidResponse = uid2Service.mapHashedDiiIdentity(mapRequestRespectOptOut); } verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); @@ -585,10 +585,10 @@ void testSpecialIdentityRefreshOptOutIdentityMap(TestIdentityInputType type, Str final RawUidResponse rawUidResponse; if(scope == IdentityScope.EUID) { - rawUidResponse = euidService.mapIdentity(mapRequestRespectOptOut); + rawUidResponse = euidService.mapHashedDiiIdentity(mapRequestRespectOptOut); } else { - rawUidResponse = uid2Service.mapIdentity(mapRequestRespectOptOut); + rawUidResponse = uid2Service.mapHashedDiiIdentity(mapRequestRespectOptOut); } verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); @@ -656,10 +656,10 @@ void testSpecialIdentityValidateIdentityMap(TestIdentityInputType type, String i final RawUidResponse rawUidResponse; if(scope == IdentityScope.EUID) { - rawUidResponse = euidService.mapIdentity(mapRequestRespectOptOut); + rawUidResponse = euidService.mapHashedDiiIdentity(mapRequestRespectOptOut); } else { - rawUidResponse = uid2Service.mapIdentity(mapRequestRespectOptOut); + rawUidResponse = uid2Service.mapHashedDiiIdentity(mapRequestRespectOptOut); } verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); @@ -775,10 +775,10 @@ void testExpiredSaltsNotifiesShutdownHandler(TestIdentityInputType type, String final RawUidResponse rawUidResponse; reset(shutdownHandler); if(scope == IdentityScope.EUID) { - rawUidResponse = euidService.mapIdentity(mapRequest); + rawUidResponse = euidService.mapHashedDiiIdentity(mapRequest); } else { - rawUidResponse = uid2Service.mapIdentity(mapRequest); + rawUidResponse = uid2Service.mapHashedDiiIdentity(mapRequest); } verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(true); verify(shutdownHandler, never()).handleSaltRetrievalResponse(false); diff --git a/src/test/java/com/uid2/operator/benchmark/IdentityMapBenchmark.java b/src/test/java/com/uid2/operator/benchmark/IdentityMapBenchmark.java index 880875fc0..59b033542 100644 --- a/src/test/java/com/uid2/operator/benchmark/IdentityMapBenchmark.java +++ b/src/test/java/com/uid2/operator/benchmark/IdentityMapBenchmark.java @@ -32,6 +32,6 @@ public RawUidResponse IdentityMapRawThroughput() { @Benchmark @BenchmarkMode(Mode.Throughput) public RawUidResponse IdentityMapWithOptOutThroughput() { - return uidService.mapIdentity(new MapRequest(hashedDiiIdentities[(idx++) & 65535], OptoutCheckPolicy.RespectOptOut, Instant.now())); + return uidService.mapHashedDiiIdentity(new MapRequest(hashedDiiIdentities[(idx++) & 65535], OptoutCheckPolicy.RespectOptOut, Instant.now())); } } From d76bceb4ae71f6576038412b96631958bdfb861a Mon Sep 17 00:00:00 2001 From: Sunny Wu Date: Mon, 4 Nov 2024 13:26:40 +1100 Subject: [PATCH 112/431] renamed to --- .../operator/vertx/UIDOperatorVerticle.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index 52a4d7e8d..9cc9d2d05 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -939,13 +939,13 @@ private void handleTokenGenerateV1(RoutingContext rc) { final InputUtil.InputVal input = this.phoneSupport ? this.getTokenInputV1(rc) : this.getTokenInput(rc); platformType = getPlatformType(rc); if (isTokenInputValid(input, rc)) { - final IdentityResponse t = this.idService.generateIdentity( + final IdentityResponse response = this.idService.generateIdentity( new IdentityRequest( new SourcePublisher(siteId), input.toHashedDiiIdentity(this.identityScope), OptoutCheckPolicy.defaultPolicy())); - ResponseUtil.Success(rc, t.toJsonV1()); - recordTokenResponseStats(siteId, TokenResponseStatsCollector.Endpoint.GenerateV1, TokenResponseStatsCollector.ResponseStatus.Success, siteProvider, t.getAdvertisingTokenVersion(), platformType); + ResponseUtil.Success(rc, response.toJsonV1()); + recordTokenResponseStats(siteId, TokenResponseStatsCollector.Endpoint.GenerateV1, TokenResponseStatsCollector.ResponseStatus.Success, siteProvider, response.getAdvertisingTokenVersion(), platformType); } } catch (Exception e) { SendServerErrorResponseAndRecordStats(rc, "Unknown error while generating token v1", siteId, TokenResponseStatsCollector.Endpoint.GenerateV1, TokenResponseStatsCollector.ResponseStatus.Unknown, siteProvider, e, platformType); @@ -991,13 +991,13 @@ private void handleTokenGenerateV2(RoutingContext rc) { return; } - final IdentityResponse t = this.idService.generateIdentity( + final IdentityResponse response = this.idService.generateIdentity( new IdentityRequest( new SourcePublisher(siteId), input.toHashedDiiIdentity(this.identityScope), OptoutCheckPolicy.respectOptOut())); - if (t.isOptedOut()) { + if (response.isOptedOut()) { if (optoutCheckPolicy.getItem1() == OptoutCheckPolicy.DoNotRespect) { // only legacy can use this policy final InputUtil.InputVal optOutTokenInput = input.getIdentityType() == IdentityType.Email ? InputUtil.InputVal.validEmail(OptOutTokenIdentityForEmail, OptOutTokenIdentityForEmail) @@ -1020,8 +1020,8 @@ private void handleTokenGenerateV2(RoutingContext rc) { recordTokenResponseStats(siteId, TokenResponseStatsCollector.Endpoint.GenerateV2, TokenResponseStatsCollector.ResponseStatus.OptOut, siteProvider, null, platformType); } } else { - ResponseUtil.SuccessV2(rc, t.toJsonV1()); - recordTokenResponseStats(siteId, TokenResponseStatsCollector.Endpoint.GenerateV2, TokenResponseStatsCollector.ResponseStatus.Success, siteProvider, t.getAdvertisingTokenVersion(), platformType); + ResponseUtil.SuccessV2(rc, response.toJsonV1()); + recordTokenResponseStats(siteId, TokenResponseStatsCollector.Endpoint.GenerateV2, TokenResponseStatsCollector.ResponseStatus.Success, siteProvider, response.getAdvertisingTokenVersion(), platformType); } } } catch (KeyManager.NoActiveKeyException e) { @@ -1047,14 +1047,14 @@ else if (!input.isValid()) { try { siteId = AuthMiddleware.getAuthClient(rc).getSiteId(); - final IdentityResponse t = this.idService.generateIdentity( + final IdentityResponse response = this.idService.generateIdentity( new IdentityRequest( new SourcePublisher(siteId), input.toHashedDiiIdentity(this.identityScope), OptoutCheckPolicy.defaultPolicy())); - recordTokenResponseStats(siteId, TokenResponseStatsCollector.Endpoint.GenerateV0, TokenResponseStatsCollector.ResponseStatus.Success, siteProvider, t.getAdvertisingTokenVersion(), TokenResponseStatsCollector.PlatformType.Other); - sendJsonResponse(rc, t.toJsonV0()); + recordTokenResponseStats(siteId, TokenResponseStatsCollector.Endpoint.GenerateV0, TokenResponseStatsCollector.ResponseStatus.Success, siteProvider, response.getAdvertisingTokenVersion(), TokenResponseStatsCollector.PlatformType.Other); + sendJsonResponse(rc, response.toJsonV0()); } catch (Exception e) { SendServerErrorResponseAndRecordStats(rc, "Unknown error while generating token", siteId, TokenResponseStatsCollector.Endpoint.GenerateV0, TokenResponseStatsCollector.ResponseStatus.Unknown, siteProvider, e, TokenResponseStatsCollector.PlatformType.Other); From 23a6f50b815c623b9be395103c3af219d24e8478 Mon Sep 17 00:00:00 2001 From: Sunny Wu Date: Mon, 4 Nov 2024 15:59:14 +1100 Subject: [PATCH 113/431] Code review feedback and renamed AdvertisingTOkenInput/RefreshTokenInput to AdvertisingTokenRequest/RefreshTOkenRequest --- ...nput.java => AdvertisingTokenRequest.java} | 9 +- .../uid2/operator/model/RefreshResponse.java | 11 +- ...kenInput.java => RefreshTokenRequest.java} | 7 +- .../service/EncryptedTokenEncoder.java | 54 ++-- .../uid2/operator/service/ITokenEncoder.java | 10 +- .../operator/service/IUIDOperatorService.java | 2 +- .../operator/service/UIDOperatorService.java | 24 +- .../operator/store/CloudSyncOptOutStore.java | 4 +- .../operator/vertx/UIDOperatorVerticle.java | 10 +- .../com/uid2/operator/TokenEncodingTest.java | 38 +-- .../uid2/operator/UIDOperatorServiceTest.java | 114 ++++---- .../operator/UIDOperatorVerticleTest.java | 248 +++++++++--------- 12 files changed, 267 insertions(+), 264 deletions(-) rename src/main/java/com/uid2/operator/model/{AdvertisingTokenInput.java => AdvertisingTokenRequest.java} (61%) rename src/main/java/com/uid2/operator/model/{RefreshTokenInput.java => RefreshTokenRequest.java} (66%) diff --git a/src/main/java/com/uid2/operator/model/AdvertisingTokenInput.java b/src/main/java/com/uid2/operator/model/AdvertisingTokenRequest.java similarity index 61% rename from src/main/java/com/uid2/operator/model/AdvertisingTokenInput.java rename to src/main/java/com/uid2/operator/model/AdvertisingTokenRequest.java index 0458498e9..226b0e122 100644 --- a/src/main/java/com/uid2/operator/model/AdvertisingTokenInput.java +++ b/src/main/java/com/uid2/operator/model/AdvertisingTokenRequest.java @@ -6,16 +6,17 @@ import com.uid2.operator.util.PrivacyBits; import com.uid2.shared.model.TokenVersion; -public class AdvertisingTokenInput extends VersionedToken { +// class containing enough data to create a new uid or advertising token +public class AdvertisingTokenRequest extends VersionedToken { public final OperatorIdentity operatorIdentity; public final SourcePublisher sourcePublisher; public final RawUidIdentity rawUidIdentity; public final PrivacyBits privacyBits; public final Instant establishedAt; - public AdvertisingTokenInput(TokenVersion version, Instant createdAt, Instant expiresAt, OperatorIdentity operatorIdentity, - SourcePublisher sourcePublisher, RawUidIdentity rawUidIdentity, PrivacyBits privacyBits, - Instant establishedAt) { + public AdvertisingTokenRequest(TokenVersion version, Instant createdAt, Instant expiresAt, OperatorIdentity operatorIdentity, + SourcePublisher sourcePublisher, RawUidIdentity rawUidIdentity, PrivacyBits privacyBits, + Instant establishedAt) { super(version, createdAt, expiresAt); this.operatorIdentity = operatorIdentity; this.sourcePublisher = sourcePublisher; diff --git a/src/main/java/com/uid2/operator/model/RefreshResponse.java b/src/main/java/com/uid2/operator/model/RefreshResponse.java index 2a520fcc4..7fc556b1f 100644 --- a/src/main/java/com/uid2/operator/model/RefreshResponse.java +++ b/src/main/java/com/uid2/operator/model/RefreshResponse.java @@ -4,11 +4,12 @@ public class RefreshResponse { - public static RefreshResponse Invalid = new RefreshResponse(Status.Invalid, IdentityResponse.OptOutIdentityResponse); - public static RefreshResponse Optout = new RefreshResponse(Status.Optout, IdentityResponse.OptOutIdentityResponse); - public static RefreshResponse Expired = new RefreshResponse(Status.Expired, IdentityResponse.OptOutIdentityResponse); - public static RefreshResponse Deprecated = new RefreshResponse(Status.Deprecated, IdentityResponse.OptOutIdentityResponse); - public static RefreshResponse NoActiveKey = new RefreshResponse(Status.NoActiveKey, IdentityResponse.OptOutIdentityResponse); + public static final RefreshResponse Invalid = new RefreshResponse(Status.Invalid, + IdentityResponse.OptOutIdentityResponse); + public static final RefreshResponse Optout = new RefreshResponse(Status.Optout, IdentityResponse.OptOutIdentityResponse); + public static final RefreshResponse Expired = new RefreshResponse(Status.Expired, IdentityResponse.OptOutIdentityResponse); + public static final RefreshResponse Deprecated = new RefreshResponse(Status.Deprecated, IdentityResponse.OptOutIdentityResponse); + public static final RefreshResponse NoActiveKey = new RefreshResponse(Status.NoActiveKey, IdentityResponse.OptOutIdentityResponse); private final Status status; private final IdentityResponse identityResponse; private final Duration durationSinceLastRefresh; diff --git a/src/main/java/com/uid2/operator/model/RefreshTokenInput.java b/src/main/java/com/uid2/operator/model/RefreshTokenRequest.java similarity index 66% rename from src/main/java/com/uid2/operator/model/RefreshTokenInput.java rename to src/main/java/com/uid2/operator/model/RefreshTokenRequest.java index 8abcf98bc..770b6dff5 100644 --- a/src/main/java/com/uid2/operator/model/RefreshTokenInput.java +++ b/src/main/java/com/uid2/operator/model/RefreshTokenRequest.java @@ -6,7 +6,8 @@ import com.uid2.operator.util.PrivacyBits; import com.uid2.shared.model.TokenVersion; -public class RefreshTokenInput extends VersionedToken { +// class containing enough data to create a new refresh token +public class RefreshTokenRequest extends VersionedToken { public final OperatorIdentity operatorIdentity; public final SourcePublisher sourcePublisher; public final FirstLevelHashIdentity firstLevelHashIdentity; @@ -14,8 +15,8 @@ public class RefreshTokenInput extends VersionedToken { public final PrivacyBits privacyBits; - public RefreshTokenInput(TokenVersion version, Instant createdAt, Instant expiresAt, OperatorIdentity operatorIdentity, - SourcePublisher sourcePublisher, FirstLevelHashIdentity firstLevelHashIdentity, PrivacyBits privacyBits) { + public RefreshTokenRequest(TokenVersion version, Instant createdAt, Instant expiresAt, OperatorIdentity operatorIdentity, + SourcePublisher sourcePublisher, FirstLevelHashIdentity firstLevelHashIdentity, PrivacyBits privacyBits) { super(version, createdAt, expiresAt); this.operatorIdentity = operatorIdentity; this.sourcePublisher = sourcePublisher; diff --git a/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java b/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java index 4dfb36937..0ca19d803 100644 --- a/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java +++ b/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java @@ -25,7 +25,7 @@ public EncryptedTokenEncoder(KeyManager keyManager) { this.keyManager = keyManager; } - public byte[] encodeIntoAdvertisingToken(AdvertisingTokenInput t, Instant asOf) { + public byte[] encodeIntoAdvertisingToken(AdvertisingTokenRequest t, Instant asOf) { final KeysetKey masterKey = this.keyManager.getMasterKey(asOf); final KeysetKey siteEncryptionKey = this.keyManager.getActiveKeyBySiteIdWithFallback(t.sourcePublisher.siteId, Data.AdvertisingTokenSiteId, asOf); @@ -34,7 +34,7 @@ public byte[] encodeIntoAdvertisingToken(AdvertisingTokenInput t, Instant asOf) : encodeIntoAdvertisingTokenV3(t, masterKey, siteEncryptionKey); //TokenVersion.V4 also calls encodeV3() since the byte array is identical between V3 and V4 } - private byte[] encodeIntoAdvertisingTokenV2(AdvertisingTokenInput t, KeysetKey masterKey, KeysetKey siteKey) { + private byte[] encodeIntoAdvertisingTokenV2(AdvertisingTokenRequest t, KeysetKey masterKey, KeysetKey siteKey) { final Buffer b = Buffer.buffer(); b.appendByte((byte) t.version.rawVersion); @@ -51,7 +51,7 @@ private byte[] encodeIntoAdvertisingTokenV2(AdvertisingTokenInput t, KeysetKey m return b.getBytes(); } - private byte[] encodeIntoAdvertisingTokenV3(AdvertisingTokenInput t, KeysetKey masterKey, KeysetKey siteKey) { + private byte[] encodeIntoAdvertisingTokenV3(AdvertisingTokenRequest t, KeysetKey masterKey, KeysetKey siteKey) { final Buffer sitePayload = Buffer.buffer(69); encodePublisherRequesterV3(sitePayload, t.sourcePublisher); sitePayload.appendInt(t.privacyBits.getAsInt()); @@ -78,7 +78,7 @@ private byte[] encodeIntoAdvertisingTokenV3(AdvertisingTokenInput t, KeysetKey m } @Override - public RefreshTokenInput decodeRefreshToken(String s) { + public RefreshTokenRequest decodeRefreshToken(String s) { if (s != null && !s.isEmpty()) { final byte[] bytes; try { @@ -97,7 +97,7 @@ public RefreshTokenInput decodeRefreshToken(String s) { throw new ClientInputValidationException("Invalid refresh token version"); } - private RefreshTokenInput decodeRefreshTokenV2(Buffer b) { + private RefreshTokenRequest decodeRefreshTokenV2(Buffer b) { final Instant createdAt = Instant.ofEpochMilli(b.getLong(1)); //final Instant expiresAt = Instant.ofEpochMilli(b.getLong(9)); final Instant validTill = Instant.ofEpochMilli(b.getLong(17)); @@ -125,7 +125,7 @@ private RefreshTokenInput decodeRefreshTokenV2(Buffer b) { final PrivacyBits privacyBits = PrivacyBits.fromInt(b2.getInt(8 + length)); final long establishedMillis = b2.getLong(8 + length + 4); - return new RefreshTokenInput( + return new RefreshTokenRequest( TokenVersion.V2, createdAt, validTill, new OperatorIdentity(0, OperatorType.Service, 0, 0), new SourcePublisher(siteId), @@ -134,7 +134,7 @@ private RefreshTokenInput decodeRefreshTokenV2(Buffer b) { privacyBits); } - private RefreshTokenInput decodeRefreshTokenV3(Buffer b, byte[] bytes) { + private RefreshTokenRequest decodeRefreshTokenV3(Buffer b, byte[] bytes) { final int keyId = b.getInt(2); final KeysetKey key = this.keyManager.getKey(keyId); @@ -162,14 +162,14 @@ private RefreshTokenInput decodeRefreshTokenV3(Buffer b, byte[] bytes) { throw new ClientInputValidationException("Failed to decode refreshTokenV3: Identity type mismatch"); } - return new RefreshTokenInput( + return new RefreshTokenRequest( TokenVersion.V3, createdAt, expiresAt, operatorIdentity, sourcePublisher, new FirstLevelHashIdentity(identityScope, identityType, firstLevelHash, establishedAt), privacyBits); } @Override - public AdvertisingTokenInput decodeAdvertisingToken(String base64AdvertisingToken) { + public AdvertisingTokenRequest decodeAdvertisingToken(String base64AdvertisingToken) { //Logic and code copied from: https://github.com/IABTechLab/uid2-client-java/blob/0220ef43c1661ecf3b8f4ed2db524e2db31c06b5/src/main/java/com/uid2/client/Uid2Encryption.java#L37 if (base64AdvertisingToken.length() < 4) { throw new ClientInputValidationException("Advertising token is too short"); @@ -204,7 +204,7 @@ public AdvertisingTokenInput decodeAdvertisingToken(String base64AdvertisingToke return decodeAdvertisingTokenV3orV4(b, bytes, tokenVersion); } - public AdvertisingTokenInput decodeAdvertisingTokenV2(Buffer b) { + public AdvertisingTokenRequest decodeAdvertisingTokenV2(Buffer b) { try { final int masterKeyId = b.getInt(1); @@ -227,7 +227,7 @@ public AdvertisingTokenInput decodeAdvertisingTokenV2(Buffer b) { final PrivacyBits privacyBits = PrivacyBits.fromInt(b3.getInt(8 + length)); final long establishedMillis = b3.getLong(8 + length + 4); - return new AdvertisingTokenInput( + return new AdvertisingTokenRequest( TokenVersion.V2, Instant.ofEpochMilli(establishedMillis), Instant.ofEpochMilli(expiresMillis), @@ -244,7 +244,7 @@ public AdvertisingTokenInput decodeAdvertisingTokenV2(Buffer b) { } - public AdvertisingTokenInput decodeAdvertisingTokenV3orV4(Buffer b, byte[] bytes, TokenVersion tokenVersion) { + public AdvertisingTokenRequest decodeAdvertisingTokenV3orV4(Buffer b, byte[] bytes, TokenVersion tokenVersion) { final int masterKeyId = b.getInt(2); final byte[] masterPayloadBytes = AesGcm.decrypt(bytes, 6, this.keyManager.getKey(masterKeyId)); @@ -274,7 +274,7 @@ public AdvertisingTokenInput decodeAdvertisingTokenV3orV4(Buffer b, byte[] bytes } } - return new AdvertisingTokenInput( + return new AdvertisingTokenRequest( tokenVersion, createdAt, expiresAt, operatorIdentity, sourcePublisher, new RawUidIdentity(identityScope, identityType, rawUid), privacyBits, establishedAt @@ -289,7 +289,7 @@ private void recordRefreshTokenVersionCount(String siteId, TokenVersion tokenVer .register(Metrics.globalRegistry).increment(); } - public byte[] encodeIntoRefreshToken(RefreshTokenInput t, Instant asOf) { + public byte[] encodeIntoRefreshToken(RefreshTokenRequest t, Instant asOf) { final KeysetKey serviceKey = this.keyManager.getRefreshKey(asOf); switch (t.version) { @@ -304,7 +304,7 @@ public byte[] encodeIntoRefreshToken(RefreshTokenInput t, Instant asOf) { } } - public byte[] encodeIntoRefreshTokenV2(RefreshTokenInput t, KeysetKey serviceKey) { + public byte[] encodeIntoRefreshTokenV2(RefreshTokenRequest t, KeysetKey serviceKey) { final Buffer b = Buffer.buffer(); b.appendByte((byte) t.version.rawVersion); b.appendLong(t.createdAt.toEpochMilli()); @@ -318,7 +318,7 @@ public byte[] encodeIntoRefreshTokenV2(RefreshTokenInput t, KeysetKey serviceKey return b.getBytes(); } - public byte[] encodeIntoRefreshTokenV3(RefreshTokenInput t, KeysetKey serviceKey) { + public byte[] encodeIntoRefreshTokenV3(RefreshTokenRequest t, KeysetKey serviceKey) { final Buffer refreshPayload = Buffer.buffer(90); refreshPayload.appendLong(t.expiresAt.toEpochMilli()); refreshPayload.appendLong(t.createdAt.toEpochMilli()); @@ -351,26 +351,26 @@ public static String bytesToBase64Token(byte[] advertisingTokenBytes, TokenVersi } @Override - public IdentityResponse encodeIntoIdentityResponse(AdvertisingTokenInput advertisingTokenInput, RefreshTokenInput refreshTokenInput, Instant refreshFrom, Instant asOf) { - final String advertisingToken = generateAdvertisingTokenString(advertisingTokenInput, asOf); - final String refreshToken = generateRefreshTokenString(refreshTokenInput, asOf); + public IdentityResponse encodeIntoIdentityResponse(AdvertisingTokenRequest advertisingTokenRequest, RefreshTokenRequest refreshTokenRequest, Instant refreshFrom, Instant asOf) { + final String advertisingToken = generateAdvertisingTokenString(advertisingTokenRequest, asOf); + final String refreshToken = generateRefreshTokenString(refreshTokenRequest, asOf); return new IdentityResponse( advertisingToken, - advertisingTokenInput.version, + advertisingTokenRequest.version, refreshToken, - advertisingTokenInput.expiresAt, - refreshTokenInput.expiresAt, + advertisingTokenRequest.expiresAt, + refreshTokenRequest.expiresAt, refreshFrom ); } - private String generateRefreshTokenString(RefreshTokenInput refreshTokenInput, Instant asOf) { - return EncodingUtils.toBase64String(encodeIntoRefreshToken(refreshTokenInput, asOf)); + private String generateRefreshTokenString(RefreshTokenRequest refreshTokenRequest, Instant asOf) { + return EncodingUtils.toBase64String(encodeIntoRefreshToken(refreshTokenRequest, asOf)); } - private String generateAdvertisingTokenString(AdvertisingTokenInput advertisingTokenInput, Instant asOf) { - final byte[] advertisingTokenBytes = encodeIntoAdvertisingToken(advertisingTokenInput, asOf); - return bytesToBase64Token(advertisingTokenBytes, advertisingTokenInput.version); + private String generateAdvertisingTokenString(AdvertisingTokenRequest advertisingTokenRequest, Instant asOf) { + final byte[] advertisingTokenBytes = encodeIntoAdvertisingToken(advertisingTokenRequest, asOf); + return bytesToBase64Token(advertisingTokenBytes, advertisingTokenRequest.version); } private byte[] encryptIdentityV2(SourcePublisher sourcePublisher, FirstLevelHashIdentity firstLevelHashIdentity, diff --git a/src/main/java/com/uid2/operator/service/ITokenEncoder.java b/src/main/java/com/uid2/operator/service/ITokenEncoder.java index 9380dc8c2..cac7994ae 100644 --- a/src/main/java/com/uid2/operator/service/ITokenEncoder.java +++ b/src/main/java/com/uid2/operator/service/ITokenEncoder.java @@ -1,15 +1,15 @@ package com.uid2.operator.service; -import com.uid2.operator.model.AdvertisingTokenInput; +import com.uid2.operator.model.AdvertisingTokenRequest; import com.uid2.operator.model.IdentityResponse; -import com.uid2.operator.model.RefreshTokenInput; +import com.uid2.operator.model.RefreshTokenRequest; import java.time.Instant; public interface ITokenEncoder { - IdentityResponse encodeIntoIdentityResponse(AdvertisingTokenInput advertisingTokenInput, RefreshTokenInput refreshTokenInput, Instant refreshFrom, Instant asOf); + IdentityResponse encodeIntoIdentityResponse(AdvertisingTokenRequest advertisingTokenRequest, RefreshTokenRequest refreshTokenRequest, Instant refreshFrom, Instant asOf); - AdvertisingTokenInput decodeAdvertisingToken(String base64String); + AdvertisingTokenRequest decodeAdvertisingToken(String base64String); - RefreshTokenInput decodeRefreshToken(String base64String); + RefreshTokenRequest decodeRefreshToken(String base64String); } diff --git a/src/main/java/com/uid2/operator/service/IUIDOperatorService.java b/src/main/java/com/uid2/operator/service/IUIDOperatorService.java index 5aef54f7a..eacb244a2 100644 --- a/src/main/java/com/uid2/operator/service/IUIDOperatorService.java +++ b/src/main/java/com/uid2/operator/service/IUIDOperatorService.java @@ -14,7 +14,7 @@ public interface IUIDOperatorService { IdentityResponse generateIdentity(IdentityRequest request); - RefreshResponse refreshIdentity(RefreshTokenInput input); + RefreshResponse refreshIdentity(RefreshTokenRequest input); RawUidResponse mapHashedDiiIdentity(MapRequest request); diff --git a/src/main/java/com/uid2/operator/service/UIDOperatorService.java b/src/main/java/com/uid2/operator/service/UIDOperatorService.java index 965afdb4b..9f0e36975 100644 --- a/src/main/java/com/uid2/operator/service/UIDOperatorService.java +++ b/src/main/java/com/uid2/operator/service/UIDOperatorService.java @@ -120,7 +120,7 @@ public IdentityResponse generateIdentity(IdentityRequest request) { } @Override - public RefreshResponse refreshIdentity(RefreshTokenInput input) { + public RefreshResponse refreshIdentity(RefreshTokenRequest input) { // should not be possible as different scopes should be using different keys, but just in case if (input.firstLevelHashIdentity.identityScope != this.identityScope) { return RefreshResponse.Invalid; @@ -211,7 +211,7 @@ public boolean advertisingTokenMatches(String advertisingToken, HashedDiiIdentit final FirstLevelHashIdentity firstLevelHashIdentity = getFirstLevelHashIdentity(diiIdentity, asOf); final RawUidResponse rawUidResponse = generateRawUid(firstLevelHashIdentity, asOf); - final AdvertisingTokenInput token = this.encoder.decodeAdvertisingToken(advertisingToken); + final AdvertisingTokenRequest token = this.encoder.decodeAdvertisingToken(advertisingToken); return Arrays.equals(rawUidResponse.rawUid, token.rawUidIdentity.rawUid); } @@ -260,19 +260,19 @@ private IdentityResponse generateIdentity(SourcePublisher sourcePublisher, rawUidResponse.rawUid); return this.encoder.encodeIntoIdentityResponse( - this.createAdvertisingTokenInput(sourcePublisher, rawUidIdentity, nowUtc, privacyBits, + this.createAdvertisingTokenRequest(sourcePublisher, rawUidIdentity, nowUtc, privacyBits, firstLevelHashIdentity.establishedAt), - this.createRefreshTokenInput(sourcePublisher, firstLevelHashIdentity, nowUtc, privacyBits), + this.createRefreshTokenRequest(sourcePublisher, firstLevelHashIdentity, nowUtc, privacyBits), nowUtc.plusMillis(refreshIdentityAfter.toMillis()), nowUtc ); } - private RefreshTokenInput createRefreshTokenInput(SourcePublisher sourcePublisher, - FirstLevelHashIdentity firstLevelHashIdentity, - Instant now, - PrivacyBits privacyBits) { - return new RefreshTokenInput( + private RefreshTokenRequest createRefreshTokenRequest(SourcePublisher sourcePublisher, + FirstLevelHashIdentity firstLevelHashIdentity, + Instant now, + PrivacyBits privacyBits) { + return new RefreshTokenRequest( this.refreshTokenVersion, now, now.plusMillis(refreshExpiresAfter.toMillis()), @@ -282,8 +282,8 @@ private RefreshTokenInput createRefreshTokenInput(SourcePublisher sourcePublishe privacyBits); } - private AdvertisingTokenInput createAdvertisingTokenInput(SourcePublisher sourcePublisher, RawUidIdentity rawUidIdentity, - Instant now, PrivacyBits privacyBits, Instant establishedAt) { + private AdvertisingTokenRequest createAdvertisingTokenRequest(SourcePublisher sourcePublisher, RawUidIdentity rawUidIdentity, + Instant now, PrivacyBits privacyBits, Instant establishedAt) { TokenVersion tokenVersion; if (siteIdsUsingV4Tokens.contains(sourcePublisher.siteId)) { tokenVersion = TokenVersion.V4; @@ -297,7 +297,7 @@ private AdvertisingTokenInput createAdvertisingTokenInput(SourcePublisher source } tokenVersion = (pseudoRandomNumber <= this.advertisingTokenV4Percentage) ? TokenVersion.V4 : this.tokenVersionToUseIfNotV4; } - return new AdvertisingTokenInput(tokenVersion, now, now.plusMillis(identityExpiresAfter.toMillis()), this.operatorIdentity, sourcePublisher, rawUidIdentity, + return new AdvertisingTokenRequest(tokenVersion, now, now.plusMillis(identityExpiresAfter.toMillis()), this.operatorIdentity, sourcePublisher, rawUidIdentity, privacyBits, establishedAt); } diff --git a/src/main/java/com/uid2/operator/store/CloudSyncOptOutStore.java b/src/main/java/com/uid2/operator/store/CloudSyncOptOutStore.java index bff6f4f9b..de52a6c75 100644 --- a/src/main/java/com/uid2/operator/store/CloudSyncOptOutStore.java +++ b/src/main/java/com/uid2/operator/store/CloudSyncOptOutStore.java @@ -92,8 +92,8 @@ public void addEntry(FirstLevelHashIdentity firstLevelHashIdentity, byte[] adver return; } - this.webClient.get(remoteApiPort, remoteApiHost, remoteApiPath). - addQueryParam("identity_hash", EncodingUtils.toBase64String(firstLevelHashIdentity.firstLevelHash)) + this.webClient.get(remoteApiPort, remoteApiHost, remoteApiPath) + .addQueryParam("identity_hash", EncodingUtils.toBase64String(firstLevelHashIdentity.firstLevelHash)) .addQueryParam("advertising_id", EncodingUtils.toBase64String(advertisingId)) // advertising id aka raw UID .putHeader("Authorization", remoteApiBearerToken) .as(BodyCodec.string()) diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index 9cc9d2d05..83c6e5ce8 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -1765,25 +1765,25 @@ private void recordRefreshTokenVersionCount(String siteId, TokenVersion tokenVer } private RefreshResponse refreshIdentity(RoutingContext rc, String tokenStr) { - final RefreshTokenInput refreshTokenInput; + final RefreshTokenRequest refreshTokenRequest; try { if (AuthMiddleware.isAuthenticated(rc)) { rc.put(Const.RoutingContextData.SiteId, AuthMiddleware.getAuthClient(ClientKey.class, rc).getSiteId()); } - refreshTokenInput = this.encoder.decodeRefreshToken(tokenStr); + refreshTokenRequest = this.encoder.decodeRefreshToken(tokenStr); } catch (ClientInputValidationException cie) { LOGGER.warn("Failed to decode refresh token for site ID: " + rc.data().get(Const.RoutingContextData.SiteId), cie); return RefreshResponse.Invalid; } - if (refreshTokenInput == null) { + if (refreshTokenRequest == null) { return RefreshResponse.Invalid; } if (!AuthMiddleware.isAuthenticated(rc)) { - rc.put(Const.RoutingContextData.SiteId, refreshTokenInput.sourcePublisher.siteId); + rc.put(Const.RoutingContextData.SiteId, refreshTokenRequest.sourcePublisher.siteId); } recordRefreshTokenVersionCount(String.valueOf(rc.data().get(Const.RoutingContextData.SiteId)), this.getRefreshTokenVersion(tokenStr)); - return this.idService.refreshIdentity(refreshTokenInput); + return this.idService.refreshIdentity(refreshTokenRequest); } public static String getSiteName(ISiteStore siteStore, Integer siteId) { diff --git a/src/test/java/com/uid2/operator/TokenEncodingTest.java b/src/test/java/com/uid2/operator/TokenEncodingTest.java index 484caae54..4affea1f0 100644 --- a/src/test/java/com/uid2/operator/TokenEncodingTest.java +++ b/src/test/java/com/uid2/operator/TokenEncodingTest.java @@ -55,7 +55,7 @@ public void testRefreshTokenEncoding(TokenVersion tokenVersion) { final byte[] firstLevelHash = TokenUtils.getFirstLevelHashFromIdentity("test@example.com", "some-salt"); - final RefreshTokenInput refreshTokenInput = new RefreshTokenInput(tokenVersion, + final RefreshTokenRequest refreshTokenRequest = new RefreshTokenRequest(tokenVersion, now, now.plusSeconds(360), new OperatorIdentity(101, OperatorType.Service, 102, 103), @@ -65,20 +65,20 @@ public void testRefreshTokenEncoding(TokenVersion tokenVersion) { ); if (tokenVersion == TokenVersion.V4) { - Assert.assertThrows(Exception.class, () -> encoder.encodeIntoRefreshToken(refreshTokenInput, now)); + Assert.assertThrows(Exception.class, () -> encoder.encodeIntoRefreshToken(refreshTokenRequest, now)); return; //V4 not supported for RefreshTokens } - final byte[] encodedBytes = encoder.encodeIntoRefreshToken(refreshTokenInput, now); - final RefreshTokenInput decoded = encoder.decodeRefreshToken(EncodingUtils.toBase64String(encodedBytes)); + final byte[] encodedBytes = encoder.encodeIntoRefreshToken(refreshTokenRequest, now); + final RefreshTokenRequest decoded = encoder.decodeRefreshToken(EncodingUtils.toBase64String(encodedBytes)); assertEquals(tokenVersion, decoded.version); - assertEquals(refreshTokenInput.createdAt, decoded.createdAt); + assertEquals(refreshTokenRequest.createdAt, decoded.createdAt); int addSeconds = (tokenVersion == TokenVersion.V2) ? 60 : 0; //todo: why is there a 60 second buffer in encodeV2() but not in encodeV3()? - assertEquals(refreshTokenInput.expiresAt.plusSeconds(addSeconds), decoded.expiresAt); - assertTrue(refreshTokenInput.firstLevelHashIdentity.matches(decoded.firstLevelHashIdentity)); - assertEquals(refreshTokenInput.privacyBits, decoded.privacyBits); - assertEquals(refreshTokenInput.firstLevelHashIdentity.establishedAt, decoded.firstLevelHashIdentity.establishedAt); - assertEquals(refreshTokenInput.sourcePublisher.siteId, decoded.sourcePublisher.siteId); + assertEquals(refreshTokenRequest.expiresAt.plusSeconds(addSeconds), decoded.expiresAt); + assertTrue(refreshTokenRequest.firstLevelHashIdentity.matches(decoded.firstLevelHashIdentity)); + assertEquals(refreshTokenRequest.privacyBits, decoded.privacyBits); + assertEquals(refreshTokenRequest.firstLevelHashIdentity.establishedAt, decoded.firstLevelHashIdentity.establishedAt); + assertEquals(refreshTokenRequest.sourcePublisher.siteId, decoded.sourcePublisher.siteId); Buffer b = Buffer.buffer(encodedBytes); int keyId = b.getInt(tokenVersion == TokenVersion.V2 ? 25 : 2); @@ -97,7 +97,7 @@ public void testAdvertisingTokenEncodings(TokenVersion tokenVersion) { final byte[] rawUid = UIDOperatorVerticleTest.getRawUid(IdentityType.Email, "test@example.com", IdentityScope.UID2, tokenVersion != TokenVersion.V2); - final AdvertisingTokenInput adTokenInput = new AdvertisingTokenInput( + final AdvertisingTokenRequest adTokenRequest = new AdvertisingTokenRequest( tokenVersion, now, now.plusSeconds(60), @@ -108,16 +108,16 @@ public void testAdvertisingTokenEncodings(TokenVersion tokenVersion) { now ); - final byte[] encodedBytes = encoder.encodeIntoAdvertisingToken(adTokenInput, now); - final AdvertisingTokenInput decoded = encoder.decodeAdvertisingToken(EncryptedTokenEncoder.bytesToBase64Token(encodedBytes, tokenVersion)); + final byte[] encodedBytes = encoder.encodeIntoAdvertisingToken(adTokenRequest, now); + final AdvertisingTokenRequest decoded = encoder.decodeAdvertisingToken(EncryptedTokenEncoder.bytesToBase64Token(encodedBytes, tokenVersion)); assertEquals(tokenVersion, decoded.version); - assertEquals(adTokenInput.createdAt, decoded.createdAt); - assertEquals(adTokenInput.expiresAt, decoded.expiresAt); - assertTrue(adTokenInput.rawUidIdentity.matches(decoded.rawUidIdentity)); - assertEquals(adTokenInput.privacyBits, decoded.privacyBits); - assertEquals(adTokenInput.establishedAt, decoded.establishedAt); - assertEquals(adTokenInput.sourcePublisher.siteId, decoded.sourcePublisher.siteId); + assertEquals(adTokenRequest.createdAt, decoded.createdAt); + assertEquals(adTokenRequest.expiresAt, decoded.expiresAt); + assertTrue(adTokenRequest.rawUidIdentity.matches(decoded.rawUidIdentity)); + assertEquals(adTokenRequest.privacyBits, decoded.privacyBits); + assertEquals(adTokenRequest.establishedAt, decoded.establishedAt); + assertEquals(adTokenRequest.sourcePublisher.siteId, decoded.sourcePublisher.siteId); Buffer b = Buffer.buffer(encodedBytes); int keyId = b.getInt(tokenVersion == TokenVersion.V2 ? 1 : 2); //TODO - extract master key from token should be a helper function diff --git a/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java b/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java index 0a4b56299..741050dcd 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java @@ -154,7 +154,7 @@ private HashedDiiIdentity createHashedDiiIdentity(String rawIdentityHash, Identi ); } - private AdvertisingTokenInput validateAndGetToken(EncryptedTokenEncoder tokenEncoder, String advertisingTokenString, IdentityScope scope, IdentityType type, int siteId) { + private AdvertisingTokenRequest validateAndGetToken(EncryptedTokenEncoder tokenEncoder, String advertisingTokenString, IdentityScope scope, IdentityType type, int siteId) { TokenVersion tokenVersion = (scope == IdentityScope.UID2) ? uid2Service.getAdvertisingTokenVersionForTests(siteId) : euidService.getAdvertisingTokenVersionForTests(siteId); UIDOperatorVerticleTest.validateAdvertisingToken(advertisingTokenString, tokenVersion, scope, type); return tokenEncoder.decodeAdvertisingToken(advertisingTokenString); @@ -181,29 +181,29 @@ public void testGenerateAndRefresh(int siteId, TokenVersion tokenVersion) { assertNotNull(identityResponse); UIDOperatorVerticleTest.validateAdvertisingToken(identityResponse.getAdvertisingToken(), tokenVersion, IdentityScope.UID2, IdentityType.Email); - AdvertisingTokenInput advertisingTokenInput = tokenEncoder.decodeAdvertisingToken(identityResponse.getAdvertisingToken()); - assertEquals(this.now.plusSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS), advertisingTokenInput.expiresAt); - assertEquals(identityRequest.sourcePublisher.siteId, advertisingTokenInput.sourcePublisher.siteId); - assertIdentityScopeIdentityType(identityRequest.hashedDiiIdentity, advertisingTokenInput.rawUidIdentity); - assertEquals(identityRequest.establishedAt, advertisingTokenInput.establishedAt); - assertEquals(identityRequest.privacyBits, advertisingTokenInput.privacyBits); - - RefreshTokenInput refreshTokenInput = tokenEncoder.decodeRefreshToken(identityResponse.getRefreshToken()); - assertEquals(this.now, refreshTokenInput.createdAt); - assertEquals(this.now.plusSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), refreshTokenInput.expiresAt); - assertEquals(identityRequest.sourcePublisher.siteId, refreshTokenInput.sourcePublisher.siteId); - assertIdentityScopeIdentityType(identityRequest.hashedDiiIdentity, refreshTokenInput.firstLevelHashIdentity); - assertEquals(identityRequest.establishedAt, refreshTokenInput.firstLevelHashIdentity.establishedAt); + AdvertisingTokenRequest advertisingTokenRequest = tokenEncoder.decodeAdvertisingToken(identityResponse.getAdvertisingToken()); + assertEquals(this.now.plusSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS), advertisingTokenRequest.expiresAt); + assertEquals(identityRequest.sourcePublisher.siteId, advertisingTokenRequest.sourcePublisher.siteId); + assertIdentityScopeIdentityType(identityRequest.hashedDiiIdentity, advertisingTokenRequest.rawUidIdentity); + assertEquals(identityRequest.establishedAt, advertisingTokenRequest.establishedAt); + assertEquals(identityRequest.privacyBits, advertisingTokenRequest.privacyBits); + + RefreshTokenRequest refreshTokenRequest = tokenEncoder.decodeRefreshToken(identityResponse.getRefreshToken()); + assertEquals(this.now, refreshTokenRequest.createdAt); + assertEquals(this.now.plusSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), refreshTokenRequest.expiresAt); + assertEquals(identityRequest.sourcePublisher.siteId, refreshTokenRequest.sourcePublisher.siteId); + assertIdentityScopeIdentityType(identityRequest.hashedDiiIdentity, refreshTokenRequest.firstLevelHashIdentity); + assertEquals(identityRequest.establishedAt, refreshTokenRequest.firstLevelHashIdentity.establishedAt); final byte[] firstLevelHash = getFirstLevelHash(identityRequest.hashedDiiIdentity.hashedDii, saltProvider.getSnapshot(this.now).getFirstLevelSalt() ); - assertArrayEquals(firstLevelHash, refreshTokenInput.firstLevelHashIdentity.firstLevelHash); + assertArrayEquals(firstLevelHash, refreshTokenRequest.firstLevelHashIdentity.firstLevelHash); setNow(Instant.now().plusSeconds(200)); reset(shutdownHandler); - final RefreshResponse refreshResponse = uid2Service.refreshIdentity(refreshTokenInput); + final RefreshResponse refreshResponse = uid2Service.refreshIdentity(refreshTokenRequest); verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); assertNotNull(refreshResponse); @@ -211,24 +211,24 @@ public void testGenerateAndRefresh(int siteId, TokenVersion tokenVersion) { assertNotNull(refreshResponse.getIdentityResponse()); UIDOperatorVerticleTest.validateAdvertisingToken(refreshResponse.getIdentityResponse().getAdvertisingToken(), tokenVersion, IdentityScope.UID2, IdentityType.Email); - AdvertisingTokenInput advertisingTokenInput2 = tokenEncoder.decodeAdvertisingToken(refreshResponse.getIdentityResponse().getAdvertisingToken()); - assertEquals(this.now.plusSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS), advertisingTokenInput2.expiresAt); - assertEquals(advertisingTokenInput.sourcePublisher.siteId, advertisingTokenInput2.sourcePublisher.siteId); - assertIdentityScopeIdentityType(advertisingTokenInput.rawUidIdentity, - advertisingTokenInput2.rawUidIdentity); - assertEquals(advertisingTokenInput.establishedAt, advertisingTokenInput2.establishedAt); - assertArrayEquals(advertisingTokenInput.rawUidIdentity.rawUid, - advertisingTokenInput2.rawUidIdentity.rawUid); - assertEquals(identityRequest.privacyBits, advertisingTokenInput2.privacyBits); - - RefreshTokenInput refreshTokenInput2 = tokenEncoder.decodeRefreshToken(refreshResponse.getIdentityResponse().getRefreshToken()); - assertEquals(this.now, refreshTokenInput2.createdAt); - assertEquals(this.now.plusSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), refreshTokenInput2.expiresAt); - assertEquals(refreshTokenInput.sourcePublisher.siteId, refreshTokenInput2.sourcePublisher.siteId); - assertIdentityScopeIdentityType(refreshTokenInput.firstLevelHashIdentity, refreshTokenInput2.firstLevelHashIdentity); - assertEquals(refreshTokenInput.firstLevelHashIdentity.establishedAt, refreshTokenInput2.firstLevelHashIdentity.establishedAt); - assertArrayEquals(refreshTokenInput.firstLevelHashIdentity.firstLevelHash, refreshTokenInput2.firstLevelHashIdentity.firstLevelHash); - assertArrayEquals(firstLevelHash, refreshTokenInput2.firstLevelHashIdentity.firstLevelHash); + AdvertisingTokenRequest advertisingTokenRequest2 = tokenEncoder.decodeAdvertisingToken(refreshResponse.getIdentityResponse().getAdvertisingToken()); + assertEquals(this.now.plusSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS), advertisingTokenRequest2.expiresAt); + assertEquals(advertisingTokenRequest.sourcePublisher.siteId, advertisingTokenRequest2.sourcePublisher.siteId); + assertIdentityScopeIdentityType(advertisingTokenRequest.rawUidIdentity, + advertisingTokenRequest2.rawUidIdentity); + assertEquals(advertisingTokenRequest.establishedAt, advertisingTokenRequest2.establishedAt); + assertArrayEquals(advertisingTokenRequest.rawUidIdentity.rawUid, + advertisingTokenRequest2.rawUidIdentity.rawUid); + assertEquals(identityRequest.privacyBits, advertisingTokenRequest2.privacyBits); + + RefreshTokenRequest refreshTokenRequest2 = tokenEncoder.decodeRefreshToken(refreshResponse.getIdentityResponse().getRefreshToken()); + assertEquals(this.now, refreshTokenRequest2.createdAt); + assertEquals(this.now.plusSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), refreshTokenRequest2.expiresAt); + assertEquals(refreshTokenRequest.sourcePublisher.siteId, refreshTokenRequest2.sourcePublisher.siteId); + assertIdentityScopeIdentityType(refreshTokenRequest.firstLevelHashIdentity, refreshTokenRequest2.firstLevelHashIdentity); + assertEquals(refreshTokenRequest.firstLevelHashIdentity.establishedAt, refreshTokenRequest2.firstLevelHashIdentity.establishedAt); + assertArrayEquals(refreshTokenRequest.firstLevelHashIdentity.firstLevelHash, refreshTokenRequest2.firstLevelHashIdentity.firstLevelHash); + assertArrayEquals(firstLevelHash, refreshTokenRequest2.firstLevelHashIdentity.firstLevelHash); } @Test @@ -246,8 +246,8 @@ public void testTestOptOutKey_DoNotRespectOptout() { assertNotNull(identityResponse); assertFalse(identityResponse.isOptedOut()); - final RefreshTokenInput refreshTokenInput = this.tokenEncoder.decodeRefreshToken(identityResponse.getRefreshToken()); - assertEquals(RefreshResponse.Optout, uid2Service.refreshIdentity(refreshTokenInput)); + final RefreshTokenRequest refreshTokenRequest = this.tokenEncoder.decodeRefreshToken(identityResponse.getRefreshToken()); + assertEquals(RefreshResponse.Optout, uid2Service.refreshIdentity(refreshTokenRequest)); } @Test @@ -280,9 +280,9 @@ public void testTestOptOutKeyIdentityScopeMismatch() { verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); assertNotNull(identityResponse); - final RefreshTokenInput refreshTokenInput = this.tokenEncoder.decodeRefreshToken(identityResponse.getRefreshToken()); + final RefreshTokenRequest refreshTokenRequest = this.tokenEncoder.decodeRefreshToken(identityResponse.getRefreshToken()); reset(shutdownHandler); - assertEquals(RefreshResponse.Invalid, uid2Service.refreshIdentity(refreshTokenInput)); + assertEquals(RefreshResponse.Invalid, uid2Service.refreshIdentity(refreshTokenRequest)); verify(shutdownHandler, never()).handleSaltRetrievalResponse(anyBoolean()); } @@ -312,13 +312,13 @@ public void testGenerateTokenForOptOutUser(IdentityType type, String id, Identit .thenReturn(Instant.now().minus(1, ChronoUnit.HOURS)); final IdentityResponse identityResponse; - final AdvertisingTokenInput advertisingTokenInput; + final AdvertisingTokenRequest advertisingTokenRequest; final IdentityResponse identityResponseAfterOptOut; if (scope == IdentityScope.UID2) { identityResponse = uid2Service.generateIdentity(identityRequestForceGenerate); verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); - advertisingTokenInput = validateAndGetToken(tokenEncoder, identityResponse.getAdvertisingToken(), IdentityScope.UID2, hashedDiiIdentity.identityType, identityRequestRespectOptOut.sourcePublisher.siteId); + advertisingTokenRequest = validateAndGetToken(tokenEncoder, identityResponse.getAdvertisingToken(), IdentityScope.UID2, hashedDiiIdentity.identityType, identityRequestRespectOptOut.sourcePublisher.siteId); reset(shutdownHandler); identityResponseAfterOptOut = uid2Service.generateIdentity(identityRequestRespectOptOut); @@ -326,14 +326,14 @@ public void testGenerateTokenForOptOutUser(IdentityType type, String id, Identit identityResponse = euidService.generateIdentity(identityRequestForceGenerate); verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); - advertisingTokenInput = validateAndGetToken(tokenEncoder, identityResponse.getAdvertisingToken(), IdentityScope.EUID, hashedDiiIdentity.identityType, identityRequestRespectOptOut.sourcePublisher.siteId); + advertisingTokenRequest = validateAndGetToken(tokenEncoder, identityResponse.getAdvertisingToken(), IdentityScope.EUID, hashedDiiIdentity.identityType, identityRequestRespectOptOut.sourcePublisher.siteId); reset(shutdownHandler); identityResponseAfterOptOut = euidService.generateIdentity(identityRequestRespectOptOut); } verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); assertNotNull(identityResponse); - assertNotNull(advertisingTokenInput.rawUidIdentity); + assertNotNull(advertisingTokenRequest.rawUidIdentity); assertNotNull(identityResponseAfterOptOut); assertTrue(identityResponseAfterOptOut.getAdvertisingToken() == null || identityResponseAfterOptOut.getAdvertisingToken().isEmpty()); assertTrue(identityResponseAfterOptOut.isOptedOut()); @@ -515,9 +515,9 @@ void testSpecialIdentityOptOutTokenRefresh(TestIdentityInputType type, String id // identity has no optout record, ensure refresh still returns optout when(this.optOutStore.getLatestEntry(any())).thenReturn(null); - final RefreshTokenInput refreshTokenInput = this.tokenEncoder.decodeRefreshToken(identityResponse.getRefreshToken()); + final RefreshTokenRequest refreshTokenRequest = this.tokenEncoder.decodeRefreshToken(identityResponse.getRefreshToken()); reset(shutdownHandler); - assertEquals(RefreshResponse.Optout, (scope == IdentityScope.EUID? euidService: uid2Service).refreshIdentity(refreshTokenInput)); + assertEquals(RefreshResponse.Optout, (scope == IdentityScope.EUID? euidService: uid2Service).refreshIdentity(refreshTokenRequest)); verify(shutdownHandler, never()).handleSaltRetrievalResponse(anyBoolean()); } @@ -557,9 +557,9 @@ void testSpecialIdentityRefreshOptOutGenerate(TestIdentityInputType type, String // identity has no optout record, ensure refresh still returns optout when(this.optOutStore.getLatestEntry(any())).thenReturn(null); - final RefreshTokenInput refreshTokenInput = this.tokenEncoder.decodeRefreshToken(identityResponse.getRefreshToken()); + final RefreshTokenRequest refreshTokenRequest = this.tokenEncoder.decodeRefreshToken(identityResponse.getRefreshToken()); reset(shutdownHandler); - assertEquals(RefreshResponse.Optout, (scope == IdentityScope.EUID? euidService: uid2Service).refreshIdentity(refreshTokenInput)); + assertEquals(RefreshResponse.Optout, (scope == IdentityScope.EUID? euidService: uid2Service).refreshIdentity(refreshTokenRequest)); verify(shutdownHandler, never()).handleSaltRetrievalResponse(anyBoolean()); } @@ -618,19 +618,19 @@ void testSpecialIdentityValidateGenerate(TestIdentityInputType type, String id, when(this.optOutStore.getLatestEntry(any())).thenReturn(Instant.now()); IdentityResponse identityResponse; - AdvertisingTokenInput advertisingTokenInput; + AdvertisingTokenRequest advertisingTokenRequest; if (scope == IdentityScope.EUID) { identityResponse = euidService.generateIdentity(identityRequest); } else { identityResponse = uid2Service.generateIdentity(identityRequest); } - advertisingTokenInput = validateAndGetToken(tokenEncoder, identityResponse.getAdvertisingToken(), scope, identityRequest.hashedDiiIdentity.identityType, identityRequest.sourcePublisher.siteId); + advertisingTokenRequest = validateAndGetToken(tokenEncoder, identityResponse.getAdvertisingToken(), scope, identityRequest.hashedDiiIdentity.identityType, identityRequest.sourcePublisher.siteId); verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); assertNotNull(identityResponse); assertNotEquals(IdentityResponse.OptOutIdentityResponse, identityResponse); - assertNotNull(advertisingTokenInput.rawUidIdentity); + assertNotNull(advertisingTokenRequest.rawUidIdentity); } @@ -693,8 +693,8 @@ void testNormalIdentityOptIn(TestIdentityInputType type, String id, IdentityScop assertNotEquals(identityResponse, IdentityResponse.OptOutIdentityResponse); assertNotNull(identityResponse); - final RefreshTokenInput refreshTokenInput = this.tokenEncoder.decodeRefreshToken(identityResponse.getRefreshToken()); - RefreshResponse refreshResponse = (scope == IdentityScope.EUID? euidService: uid2Service).refreshIdentity(refreshTokenInput); + final RefreshTokenRequest refreshTokenRequest = this.tokenEncoder.decodeRefreshToken(identityResponse.getRefreshToken()); + RefreshResponse refreshResponse = (scope == IdentityScope.EUID? euidService: uid2Service).refreshIdentity(refreshTokenRequest); assertTrue(refreshResponse.isRefreshed()); assertNotNull(refreshResponse.getIdentityResponse()); assertNotEquals(RefreshResponse.Optout, refreshResponse); @@ -743,25 +743,25 @@ void testExpiredSaltsNotifiesShutdownHandler(TestIdentityInputType type, String OptoutCheckPolicy.RespectOptOut, PrivacyBits.fromInt(0), this.now); IdentityResponse identityResponse; - AdvertisingTokenInput advertisingTokenInput; + AdvertisingTokenRequest advertisingTokenRequest; reset(shutdownHandler); if(scope == IdentityScope.EUID) { identityResponse = euidService.generateIdentity(identityRequest); - advertisingTokenInput = validateAndGetToken(tokenEncoder, identityResponse.getAdvertisingToken(), IdentityScope.EUID, identityRequest.hashedDiiIdentity.identityType, identityRequest.sourcePublisher.siteId); + advertisingTokenRequest = validateAndGetToken(tokenEncoder, identityResponse.getAdvertisingToken(), IdentityScope.EUID, identityRequest.hashedDiiIdentity.identityType, identityRequest.sourcePublisher.siteId); } else { identityResponse = uid2Service.generateIdentity(identityRequest); - advertisingTokenInput = validateAndGetToken(tokenEncoder, identityResponse.getAdvertisingToken(), IdentityScope.UID2, identityRequest.hashedDiiIdentity.identityType, identityRequest.sourcePublisher.siteId); + advertisingTokenRequest = validateAndGetToken(tokenEncoder, identityResponse.getAdvertisingToken(), IdentityScope.UID2, identityRequest.hashedDiiIdentity.identityType, identityRequest.sourcePublisher.siteId); } verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(true); verify(shutdownHandler, never()).handleSaltRetrievalResponse(false); assertNotNull(identityResponse); assertNotEquals(IdentityResponse.OptOutIdentityResponse, identityResponse); - assertNotNull(advertisingTokenInput.rawUidIdentity); + assertNotNull(advertisingTokenRequest.rawUidIdentity); - final RefreshTokenInput refreshTokenInput = this.tokenEncoder.decodeRefreshToken(identityResponse.getRefreshToken()); + final RefreshTokenRequest refreshTokenRequest = this.tokenEncoder.decodeRefreshToken(identityResponse.getRefreshToken()); reset(shutdownHandler); - RefreshResponse refreshResponse = (scope == IdentityScope.EUID? euidService: uid2Service).refreshIdentity(refreshTokenInput); + RefreshResponse refreshResponse = (scope == IdentityScope.EUID? euidService: uid2Service).refreshIdentity(refreshTokenRequest); verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(true); verify(shutdownHandler, never()).handleSaltRetrievalResponse(false); assertTrue(refreshResponse.isRefreshed()); diff --git a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java index 452b6ecc9..6ceeca9be 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java @@ -813,14 +813,14 @@ private void assertStatsCollector(String path, String referer, String apiContact assertEquals(siteId, messageItem.getSiteId()); } - private AdvertisingTokenInput validateAndGetToken(EncryptedTokenEncoder encoder, JsonObject body, IdentityType identityType) { //See UID2-79+Token+and+ID+format+v3 + private AdvertisingTokenRequest validateAndGetToken(EncryptedTokenEncoder encoder, JsonObject body, IdentityType identityType) { //See UID2-79+Token+and+ID+format+v3 final String advertisingTokenString = body.getString("advertising_token"); validateAdvertisingToken(advertisingTokenString, getTokenVersion(), getIdentityScope(), identityType); - AdvertisingTokenInput advertisingTokenInput = encoder.decodeAdvertisingToken(advertisingTokenString); + AdvertisingTokenRequest advertisingTokenRequest = encoder.decodeAdvertisingToken(advertisingTokenString); if (getTokenVersion() == TokenVersion.V4) { - assertEquals(identityType, advertisingTokenInput.rawUidIdentity.identityType); + assertEquals(identityType, advertisingTokenRequest.rawUidIdentity.identityType); } - return advertisingTokenInput; + return advertisingTokenRequest; } public static void validateAdvertisingToken(String advertisingTokenString, TokenVersion tokenVersion, IdentityScope identityScope, IdentityType identityType) { @@ -848,11 +848,11 @@ public static void validateAdvertisingToken(String advertisingTokenString, Token } } - RefreshTokenInput decodeRefreshToken(EncryptedTokenEncoder encoder, String refreshTokenString, IdentityType identityType) { - RefreshTokenInput refreshTokenInput = encoder.decodeRefreshToken(refreshTokenString); - assertEquals(getIdentityScope(), refreshTokenInput.firstLevelHashIdentity.identityScope); - assertEquals(identityType, refreshTokenInput.firstLevelHashIdentity.identityType); - return refreshTokenInput; + RefreshTokenRequest decodeRefreshToken(EncryptedTokenEncoder encoder, String refreshTokenString, IdentityType identityType) { + RefreshTokenRequest refreshTokenRequest = encoder.decodeRefreshToken(refreshTokenString); + assertEquals(getIdentityScope(), refreshTokenRequest.firstLevelHashIdentity.identityScope); + assertEquals(identityType, refreshTokenRequest.firstLevelHashIdentity.identityType); + return refreshTokenRequest; } @ParameterizedTest @@ -1164,23 +1164,23 @@ void tokenGenerateOptOutToken(String policyParameterKey, String identity, Identi EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); - AdvertisingTokenInput advertisingTokenInput = validateAndGetToken(encoder, body, identityType); - RefreshTokenInput refreshTokenInput = encoder.decodeRefreshToken(body.getString("decrypted_refresh_token")); + AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, body, identityType); + RefreshTokenRequest refreshTokenRequest = encoder.decodeRefreshToken(body.getString("decrypted_refresh_token")); final byte[] rawUid = getRawUidFromIdentity(identityType, optOutTokenInput.getNormalized(), firstLevelSalt, rotatingSalt123.getSalt()); final byte[] firstLevelHash = TokenUtils.getFirstLevelHashFromIdentity(optOutTokenInput.getNormalized(), firstLevelSalt); - assertArrayEquals(rawUid, advertisingTokenInput.rawUidIdentity.rawUid); - assertArrayEquals(firstLevelHash, refreshTokenInput.firstLevelHashIdentity.firstLevelHash); + assertArrayEquals(rawUid, advertisingTokenRequest.rawUidIdentity.rawUid); + assertArrayEquals(firstLevelHash, refreshTokenRequest.firstLevelHashIdentity.firstLevelHash); String advertisingTokenString = body.getString("advertising_token"); final Instant now = Instant.now(); final String token = advertisingTokenString; final boolean matchedOptedOutIdentity = this.uidOperatorVerticle.getIdService().advertisingTokenMatches(token, optOutTokenInput.toHashedDiiIdentity(getIdentityScope()), now); assertTrue(matchedOptedOutIdentity); - assertFalse(advertisingTokenInput.privacyBits.isClientSideTokenGenerated()); - assertTrue(advertisingTokenInput.privacyBits.isClientSideTokenOptedOut()); + assertFalse(advertisingTokenRequest.privacyBits.isClientSideTokenGenerated()); + assertTrue(advertisingTokenRequest.privacyBits.isClientSideTokenOptedOut()); assertTokenStatusMetrics( 201, @@ -1224,10 +1224,10 @@ void tokenGenerateForEmail(String apiVersion, Vertx vertx, VertxTestContext test assertNotNull(body); EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); - AdvertisingTokenInput advertisingTokenInput = validateAndGetToken(encoder, body, IdentityType.Email); - RefreshTokenInput refreshTokenInput = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), IdentityType.Email); + AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, body, IdentityType.Email); + RefreshTokenRequest refreshTokenRequest = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), IdentityType.Email); - assertAdvertisingTokenRefreshTokenInputs(advertisingTokenInput, refreshTokenInput, clientSiteId, + assertAdvertisingTokenRefreshTokenRequests(advertisingTokenRequest, refreshTokenRequest, clientSiteId, getRawUidFromIdentity(IdentityType.Email, emailAddress, firstLevelSalt, rotatingSalt123.getSalt()), PrivacyBits.DEFAULT, body, @@ -1239,25 +1239,25 @@ void tokenGenerateForEmail(String apiVersion, Vertx vertx, VertxTestContext test }); } - public void assertAdvertisingTokenRefreshTokenInputs(AdvertisingTokenInput advertisingTokenInput, RefreshTokenInput refreshTokenInput, - int expectedClientSiteId, byte[] expectedRawUidIdentity, PrivacyBits expectedPrivacyBits, JsonObject identityResponse, byte[] firstLevelHashIdentity) { + public void assertAdvertisingTokenRefreshTokenRequests(AdvertisingTokenRequest advertisingTokenRequest, RefreshTokenRequest refreshTokenRequest, + int expectedClientSiteId, byte[] expectedRawUidIdentity, PrivacyBits expectedPrivacyBits, JsonObject identityResponse, byte[] firstLevelHashIdentity) { - assertEquals(expectedClientSiteId, advertisingTokenInput.sourcePublisher.siteId); - assertEquals(expectedClientSiteId, refreshTokenInput.sourcePublisher.siteId); - assertArrayEquals(expectedRawUidIdentity, advertisingTokenInput.rawUidIdentity.rawUid); + assertEquals(expectedClientSiteId, advertisingTokenRequest.sourcePublisher.siteId); + assertEquals(expectedClientSiteId, refreshTokenRequest.sourcePublisher.siteId); + assertArrayEquals(expectedRawUidIdentity, advertisingTokenRequest.rawUidIdentity.rawUid); - verifyPrivacyBits(expectedPrivacyBits, advertisingTokenInput, refreshTokenInput); - verifyFirstLevelHashIdentityAndEstablishedAt(firstLevelHashIdentity, refreshTokenInput, identityResponse, advertisingTokenInput.establishedAt); + verifyPrivacyBits(expectedPrivacyBits, advertisingTokenRequest, refreshTokenRequest); + verifyFirstLevelHashIdentityAndEstablishedAt(firstLevelHashIdentity, refreshTokenRequest, identityResponse, advertisingTokenRequest.establishedAt); assertEqualsClose(now.plusMillis(identityExpiresAfter.toMillis()), Instant.ofEpochMilli(identityResponse.getLong("identity_expires")), 10); assertEqualsClose(now.plusMillis(refreshExpiresAfter.toMillis()), Instant.ofEpochMilli(identityResponse.getLong("refresh_expires")), 10); assertEqualsClose(now.plusMillis(refreshIdentityAfter.toMillis()), Instant.ofEpochMilli(identityResponse.getLong("refresh_from")), 10); } - public void verifyPrivacyBits(PrivacyBits expectedValue, AdvertisingTokenInput advertisingTokenInput, - RefreshTokenInput refreshTokenInput) { - assertEquals(advertisingTokenInput.privacyBits, expectedValue); - assertEquals(advertisingTokenInput.privacyBits, refreshTokenInput.privacyBits); + public void verifyPrivacyBits(PrivacyBits expectedValue, AdvertisingTokenRequest advertisingTokenRequest, + RefreshTokenRequest refreshTokenRequest) { + assertEquals(advertisingTokenRequest.privacyBits, expectedValue); + assertEquals(advertisingTokenRequest.privacyBits, refreshTokenRequest.privacyBits); } @ParameterizedTest @@ -1281,10 +1281,10 @@ void tokenGenerateForEmailHash(String apiVersion, Vertx vertx, VertxTestContext assertNotNull(body); EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); - AdvertisingTokenInput advertisingTokenInput = validateAndGetToken(encoder, body, IdentityType.Email); - RefreshTokenInput refreshTokenInput = decodeRefreshToken(encoder, apiVersion.equals("v2") ? body.getString("decrypted_refresh_token") : body.getString("refresh_token"), IdentityType.Email); + AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, body, IdentityType.Email); + RefreshTokenRequest refreshTokenRequest = decodeRefreshToken(encoder, apiVersion.equals("v2") ? body.getString("decrypted_refresh_token") : body.getString("refresh_token"), IdentityType.Email); - assertAdvertisingTokenRefreshTokenInputs(advertisingTokenInput, refreshTokenInput, clientSiteId, + assertAdvertisingTokenRefreshTokenRequests(advertisingTokenRequest, refreshTokenRequest, clientSiteId, getRawUidFromIdentityHash(IdentityType.Email, emailHash, firstLevelSalt, rotatingSalt123.getSalt()), PrivacyBits.DEFAULT, body, @@ -1311,12 +1311,12 @@ void tokenGenerateThenRefresh(String apiVersion, Vertx vertx, VertxTestContext t String genRefreshToken = bodyJson.getString("refresh_token"); EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); - AdvertisingTokenInput firstAdvertisingTokenInput = validateAndGetToken(encoder, bodyJson, + AdvertisingTokenRequest firstAdvertisingTokenRequest = validateAndGetToken(encoder, bodyJson, IdentityType.Email); - RefreshTokenInput firstRefreshTokenInput = decodeRefreshToken(encoder, bodyJson.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), IdentityType.Email); + RefreshTokenRequest firstRefreshTokenRequest = decodeRefreshToken(encoder, bodyJson.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), IdentityType.Email); - assertEquals(firstAdvertisingTokenInput.establishedAt, firstRefreshTokenInput.firstLevelHashIdentity.establishedAt); + assertEquals(firstAdvertisingTokenRequest.establishedAt, firstRefreshTokenRequest.firstLevelHashIdentity.establishedAt); when(this.optOutStore.getLatestEntry(any())).thenReturn(null); @@ -1324,7 +1324,7 @@ void tokenGenerateThenRefresh(String apiVersion, Vertx vertx, VertxTestContext t byte[] expectedRawUidIdentity = getRawUidFromIdentity(IdentityType.Email, emailAddress, firstLevelSalt, rotatingSalt123.getSalt()); byte[] expectedFirstLevelHashIdentity = TokenUtils.getFirstLevelHashFromIdentity(emailAddress, firstLevelSalt); - assertAdvertisingTokenRefreshTokenInputs(firstAdvertisingTokenInput, firstRefreshTokenInput, clientSiteId, + assertAdvertisingTokenRefreshTokenRequests(firstAdvertisingTokenRequest, firstRefreshTokenRequest, clientSiteId, expectedRawUidIdentity, PrivacyBits.DEFAULT, bodyJson, @@ -1338,31 +1338,31 @@ void tokenGenerateThenRefresh(String apiVersion, Vertx vertx, VertxTestContext t assertNotNull(refreshBody); - AdvertisingTokenInput advertisingTokenInput = validateAndGetToken(encoder, refreshBody, IdentityType.Email); + AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, refreshBody, IdentityType.Email); String refreshTokenStringNew = refreshBody.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"); assertNotEquals(genRefreshToken, refreshTokenStringNew); - RefreshTokenInput refreshTokenInput = decodeRefreshToken(encoder, refreshTokenStringNew, IdentityType.Email); + RefreshTokenRequest refreshTokenRequest = decodeRefreshToken(encoder, refreshTokenStringNew, IdentityType.Email); // assert if the ad/refresh tokens from original token/generate is same as the ad/refresh tokens from token/refresh - assertAdvertisingTokenRefreshTokenInputs( - advertisingTokenInput, - firstRefreshTokenInput, + assertAdvertisingTokenRefreshTokenRequests( + advertisingTokenRequest, + firstRefreshTokenRequest, clientSiteId, expectedRawUidIdentity, PrivacyBits.DEFAULT, bodyJson, expectedFirstLevelHashIdentity); - assertAdvertisingTokenRefreshTokenInputs( - firstAdvertisingTokenInput, - refreshTokenInput, + assertAdvertisingTokenRefreshTokenRequests( + firstAdvertisingTokenRequest, + refreshTokenRequest, clientSiteId, expectedRawUidIdentity, PrivacyBits.DEFAULT, bodyJson, expectedFirstLevelHashIdentity); - assertAdvertisingTokenRefreshTokenInputs( - advertisingTokenInput, - refreshTokenInput, + assertAdvertisingTokenRefreshTokenRequests( + advertisingTokenRequest, + refreshTokenRequest, clientSiteId, expectedRawUidIdentity, PrivacyBits.DEFAULT, @@ -1411,18 +1411,18 @@ void tokenGenerateThenRefreshSaltsExpired(String apiVersion, Vertx vertx, VertxT assertNotNull(refreshBody); EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); - AdvertisingTokenInput advertisingTokenInput = validateAndGetToken(encoder, refreshBody, IdentityType.Email); + AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, refreshBody, IdentityType.Email); - assertFalse(advertisingTokenInput.privacyBits.isClientSideTokenGenerated()); - assertFalse(advertisingTokenInput.privacyBits.isClientSideTokenOptedOut()); - assertEquals(clientSiteId, advertisingTokenInput.sourcePublisher.siteId); - assertArrayEquals(getRawUidFromIdentity(IdentityType.Email, emailAddress, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenInput.rawUidIdentity.rawUid); + assertFalse(advertisingTokenRequest.privacyBits.isClientSideTokenGenerated()); + assertFalse(advertisingTokenRequest.privacyBits.isClientSideTokenOptedOut()); + assertEquals(clientSiteId, advertisingTokenRequest.sourcePublisher.siteId); + assertArrayEquals(getRawUidFromIdentity(IdentityType.Email, emailAddress, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenRequest.rawUidIdentity.rawUid); String refreshTokenStringNew = refreshBody.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"); assertNotEquals(genRefreshToken, refreshTokenStringNew); - RefreshTokenInput refreshTokenInput = decodeRefreshToken(encoder, refreshTokenStringNew, IdentityType.Email); - assertEquals(clientSiteId, refreshTokenInput.sourcePublisher.siteId); - assertArrayEquals(TokenUtils.getFirstLevelHashFromIdentity(emailAddress, firstLevelSalt), refreshTokenInput.firstLevelHashIdentity.firstLevelHash); + RefreshTokenRequest refreshTokenRequest = decodeRefreshToken(encoder, refreshTokenStringNew, IdentityType.Email); + assertEquals(clientSiteId, refreshTokenRequest.sourcePublisher.siteId); + assertArrayEquals(TokenUtils.getFirstLevelHashFromIdentity(emailAddress, firstLevelSalt), refreshTokenRequest.firstLevelHashIdentity.firstLevelHash); assertEqualsClose(now.plusMillis(identityExpiresAfter.toMillis()), Instant.ofEpochMilli(refreshBody.getLong("identity_expires")), 10); assertEqualsClose(now.plusMillis(refreshExpiresAfter.toMillis()), Instant.ofEpochMilli(refreshBody.getLong("refresh_expires")), 10); @@ -1589,13 +1589,13 @@ void tokenGenerateUsingCustomSiteKey(String apiVersion, Vertx vertx, VertxTestCo assertNotNull(body); EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); - AdvertisingTokenInput advertisingTokenInput = validateAndGetToken(encoder, body, IdentityType.Email); - assertEquals(clientSiteId, advertisingTokenInput.sourcePublisher.siteId); - assertArrayEquals(getRawUidFromIdentity(IdentityType.Email, emailAddress, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenInput.rawUidIdentity.rawUid); + AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, body, IdentityType.Email); + assertEquals(clientSiteId, advertisingTokenRequest.sourcePublisher.siteId); + assertArrayEquals(getRawUidFromIdentity(IdentityType.Email, emailAddress, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenRequest.rawUidIdentity.rawUid); - RefreshTokenInput refreshTokenInput = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), IdentityType.Email); - assertEquals(clientSiteId, refreshTokenInput.sourcePublisher.siteId); - assertArrayEquals(TokenUtils.getFirstLevelHashFromIdentity(emailAddress, firstLevelSalt), refreshTokenInput.firstLevelHashIdentity.firstLevelHash); + RefreshTokenRequest refreshTokenRequest = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), IdentityType.Email); + assertEquals(clientSiteId, refreshTokenRequest.sourcePublisher.siteId); + assertArrayEquals(TokenUtils.getFirstLevelHashFromIdentity(emailAddress, firstLevelSalt), refreshTokenRequest.firstLevelHashIdentity.firstLevelHash); testContext.completeNow(); }); @@ -1623,18 +1623,18 @@ void tokenGenerateSaltsExpired(String apiVersion, Vertx vertx, VertxTestContext assertNotNull(body); EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); - AdvertisingTokenInput advertisingTokenInput = validateAndGetToken(encoder, body, IdentityType.Email); + AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, body, IdentityType.Email); - assertTrue(advertisingTokenInput.privacyBits.isLegacyBitSet()); - assertEquals(advertisingTokenInput.privacyBits, PrivacyBits.DEFAULT); - assertFalse(advertisingTokenInput.privacyBits.isClientSideTokenGenerated()); - assertFalse(advertisingTokenInput.privacyBits.isClientSideTokenOptedOut()); - assertEquals(clientSiteId, advertisingTokenInput.sourcePublisher.siteId); - assertArrayEquals(getRawUidFromIdentity(IdentityType.Email, emailAddress, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenInput.rawUidIdentity.rawUid); + assertTrue(advertisingTokenRequest.privacyBits.isLegacyBitSet()); + assertEquals(advertisingTokenRequest.privacyBits, PrivacyBits.DEFAULT); + assertFalse(advertisingTokenRequest.privacyBits.isClientSideTokenGenerated()); + assertFalse(advertisingTokenRequest.privacyBits.isClientSideTokenOptedOut()); + assertEquals(clientSiteId, advertisingTokenRequest.sourcePublisher.siteId); + assertArrayEquals(getRawUidFromIdentity(IdentityType.Email, emailAddress, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenRequest.rawUidIdentity.rawUid); - RefreshTokenInput refreshTokenInput = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), IdentityType.Email); - assertEquals(clientSiteId, refreshTokenInput.sourcePublisher.siteId); - assertArrayEquals(TokenUtils.getFirstLevelHashFromIdentity(emailAddress, firstLevelSalt), refreshTokenInput.firstLevelHashIdentity.firstLevelHash); + RefreshTokenRequest refreshTokenRequest = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), IdentityType.Email); + assertEquals(clientSiteId, refreshTokenRequest.sourcePublisher.siteId); + assertArrayEquals(TokenUtils.getFirstLevelHashFromIdentity(emailAddress, firstLevelSalt), refreshTokenRequest.firstLevelHashIdentity.firstLevelHash); assertEqualsClose(now.plusMillis(identityExpiresAfter.toMillis()), Instant.ofEpochMilli(body.getLong("identity_expires")), 10); assertEqualsClose(now.plusMillis(refreshExpiresAfter.toMillis()), Instant.ofEpochMilli(body.getLong("refresh_expires")), 10); @@ -2537,10 +2537,10 @@ void tokenGenerateForPhone(String apiVersion, Vertx vertx, VertxTestContext test assertNotNull(body); EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); - AdvertisingTokenInput advertisingTokenInput = validateAndGetToken(encoder, body, IdentityType.Phone); - RefreshTokenInput refreshTokenInput = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), IdentityType.Phone); + AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, body, IdentityType.Phone); + RefreshTokenRequest refreshTokenRequest = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), IdentityType.Phone); - assertAdvertisingTokenRefreshTokenInputs(advertisingTokenInput, refreshTokenInput, clientSiteId, + assertAdvertisingTokenRefreshTokenRequests(advertisingTokenRequest, refreshTokenRequest, clientSiteId, getRawUidFromIdentity(IdentityType.Phone, phone, firstLevelSalt, rotatingSalt123.getSalt()), PrivacyBits.DEFAULT, body, @@ -2551,15 +2551,15 @@ void tokenGenerateForPhone(String apiVersion, Vertx vertx, VertxTestContext test } void verifyFirstLevelHashIdentityAndEstablishedAt(byte[] expectedFirstLevelHash, - RefreshTokenInput refreshTokenInput, + RefreshTokenRequest refreshTokenRequest, JsonObject receivedJsonBody, Instant expectedEstablishedTime) { - assertArrayEquals(expectedFirstLevelHash, refreshTokenInput.firstLevelHashIdentity.firstLevelHash); - assertEquals(expectedEstablishedTime, refreshTokenInput.firstLevelHashIdentity.establishedAt); - assertTrue(refreshTokenInput.firstLevelHashIdentity.establishedAt.toEpochMilli() < receivedJsonBody.getLong("identity_expires") ); - assertTrue(refreshTokenInput.firstLevelHashIdentity.establishedAt.toEpochMilli() < receivedJsonBody.getLong("refresh_expires") ); - assertTrue(refreshTokenInput.firstLevelHashIdentity.establishedAt.toEpochMilli() < receivedJsonBody.getLong("refresh_from") ); + assertArrayEquals(expectedFirstLevelHash, refreshTokenRequest.firstLevelHashIdentity.firstLevelHash); + assertEquals(expectedEstablishedTime, refreshTokenRequest.firstLevelHashIdentity.establishedAt); + assertTrue(refreshTokenRequest.firstLevelHashIdentity.establishedAt.toEpochMilli() < receivedJsonBody.getLong("identity_expires") ); + assertTrue(refreshTokenRequest.firstLevelHashIdentity.establishedAt.toEpochMilli() < receivedJsonBody.getLong("refresh_expires") ); + assertTrue(refreshTokenRequest.firstLevelHashIdentity.establishedAt.toEpochMilli() < receivedJsonBody.getLong("refresh_from") ); } @ParameterizedTest @@ -2582,10 +2582,10 @@ void tokenGenerateForPhoneHash(String apiVersion, Vertx vertx, VertxTestContext assertNotNull(body); EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); - AdvertisingTokenInput advertisingTokenInput = validateAndGetToken(encoder, body, IdentityType.Phone); - RefreshTokenInput refreshTokenInput = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), IdentityType.Phone); + AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, body, IdentityType.Phone); + RefreshTokenRequest refreshTokenRequest = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), IdentityType.Phone); - assertAdvertisingTokenRefreshTokenInputs(advertisingTokenInput, refreshTokenInput, clientSiteId, + assertAdvertisingTokenRefreshTokenRequests(advertisingTokenRequest, refreshTokenRequest, clientSiteId, getRawUidFromIdentity(IdentityType.Phone, phone, firstLevelSalt, rotatingSalt123.getSalt()), PrivacyBits.DEFAULT, body, @@ -2611,17 +2611,17 @@ void tokenGenerateThenRefreshForPhone(String apiVersion, Vertx vertx, VertxTestC assertNotNull(bodyJson); EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); - AdvertisingTokenInput firstAdvertisingTokenInput = validateAndGetToken(encoder, bodyJson, + AdvertisingTokenRequest firstAdvertisingTokenRequest = validateAndGetToken(encoder, bodyJson, IdentityType.Phone); String genRefreshToken = bodyJson.getString("refresh_token"); - RefreshTokenInput firstRefreshTokenInput = decodeRefreshToken(encoder, bodyJson.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), IdentityType.Phone); + RefreshTokenRequest firstRefreshTokenRequest = decodeRefreshToken(encoder, bodyJson.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), IdentityType.Phone); when(this.optOutStore.getLatestEntry(any())).thenReturn(null); byte[] expectedRawUidIdentity = getRawUidFromIdentity(IdentityType.Phone, phone, firstLevelSalt, rotatingSalt123.getSalt()); byte[] expectedFirstLevelHashIdentity = TokenUtils.getFirstLevelHashFromIdentity(phone, firstLevelSalt); - assertAdvertisingTokenRefreshTokenInputs(firstAdvertisingTokenInput, firstRefreshTokenInput, clientSiteId, + assertAdvertisingTokenRefreshTokenRequests(firstAdvertisingTokenRequest, firstRefreshTokenRequest, clientSiteId, expectedRawUidIdentity, PrivacyBits.DEFAULT, bodyJson, @@ -2634,31 +2634,31 @@ void tokenGenerateThenRefreshForPhone(String apiVersion, Vertx vertx, VertxTestC assertNotNull(refreshBody); - AdvertisingTokenInput advertisingTokenInput = validateAndGetToken(encoder, refreshBody, IdentityType.Phone); + AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, refreshBody, IdentityType.Phone); String refreshTokenStringNew = refreshBody.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"); assertNotEquals(genRefreshToken, refreshTokenStringNew); - RefreshTokenInput refreshTokenInput = decodeRefreshToken(encoder, refreshTokenStringNew, IdentityType.Phone); + RefreshTokenRequest refreshTokenRequest = decodeRefreshToken(encoder, refreshTokenStringNew, IdentityType.Phone); // assert if the ad/refresh tokens from original token/generate is same as the ad/refresh tokens from token/refresh - assertAdvertisingTokenRefreshTokenInputs( - advertisingTokenInput, - firstRefreshTokenInput, + assertAdvertisingTokenRefreshTokenRequests( + advertisingTokenRequest, + firstRefreshTokenRequest, clientSiteId, expectedRawUidIdentity, PrivacyBits.DEFAULT, bodyJson, expectedFirstLevelHashIdentity); - assertAdvertisingTokenRefreshTokenInputs( - firstAdvertisingTokenInput, - refreshTokenInput, + assertAdvertisingTokenRefreshTokenRequests( + firstAdvertisingTokenRequest, + refreshTokenRequest, clientSiteId, expectedRawUidIdentity, PrivacyBits.DEFAULT, bodyJson, expectedFirstLevelHashIdentity); - assertAdvertisingTokenRefreshTokenInputs( - advertisingTokenInput, - refreshTokenInput, + assertAdvertisingTokenRefreshTokenRequests( + advertisingTokenRequest, + refreshTokenRequest, clientSiteId, expectedRawUidIdentity, PrivacyBits.DEFAULT, @@ -4085,14 +4085,14 @@ void cstgUserOptsOutAfterTokenGenerate(String id, IdentityType identityType, Ver assertEquals("success", response.getString("status")); final JsonObject genBody = response.getJsonObject("body"); - final AdvertisingTokenInput advertisingTokenInput = validateAndGetToken(encoder, genBody, identityType); - final RefreshTokenInput refreshTokenInput = decodeRefreshToken(encoder, decodeV2RefreshToken(response), identityType); + final AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, genBody, identityType); + final RefreshTokenRequest refreshTokenRequest = decodeRefreshToken(encoder, decodeV2RefreshToken(response), identityType); - assertAreClientSideGeneratedTokens(advertisingTokenInput, refreshTokenInput, clientSideTokenGenerateSiteId, identityType, id); + assertAreClientSideGeneratedTokens(advertisingTokenRequest, refreshTokenRequest, clientSideTokenGenerateSiteId, identityType, id); // When we refresh the token the user has opted out. when(optOutStore.getLatestEntry(any(FirstLevelHashIdentity.class))) - .thenReturn(advertisingTokenInput.establishedAt.plusSeconds(1)); + .thenReturn(advertisingTokenRequest.establishedAt.plusSeconds(1)); sendTokenRefresh("v2", vertx, testContext, genBody.getString("refresh_token"), genBody.getString("refresh_response_key"), 200, refreshRespJson -> { assertEquals("optout", refreshRespJson.getString("status")); @@ -4149,9 +4149,9 @@ void cstgSuccessForBothOptedAndNonOptedOutTest(boolean optOutExpected, String id decodeV2RefreshToken(respJson); EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); - AdvertisingTokenInput advertisingTokenInput = validateAndGetToken(encoder, genBody, identityType); + AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, genBody, identityType); - RefreshTokenInput refreshTokenInput = decodeRefreshToken(encoder, genBody.getString("decrypted_refresh_token"), identityType); + RefreshTokenRequest refreshTokenRequest = decodeRefreshToken(encoder, genBody.getString("decrypted_refresh_token"), identityType); @@ -4162,13 +4162,13 @@ void cstgSuccessForBothOptedAndNonOptedOutTest(boolean optOutExpected, String id expectedPrivacyBits.setLegacyBit(); expectedPrivacyBits.setClientSideTokenGenerate(); - assertAdvertisingTokenRefreshTokenInputs(advertisingTokenInput, refreshTokenInput, + assertAdvertisingTokenRefreshTokenRequests(advertisingTokenRequest, refreshTokenRequest, clientSideTokenGenerateSiteId, expectedRawUidIdentity, expectedPrivacyBits, genBody, expectedFirstLevelHashIdentity); - assertAreClientSideGeneratedTokens(advertisingTokenInput, refreshTokenInput, clientSideTokenGenerateSiteId, identityType, id); + assertAreClientSideGeneratedTokens(advertisingTokenRequest, refreshTokenRequest, clientSideTokenGenerateSiteId, identityType, id); assertEqualsClose(Instant.now().plusMillis(identityExpiresAfter.toMillis()), Instant.ofEpochMilli(genBody.getLong("identity_expires")), 10); assertEqualsClose(Instant.now().plusMillis(refreshExpiresAfter.toMillis()), Instant.ofEpochMilli(genBody.getLong("refresh_expires")), 10); assertEqualsClose(Instant.now().plusMillis(refreshIdentityAfter.toMillis()), Instant.ofEpochMilli(genBody.getLong("refresh_from")), 10); @@ -4189,13 +4189,13 @@ void cstgSuccessForBothOptedAndNonOptedOutTest(boolean optOutExpected, String id EncryptedTokenEncoder encoder2 = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); //make sure the new advertising token from refresh looks right - AdvertisingTokenInput adTokenFromRefresh = validateAndGetToken(encoder2, refreshBody, identityType); + AdvertisingTokenRequest adTokenFromRefresh = validateAndGetToken(encoder2, refreshBody, identityType); String refreshTokenStringNew = refreshBody.getString("decrypted_refresh_token"); assertNotEquals(genRefreshToken, refreshTokenStringNew); - RefreshTokenInput refreshTokenAfterRefreshSource = decodeRefreshToken(encoder, refreshTokenStringNew, identityType); + RefreshTokenRequest refreshTokenAfterRefreshSource = decodeRefreshToken(encoder, refreshTokenStringNew, identityType); - assertAdvertisingTokenRefreshTokenInputs(adTokenFromRefresh, refreshTokenAfterRefreshSource, + assertAdvertisingTokenRefreshTokenRequests(adTokenFromRefresh, refreshTokenAfterRefreshSource, clientSideTokenGenerateSiteId, expectedRawUidIdentity, expectedPrivacyBits, @@ -4295,18 +4295,18 @@ void cstgInvalidInput(String identityType, String rawUID, Vertx vertx, VertxTest }); } - private void assertAreClientSideGeneratedTokens(AdvertisingTokenInput advertisingTokenInput, RefreshTokenInput refreshTokenInput, int siteId, IdentityType identityType, String identity) { - assertAreClientSideGeneratedTokens(advertisingTokenInput, - refreshTokenInput, + private void assertAreClientSideGeneratedTokens(AdvertisingTokenRequest advertisingTokenRequest, RefreshTokenRequest refreshTokenRequest, int siteId, IdentityType identityType, String identity) { + assertAreClientSideGeneratedTokens(advertisingTokenRequest, + refreshTokenRequest, siteId, identityType, identity, false); } - private void assertAreClientSideGeneratedTokens(AdvertisingTokenInput advertisingTokenInput, RefreshTokenInput refreshTokenInput, int siteId, IdentityType identityType, String identity, boolean expectedOptOut) { - final PrivacyBits advertisingTokenPrivacyBits = advertisingTokenInput.privacyBits; - final PrivacyBits refreshTokenPrivacyBits = refreshTokenInput.privacyBits; + private void assertAreClientSideGeneratedTokens(AdvertisingTokenRequest advertisingTokenRequest, RefreshTokenRequest refreshTokenRequest, int siteId, IdentityType identityType, String identity, boolean expectedOptOut) { + final PrivacyBits advertisingTokenPrivacyBits = advertisingTokenRequest.privacyBits; + final PrivacyBits refreshTokenPrivacyBits = refreshTokenRequest.privacyBits; final byte[] rawUid = getRawUidFromIdentity(identityType, identity, @@ -4322,11 +4322,11 @@ private void assertAreClientSideGeneratedTokens(AdvertisingTokenInput advertisin () -> assertTrue(refreshTokenPrivacyBits.isClientSideTokenGenerated(), "Refresh token privacy bits CSTG flag is incorrect"), () -> assertEquals(expectedOptOut, refreshTokenPrivacyBits.isClientSideTokenOptedOut(), "Refresh token privacy bits CSTG optout flag is incorrect"), - () -> assertEquals(siteId, advertisingTokenInput.sourcePublisher.siteId, "Advertising token site ID is incorrect"), - () -> assertEquals(siteId, refreshTokenInput.sourcePublisher.siteId, "Refresh token site ID is incorrect"), + () -> assertEquals(siteId, advertisingTokenRequest.sourcePublisher.siteId, "Advertising token site ID is incorrect"), + () -> assertEquals(siteId, refreshTokenRequest.sourcePublisher.siteId, "Refresh token site ID is incorrect"), - () -> assertArrayEquals(rawUid, advertisingTokenInput.rawUidIdentity.rawUid, "Advertising token ID is incorrect"), - () -> assertArrayEquals(firstLevelHash, refreshTokenInput.firstLevelHashIdentity.firstLevelHash, "Refresh token ID is incorrect") + () -> assertArrayEquals(rawUid, advertisingTokenRequest.rawUidIdentity.rawUid, "Advertising token ID is incorrect"), + () -> assertArrayEquals(firstLevelHash, refreshTokenRequest.firstLevelHashIdentity.firstLevelHash, "Refresh token ID is incorrect") ); } @@ -4556,16 +4556,16 @@ void tokenGenerateRotatingKeysets_GENERATOR(String testRun, Vertx vertx, VertxTe assertNotNull(body); EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); - AdvertisingTokenInput advertisingTokenInput = validateAndGetToken(encoder, body, IdentityType.Email); - assertEquals(clientSiteId, advertisingTokenInput.sourcePublisher.siteId); + AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, body, IdentityType.Email); + assertEquals(clientSiteId, advertisingTokenRequest.sourcePublisher.siteId); //Uses a key from default keyset int clientKeyId; - if (advertisingTokenInput.version == TokenVersion.V3 || advertisingTokenInput.version == TokenVersion.V4) { + if (advertisingTokenRequest.version == TokenVersion.V3 || advertisingTokenRequest.version == TokenVersion.V4) { String advertisingTokenString = body.getString("advertising_token"); byte[] bytes = null; - if (advertisingTokenInput.version == TokenVersion.V3) { + if (advertisingTokenRequest.version == TokenVersion.V3) { bytes = EncodingUtils.fromBase64(advertisingTokenString); - } else if (advertisingTokenInput.version == TokenVersion.V4) { + } else if (advertisingTokenRequest.version == TokenVersion.V4) { bytes = Uid2Base64UrlCoder.decode(advertisingTokenString); //same as V3 but use Base64URL encoding } final Buffer b = Buffer.buffer(bytes); @@ -4575,7 +4575,7 @@ void tokenGenerateRotatingKeysets_GENERATOR(String testRun, Vertx vertx, VertxTe final Buffer masterPayload = Buffer.buffer(masterPayloadBytes); clientKeyId = masterPayload.getInt(29); } else { - clientKeyId = advertisingTokenInput.sourcePublisher.clientKeyId; + clientKeyId = advertisingTokenRequest.sourcePublisher.clientKeyId; } switch (testRun) { case "MultiKeysets": From bfde65ca8da3f7302fc1f719551a0e954dbc26dc Mon Sep 17 00:00:00 2001 From: Ian Nara Date: Tue, 5 Nov 2024 14:21:18 -0700 Subject: [PATCH 114/431] add back use metrics for the operator-served SDKs --- .../vertx/ClientVersionCapturingHandler.java | 52 +++++++++++++++++++ .../operator/vertx/UIDOperatorVerticle.java | 1 + 2 files changed, 53 insertions(+) create mode 100644 src/main/java/com/uid2/operator/vertx/ClientVersionCapturingHandler.java diff --git a/src/main/java/com/uid2/operator/vertx/ClientVersionCapturingHandler.java b/src/main/java/com/uid2/operator/vertx/ClientVersionCapturingHandler.java new file mode 100644 index 000000000..d63626952 --- /dev/null +++ b/src/main/java/com/uid2/operator/vertx/ClientVersionCapturingHandler.java @@ -0,0 +1,52 @@ +package com.uid2.operator.vertx; + +import com.uid2.shared.Const; +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.Metrics; +import io.vertx.core.Handler; +import io.vertx.ext.web.RoutingContext; + +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.Map; + +public class ClientVersionCapturingHandler implements Handler { + private final Map _clientVersionCounters = new HashMap<>(); + + public ClientVersionCapturingHandler(String dir, String whitelistGlob) throws IOException { + try (DirectoryStream dirStream = Files.newDirectoryStream(Paths.get(dir), whitelistGlob)) { + dirStream.forEach(path -> { + final String version = getFileNameWithoutExtension(path); + final Counter counter = Counter + .builder("uid2.client_sdk_versions") + .description("counter for how many http requests are processed per each client sdk version") + .tags("client_version", version) + .register(Metrics.globalRegistry); + _clientVersionCounters.put(version, counter); + }); + } + } + @Override + public void handle(RoutingContext context) { + String clientVersion = context.request().headers().get(Const.Http.ClientVersionHeader); + if (clientVersion == null) { + clientVersion = !context.queryParam("client").isEmpty() ? context.queryParam("client").get(0) : null; + } + if (clientVersion != null) { + final Counter counter = _clientVersionCounters.get(clientVersion); + if (counter != null) { + counter.increment(); + } + } + context.next(); + } + + private static String getFileNameWithoutExtension(Path path) { + final String fileName = path.getFileName().toString(); + return fileName.indexOf(".") > 0 ? fileName.substring(0, fileName.lastIndexOf(".")) : fileName; + } +} \ No newline at end of file diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index 210b08571..245aae445 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -218,6 +218,7 @@ private Router createRoutesSetup() throws IOException { router.allowForward(AllowForwardHeaders.X_FORWARD); router.route().handler(new RequestCapturingHandler()); + router.route().handler(new ClientVersionCapturingHandler("static/js", "*.js")); router.route().handler(CorsHandler.create() .addRelativeOrigin(".*.") .allowedMethod(io.vertx.core.http.HttpMethod.GET) From 7149e815b59b74d6a19055b67d594a3dcfc420c0 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Tue, 5 Nov 2024 22:35:29 +0000 Subject: [PATCH 115/431] [CI Pipeline] Released Patch version: 5.41.6 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 4dddd15aa..66d95d376 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.41.0 + 5.41.6 UTF-8 From b880e15ac78afe6af84565f9507340c87a144af5 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Wed, 6 Nov 2024 16:51:24 +0000 Subject: [PATCH 116/431] [CI Pipeline] Released Patch version: 5.41.8 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 66d95d376..f92477342 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.41.6 + 5.41.8 UTF-8 From 5c81dc86f2a722fd94d7ac00e53a2ff4af527489 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Wed, 6 Nov 2024 17:42:51 +0000 Subject: [PATCH 117/431] [CI Pipeline] Released Patch version: 5.41.8 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 66d95d376..f92477342 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.41.6 + 5.41.8 UTF-8 From facd5a609f4708a00e2d27e6ff438518f41d3b99 Mon Sep 17 00:00:00 2001 From: Cody Constine Date: Wed, 6 Nov 2024 14:18:15 -0700 Subject: [PATCH 118/431] Added counter for tcf --- conf/local-config.json | 2 +- .../uid2/operator/vertx/UIDOperatorVerticle.java | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/conf/local-config.json b/conf/local-config.json index f19a4357d..f4687b6c1 100644 --- a/conf/local-config.json +++ b/conf/local-config.json @@ -18,7 +18,7 @@ "refresh_token_v3": false, "identity_v3": false, "identity_scope": "uid2", - "enable_v2_encryption": false, + "enable_v2_encryption": true, "sharing_token_expiry_seconds": 2592000, "cloud_download_threads": 8, "cloud_upload_threads": 2, diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index 4f6fd97db..6433268bd 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -99,6 +99,7 @@ public class UIDOperatorVerticle extends AbstractVerticle { private final Map, DistributionSummary> _refreshDurationMetricSummaries = new HashMap<>(); private final Map, Counter> _advertisingTokenExpiryStatus = new HashMap<>(); private final Map, Counter> _tokenGeneratePolicyCounters = new HashMap<>(); + private final Map _tokenGenerateTCFUsage = new HashMap<>(); private final Map> _identityMapUnmappedIdentifiers = new HashMap<>(); private final Map _identityMapRequestWithUnmapped = new HashMap<>(); @@ -962,7 +963,7 @@ private void handleTokenGenerateV2(RoutingContext rc) { if (isTokenInputValid(input, rc)) { final String apiContact = getApiContact(rc); - switch (validateUserConsent(req)) { + switch (validateUserConsent(req, apiContact)) { case INVALID: { SendClientErrorResponseAndRecordStats(ResponseStatus.ClientError, 400, rc, "User consent is invalid", siteId, TokenResponseStatsCollector.Endpoint.GenerateV2, TokenResponseStatsCollector.ResponseStatus.InvalidUserConsentString, siteProvider, platformType); return; @@ -1891,9 +1892,10 @@ private InputUtil.InputVal[] createInputListV1(JsonArray a, IdentityType identit return resp; } - private UserConsentStatus validateUserConsent(JsonObject req) { - // TCF string is an optional parameter and we should only check tcf if in EUID and the string is present + private UserConsentStatus validateUserConsent(JsonObject req, String apiContact) { + // TCF string is an optional parameter, and we should only check tcf if in EUID and the string is present if (identityScope.equals(IdentityScope.EUID) && req.containsKey("tcf_consent_string")) { + recordTokenGenerateTCFUsage(apiContact); TransparentConsentParseResult tcResult = this.getUserConsentV2(req); if (!tcResult.isSuccess()) { return UserConsentStatus.INVALID; @@ -1960,6 +1962,13 @@ private void recordTokenGeneratePolicy(String apiContact, OptoutCheckPolicy poli .register(Metrics.globalRegistry)).increment(); } + private void recordTokenGenerateTCFUsage(String apiContact) { + _tokenGenerateTCFUsage.computeIfAbsent(apiContact, contact -> Counter + .builder("uid2.token_generate_tcf_usage") + .description("Counter for token generate tcf usage") + .tags("api_contact", contact, "policy") + .register(Metrics.globalRegistry)).increment(); + } private TransparentConsentParseResult getUserConsentV2(JsonObject req) { final String rawTcString = req.getString("tcf_consent_string"); From de1569a25e006eab52d652fc15e0545f541b8770 Mon Sep 17 00:00:00 2001 From: Cody Constine Date: Wed, 6 Nov 2024 14:19:18 -0700 Subject: [PATCH 119/431] fixed bad config value: --- conf/local-config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/local-config.json b/conf/local-config.json index f4687b6c1..f19a4357d 100644 --- a/conf/local-config.json +++ b/conf/local-config.json @@ -18,7 +18,7 @@ "refresh_token_v3": false, "identity_v3": false, "identity_scope": "uid2", - "enable_v2_encryption": true, + "enable_v2_encryption": false, "sharing_token_expiry_seconds": 2592000, "cloud_download_threads": 8, "cloud_upload_threads": 2, From d58eb09b7ba5dfeb30a2ecd39a51b2a8bf9bcf1e Mon Sep 17 00:00:00 2001 From: Ian Nara Date: Thu, 7 Nov 2024 10:32:59 -0700 Subject: [PATCH 120/431] find participants on old sdks --- .../vertx/ClientVersionCapturingHandler.java | 63 +++++++++++++++---- .../operator/vertx/UIDOperatorVerticle.java | 2 +- 2 files changed, 51 insertions(+), 14 deletions(-) diff --git a/src/main/java/com/uid2/operator/vertx/ClientVersionCapturingHandler.java b/src/main/java/com/uid2/operator/vertx/ClientVersionCapturingHandler.java index d63626952..1626bb397 100644 --- a/src/main/java/com/uid2/operator/vertx/ClientVersionCapturingHandler.java +++ b/src/main/java/com/uid2/operator/vertx/ClientVersionCapturingHandler.java @@ -1,10 +1,16 @@ package com.uid2.operator.vertx; +import com.uid2.operator.util.Tuple; import com.uid2.shared.Const; +import com.uid2.shared.auth.IAuthorizable; +import com.uid2.shared.auth.IAuthorizableProvider; +import com.uid2.shared.middleware.AuthMiddleware; import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.Metrics; import io.vertx.core.Handler; import io.vertx.ext.web.RoutingContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.nio.file.DirectoryStream; @@ -12,21 +18,23 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.HashMap; +import java.util.HashSet; import java.util.Map; +import java.util.Set; public class ClientVersionCapturingHandler implements Handler { - private final Map _clientVersionCounters = new HashMap<>(); + private static final Logger LOGGER = LoggerFactory.getLogger(ClientVersionCapturingHandler.class); + private static final String BEARER_TOKEN_PREFIX = "bearer "; + private final Map, Counter> _clientVersionCounters = new HashMap<>(); + private IAuthorizableProvider authKeyStore; + private final Set versions = new HashSet<>(); - public ClientVersionCapturingHandler(String dir, String whitelistGlob) throws IOException { + public ClientVersionCapturingHandler(String dir, String whitelistGlob, IAuthorizableProvider authKeyStore) throws IOException { + this.authKeyStore = authKeyStore; try (DirectoryStream dirStream = Files.newDirectoryStream(Paths.get(dir), whitelistGlob)) { dirStream.forEach(path -> { final String version = getFileNameWithoutExtension(path); - final Counter counter = Counter - .builder("uid2.client_sdk_versions") - .description("counter for how many http requests are processed per each client sdk version") - .tags("client_version", version) - .register(Metrics.globalRegistry); - _clientVersionCounters.put(version, counter); + versions.add(version); }); } } @@ -36,11 +44,22 @@ public void handle(RoutingContext context) { if (clientVersion == null) { clientVersion = !context.queryParam("client").isEmpty() ? context.queryParam("client").get(0) : null; } - if (clientVersion != null) { - final Counter counter = _clientVersionCounters.get(clientVersion); - if (counter != null) { - counter.increment(); - } + String apiContact; + try { + final String authHeaderValue = context.request().getHeader("Authorization"); + final String authKey = extractBearerToken(authHeaderValue); + final IAuthorizable profile = this.authKeyStore.get(authKey); + apiContact = profile.getContact(); + apiContact = apiContact == null ? "unknown" : apiContact; + } catch (Exception ex) { + apiContact = "unknown"; + } + if (clientVersion != null && versions.contains(clientVersion)) { + _clientVersionCounters.computeIfAbsent(new Tuple.Tuple2<>(apiContact, clientVersion), tuple -> Counter + .builder("uid2.client_sdk_versions") + .description("counter for how many http requests are processed per each client sdk version") + .tags("api_contact", tuple.getItem1(), "client_version", tuple.getItem2()) + .register(Metrics.globalRegistry)).increment();; } context.next(); } @@ -49,4 +68,22 @@ private static String getFileNameWithoutExtension(Path path) { final String fileName = path.getFileName().toString(); return fileName.indexOf(".") > 0 ? fileName.substring(0, fileName.lastIndexOf(".")) : fileName; } + + private static String extractBearerToken(final String headerValue) { + if (headerValue == null) { + return null; + } + + final String v = headerValue.trim(); + if (v.length() < BEARER_TOKEN_PREFIX.length()) { + return null; + } + + final String givenPrefix = v.substring(0, BEARER_TOKEN_PREFIX.length()); + + if (!BEARER_TOKEN_PREFIX.equals(givenPrefix.toLowerCase())) { + return null; + } + return v.substring(BEARER_TOKEN_PREFIX.length()); + } } \ No newline at end of file diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index 4f6fd97db..8c92da12e 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -221,7 +221,7 @@ private Router createRoutesSetup() throws IOException { router.allowForward(AllowForwardHeaders.X_FORWARD); router.route().handler(new RequestCapturingHandler()); - router.route().handler(new ClientVersionCapturingHandler("static/js", "*.js")); + router.route().handler(new ClientVersionCapturingHandler("static/js", "*.js", clientKeyProvider)); router.route().handler(CorsHandler.create() .addRelativeOrigin(".*.") .allowedMethod(io.vertx.core.http.HttpMethod.GET) From 015091b0296a32bf423f6adccf640971927dc87c Mon Sep 17 00:00:00 2001 From: Ian Nara Date: Thu, 7 Nov 2024 10:42:55 -0700 Subject: [PATCH 121/431] update workflow refs --- .github/workflows/build-and-test.yaml | 2 +- .github/workflows/publish-all-operators.yaml | 2 +- .github/workflows/validate-image.yaml | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-and-test.yaml b/.github/workflows/build-and-test.yaml index 00695f1db..aa13387c6 100644 --- a/.github/workflows/build-and-test.yaml +++ b/.github/workflows/build-and-test.yaml @@ -3,7 +3,7 @@ on: [pull_request, push, workflow_dispatch] jobs: build: - uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-build-and-test.yaml@v2 + uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-build-and-test.yaml@v3 with: java_version: 21 secrets: inherit \ No newline at end of file diff --git a/.github/workflows/publish-all-operators.yaml b/.github/workflows/publish-all-operators.yaml index c5db3a3b0..5e5bf559b 100644 --- a/.github/workflows/publish-all-operators.yaml +++ b/.github/workflows/publish-all-operators.yaml @@ -55,7 +55,7 @@ jobs: fetch-depth: 0 - name: Scan vulnerabilities - uses: IABTechLab/uid2-shared-actions/actions/vulnerability_scan_filesystem@v2 + uses: IABTechLab/uid2-shared-actions/actions/vulnerability_scan_filesystem@v3 with: scan_severity: HIGH,CRITICAL failure_severity: CRITICAL diff --git a/.github/workflows/validate-image.yaml b/.github/workflows/validate-image.yaml index 524f19102..37b4bf912 100644 --- a/.github/workflows/validate-image.yaml +++ b/.github/workflows/validate-image.yaml @@ -19,7 +19,7 @@ on: jobs: build-publish-docker-default: - uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-validate-image.yaml@v2 + uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-validate-image.yaml@v3 with: failure_severity: ${{ inputs.failure_severity || 'CRITICAL,HIGH' }} fail_on_error: ${{ inputs.fail_on_error || true }} @@ -27,7 +27,7 @@ jobs: java_version: 21 secrets: inherit build-publish-docker-aws: - uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-validate-image.yaml@v2 + uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-validate-image.yaml@v3 with: failure_severity: ${{ inputs.failure_severity || 'CRITICAL,HIGH' }} fail_on_error: ${{ inputs.fail_on_error || true }} @@ -36,7 +36,7 @@ jobs: secrets: inherit needs: [build-publish-docker-default] build-publish-docker-gcp: - uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-validate-image.yaml@v2 + uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-validate-image.yaml@v3 with: failure_severity: ${{ inputs.failure_severity || 'CRITICAL,HIGH' }} fail_on_error: ${{ inputs.fail_on_error || true }} @@ -45,7 +45,7 @@ jobs: secrets: inherit needs: [build-publish-docker-aws] build-publish-docker-azure: - uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-validate-image.yaml@v2 + uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-validate-image.yaml@v3 with: failure_severity: ${{ inputs.failure_severity || 'CRITICAL,HIGH' }} fail_on_error: ${{ inputs.fail_on_error || true }} From 13e033ee1bb666a177565f9d897391efb6cca42d Mon Sep 17 00:00:00 2001 From: Cody Constine Date: Thu, 7 Nov 2024 13:24:34 -0700 Subject: [PATCH 122/431] Fixed extra tag --- src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index 6433268bd..cc1639b22 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -1966,7 +1966,7 @@ private void recordTokenGenerateTCFUsage(String apiContact) { _tokenGenerateTCFUsage.computeIfAbsent(apiContact, contact -> Counter .builder("uid2.token_generate_tcf_usage") .description("Counter for token generate tcf usage") - .tags("api_contact", contact, "policy") + .tags("api_contact", contact) .register(Metrics.globalRegistry)).increment(); } From 81260018c5fa6cb693c751a8432e99dab1495831 Mon Sep 17 00:00:00 2001 From: Scott Sundahl Date: Thu, 7 Nov 2024 14:29:45 -0700 Subject: [PATCH 123/431] support phone number --- scripts/aws/conf/prod-euid-config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/aws/conf/prod-euid-config.json b/scripts/aws/conf/prod-euid-config.json index 104f3f4d4..a8befc45f 100644 --- a/scripts/aws/conf/prod-euid-config.json +++ b/scripts/aws/conf/prod-euid-config.json @@ -26,7 +26,7 @@ "identity_scope": "euid", "advertising_token_v3": true, "refresh_token_v3": true, - "enable_phone_support": false, + "enable_phone_support": true, "enable_v1_phone_support": false, "enable_v2_encryption": true } From aae3f7365de11f4428d9cdf9bae9b42ce2464206 Mon Sep 17 00:00:00 2001 From: Ian Nara Date: Thu, 7 Nov 2024 14:55:32 -0700 Subject: [PATCH 124/431] update shared version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f92477342..af18e37d4 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ 2.1.0 2.1.0 2.1.0 - 7.20.0 + 7.20.4 ${project.version} 21 21 From 49761070ad9887995fb58296888831b571f0a225 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Thu, 7 Nov 2024 22:12:11 +0000 Subject: [PATCH 125/431] [CI Pipeline] Released Patch version: 5.41.15 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index af18e37d4..38e26b327 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.41.8 + 5.41.15 UTF-8 From 8b86c2c397a26bbdc110f7c7af2fcf9394e0afb4 Mon Sep 17 00:00:00 2001 From: Andrei Tarassov Date: Mon, 28 Oct 2024 13:14:44 +1100 Subject: [PATCH 126/431] UID2-4429 More efficient disk space utilisation when building operator enclaves - Clean up unneeded files/resources after they no longer needed to reduce disk space usage and reducing the risk of github runner running out of disk space --- Makefile.eif | 4 ++-- scripts/aws/pipeline/amazonlinux2023.Dockerfile | 11 +++++++---- scripts/aws/pipeline/aws_nitro_eif.sh | 1 + 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/Makefile.eif b/Makefile.eif index 9b5b391ac..c50b59b57 100644 --- a/Makefile.eif +++ b/Makefile.eif @@ -14,11 +14,11 @@ all: build_eif build_eif: uid2operator.eif euidoperator.eif uid2operator.eif: build_artifacts build_configs build/proxies.nitro.yaml build/syslog-ng-client.conf build/syslog-ng-core_4.6.0-1_amd64.deb build/syslog-ng-ose-pub.asc build/entrypoint.sh build/vsockpx build/Dockerfile build/load_config.py build/make_config.py - cd build; docker build -t uid2operator . --build-arg JAR_VERSION=`cat package.version` --build-arg IMAGE_VERSION=`cat package.version`-`git show --format="%h" --no-patch`; docker save -o ./uid2operator.tar uid2operator; docker cp ./uid2operator.tar amazonlinux:/uid2operator.tar + cd build; docker build -t uid2operator . --build-arg JAR_VERSION=`cat package.version` --build-arg IMAGE_VERSION=`cat package.version`-`git show --format="%h" --no-patch`; docker save -o ./uid2operator.tar uid2operator; docker cp ./uid2operator.tar amazonlinux:/uid2operator.tar; rm -f ./uid2operator.tar docker exec amazonlinux bash aws_nitro_eif.sh uid2operator euidoperator.eif: build_artifacts build_configs build/proxies.nitro.yaml build/syslog-ng-client.conf build/syslog-ng-core_4.6.0-1_amd64.deb build/syslog-ng-ose-pub.asc build/entrypoint.sh build/vsockpx build/Dockerfile build/load_config.py build/make_config.py - cd build; docker build -t euidoperator . --build-arg IDENTITY_SCOPE='EUID' --build-arg JAR_VERSION=`cat package.version` --build-arg IMAGE_VERSION=`cat package.version`-`git show --format="%h" --no-patch`; docker save -o ./euidoperator.tar euidoperator; docker cp ./euidoperator.tar amazonlinux:/euidoperator.tar + cd build; docker build -t euidoperator . --build-arg IDENTITY_SCOPE='EUID' --build-arg JAR_VERSION=`cat package.version` --build-arg IMAGE_VERSION=`cat package.version`-`git show --format="%h" --no-patch`; docker save -o ./euidoperator.tar euidoperator; docker cp ./euidoperator.tar amazonlinux:/euidoperator.tar; rm -f ./euidoperator.tar docker exec amazonlinux bash aws_nitro_eif.sh euidoperator ################################################################################################################################################################## diff --git a/scripts/aws/pipeline/amazonlinux2023.Dockerfile b/scripts/aws/pipeline/amazonlinux2023.Dockerfile index 2914c9ee3..79bcd66df 100644 --- a/scripts/aws/pipeline/amazonlinux2023.Dockerfile +++ b/scripts/aws/pipeline/amazonlinux2023.Dockerfile @@ -4,8 +4,9 @@ FROM amazonlinux:2023 RUN dnf update -y # systemd is not a hard requirement for Amazon ECS Anywhere, but the installation script currently only supports systemd to run. # Amazon ECS Anywhere can be used without systemd, if you set up your nodes and register them into your ECS cluster **without** the installation script. -RUN dnf -y groupinstall "Development Tools" -RUN dnf -y install systemd vim-common wget git tar libstdc++-static.x86_64 cmake cmake3 aws-nitro-enclaves-cli aws-nitro-enclaves-cli-devel +RUN dnf -y groupinstall "Development Tools" \ + && dnf -y install systemd vim-common wget git tar libstdc++-static.x86_64 cmake cmake3 aws-nitro-enclaves-cli aws-nitro-enclaves-cli-devel \ + && dnf clean all RUN systemctl enable docker @@ -14,12 +15,14 @@ RUN wget https://www.inet.no/dante/files/dante-1.4.3.tar.gz \ && sha256sum --check dante_checksum \ && tar -xf dante-1.4.3.tar.gz \ && cd dante-1.4.3; ./configure; make; cd .. \ - && cp dante-1.4.3/sockd/sockd ./ + && cp dante-1.4.3/sockd/sockd ./ \ + && rm -rf dante-1.4.3 dante-1.4.3.tar.gz RUN git clone https://github.com/IABTechLab/uid2-aws-enclave-vsockproxy.git \ && mkdir uid2-aws-enclave-vsockproxy/build \ && cd uid2-aws-enclave-vsockproxy/build; cmake .. -DCMAKE_BUILD_TYPE=RelWithDebInfo; make; cd ../.. \ - && cp uid2-aws-enclave-vsockproxy/build/vsock-bridge/src/vsock-bridge ./vsockpx + && cp uid2-aws-enclave-vsockproxy/build/vsock-bridge/src/vsock-bridge ./vsockpx \ + && rm -rf uid2-aws-enclave-vsockproxy COPY ./scripts/aws/pipeline/aws_nitro_eif.sh /aws_nitro_eif.sh diff --git a/scripts/aws/pipeline/aws_nitro_eif.sh b/scripts/aws/pipeline/aws_nitro_eif.sh index 2d8f0216b..904d3f3ea 100644 --- a/scripts/aws/pipeline/aws_nitro_eif.sh +++ b/scripts/aws/pipeline/aws_nitro_eif.sh @@ -10,5 +10,6 @@ while (! docker stats --no-stream >/dev/null 2>&1); do sleep 1 done docker load -i $1.tar +rm -f $1.tar nitro-cli build-enclave --docker-uri $1 --output-file $1.eif nitro-cli describe-eif --eif-path $1.eif | jq -r '.Measurements.PCR0' | xxd -r -p | base64 > pcr0.txt From 62421ce7d26371c8ada45d103736129c31a6d496 Mon Sep 17 00:00:00 2001 From: Andrei Tarassov Date: Wed, 30 Oct 2024 10:51:43 +1100 Subject: [PATCH 127/431] UID2-4430 Automatic allocation of cores to vsockpx and operator vertx - Avoid oversubscribing host and enclave - Host: half cores to vsockpx - Enclave: 2/3 cores to operator reactor vertices, 1/4 cores to vsockpx --- scripts/aws/eks-pod/entrypoint.sh | 2 +- scripts/aws/entrypoint.sh | 2 +- scripts/aws/make_config.py | 2 +- scripts/aws/start.sh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/aws/eks-pod/entrypoint.sh b/scripts/aws/eks-pod/entrypoint.sh index adc8a4634..2dc0483e2 100644 --- a/scripts/aws/eks-pod/entrypoint.sh +++ b/scripts/aws/eks-pod/entrypoint.sh @@ -27,7 +27,7 @@ function setup_vsockproxy() { echo "setup_vsockproxy" VSOCK_PROXY=${VSOCK_PROXY:-/home/vsockpx} VSOCK_CONFIG=${VSOCK_CONFIG:-/home/proxies.host.yaml} - VSOCK_THREADS=${VSOCK_THREADS:-$(( $(nproc) * 2 )) } + VSOCK_THREADS=${VSOCK_THREADS:-$(( ( $(nproc) + 1 ) / 2 )) } VSOCK_LOG_LEVEL=${VSOCK_LOG_LEVEL:-3} echo "starting vsock proxy at $VSOCK_PROXY with $VSOCK_THREADS worker threads..." $VSOCK_PROXY -c $VSOCK_CONFIG --workers $VSOCK_THREADS --log-level $VSOCK_LOG_LEVEL --daemon diff --git a/scripts/aws/entrypoint.sh b/scripts/aws/entrypoint.sh index ef622ff80..37214388b 100755 --- a/scripts/aws/entrypoint.sh +++ b/scripts/aws/entrypoint.sh @@ -16,7 +16,7 @@ ifconfig lo 127.0.0.1 # -- start vsock proxy echo "Starting vsock proxy..." -/app/vsockpx --config /app/proxies.nitro.yaml --daemon --workers $(( $(nproc) * 2 )) --log-level 3 +/app/vsockpx --config /app/proxies.nitro.yaml --daemon --workers $(( ( $(nproc) + 3 ) / 4 )) --log-level 3 # -- load config from identity service echo "Loading config from identity service via proxy..." diff --git a/scripts/aws/make_config.py b/scripts/aws/make_config.py index 5777dce61..93439ef28 100644 --- a/scripts/aws/make_config.py +++ b/scripts/aws/make_config.py @@ -26,7 +26,7 @@ def apply_override(config, overrides, key, type): config['optout_api_token'] = overrides['api_token'] # number of threads -config['service_instances'] = thread_count +config['service_instances'] = int((thread_count + 1) * 2 / 3) # environment if overrides.get('environment') == 'integ': diff --git a/scripts/aws/start.sh b/scripts/aws/start.sh index 440ae58d7..429826928 100644 --- a/scripts/aws/start.sh +++ b/scripts/aws/start.sh @@ -81,7 +81,7 @@ function update_allocation() { function setup_vsockproxy() { VSOCK_PROXY=${VSOCK_PROXY:-/usr/bin/vsockpx} VSOCK_CONFIG=${VSOCK_CONFIG:-/etc/uid2operator/proxy.yaml} - VSOCK_THREADS=${VSOCK_THREADS:-$(( $(nproc) * 2 )) } + VSOCK_THREADS=${VSOCK_THREADS:-$(( ( $(nproc) + 1 ) / 2 )) } VSOCK_LOG_LEVEL=${VSOCK_LOG_LEVEL:-3} echo "starting vsock proxy at $VSOCK_PROXY with $VSOCK_THREADS worker threads..." $VSOCK_PROXY -c $VSOCK_CONFIG --workers $VSOCK_THREADS --log-level $VSOCK_LOG_LEVEL --daemon From 3dd9ba5e1246c78e3b1b6add06215328f90927fa Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Fri, 8 Nov 2024 20:11:30 +0000 Subject: [PATCH 128/431] [CI Pipeline] Released Minor version: 5.42.0 --- pom.xml | 2 +- version.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 38e26b327..ac7ec96b4 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.41.15 + 5.42.0 UTF-8 diff --git a/version.json b/version.json index a43cbb357..ad32c1ceb 100644 --- a/version.json +++ b/version.json @@ -1 +1 @@ -{ "$schema": "https://raw.githubusercontent.com/dotnet/Nerdbank.GitVersioning/master/src/NerdBank.GitVersioning/version.schema.json", "version": "5.41", "publicReleaseRefSpec": [ "^refs/heads/master$", "^refs/heads/v\\d+(?:\\.\\d+)?$" ], "cloudBuild": { "setVersionVariables": true, "buildNumber": { "enabled": true, "includeCommitId": { "when": "always" } } } } +{ "$schema": "https://raw.githubusercontent.com/dotnet/Nerdbank.GitVersioning/master/src/NerdBank.GitVersioning/version.schema.json", "version": "5.42", "publicReleaseRefSpec": [ "^refs/heads/master$", "^refs/heads/v\\d+(?:\\.\\d+)?$" ], "cloudBuild": { "setVersionVariables": true, "buildNumber": { "enabled": true, "includeCommitId": { "when": "always" } } } } From 564b274549509a2e9a58b20cadd752e4440f71a5 Mon Sep 17 00:00:00 2001 From: thomasm-ttd <117058351+thomasm-ttd@users.noreply.github.com> Date: Fri, 15 Nov 2024 18:05:45 +1100 Subject: [PATCH 129/431] Ignore CVE-2024-47535 and CVE-2024-7254 (#1153) --- .trivyignore | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.trivyignore b/.trivyignore index 3aa85f54a..f3b81e9c6 100644 --- a/.trivyignore +++ b/.trivyignore @@ -2,4 +2,9 @@ # See https://aquasecurity.github.io/trivy/v0.35/docs/vulnerability/examples/filter/ # for more details # e.g. -# CVE-2022-3996 + +# https://thetradedesk.atlassian.net/browse/UID2-4460 +CVE-2024-47535 + +# https://thetradedesk.atlassian.net/browse/UID2-4461 +CVE-2024-7254 From 75dfd6d3416e8f9d8d5c2e7ea29315b62b9fbb79 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Mon, 18 Nov 2024 01:36:07 +0000 Subject: [PATCH 130/431] [CI Pipeline] Released Patch version: 5.42.3 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ac7ec96b4..6965c297b 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.42.0 + 5.42.3 UTF-8 From b83fa903016dc98e8ede2bf4e9de234ef953febf Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Mon, 18 Nov 2024 01:43:44 +0000 Subject: [PATCH 131/431] [CI Pipeline] Released Patch version: 5.42.4 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 6965c297b..fbce103bd 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.42.3 + 5.42.4 UTF-8 From 0a51642d52cb243d6ecf77df3cdc58b5c31f6f22 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Mon, 18 Nov 2024 02:58:28 +0000 Subject: [PATCH 132/431] [CI Pipeline] Released Patch version: 5.42.5 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index fbce103bd..28eab1fc8 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.42.4 + 5.42.5 UTF-8 From 1385f2feb81fd121508687e068eb24193973d045 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Mon, 18 Nov 2024 03:05:59 +0000 Subject: [PATCH 133/431] [CI Pipeline] Released Patch version: 5.42.6 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 28eab1fc8..a1311d45d 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.42.5 + 5.42.6 UTF-8 From 75a9c29ca63dc23d1a5a54c8d90b0661e0f2f063 Mon Sep 17 00:00:00 2001 From: Cody Constine Date: Mon, 18 Nov 2024 13:37:18 -0700 Subject: [PATCH 134/431] Added site name as contact for CSTG --- src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index f64405588..febbbd931 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -357,6 +357,7 @@ private void handleClientSideTokenGenerateImpl(RoutingContext rc) throws NoSuchA null, TokenResponseStatsCollector.Endpoint.ClientSideTokenGenerateV2, TokenResponseStatsCollector.ResponseStatus.BadSubscriptionId, siteProvider, platformType); return; } + rc.data().put("api-contact", siteProvider.getSite(clientSideKeypair.getSiteId()).getName()); if(clientSideKeypair.isDisabled()) { SendClientErrorResponseAndRecordStats(ResponseStatus.Unauthorized, 401, rc, "Unauthorized", From c20cc48601fca945a2ffde48ab68129e23b8a0ec Mon Sep 17 00:00:00 2001 From: Cody Constine Date: Mon, 18 Nov 2024 14:03:34 -0700 Subject: [PATCH 135/431] Added site ID --- src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index febbbd931..7cba5a866 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -357,7 +357,8 @@ private void handleClientSideTokenGenerateImpl(RoutingContext rc) throws NoSuchA null, TokenResponseStatsCollector.Endpoint.ClientSideTokenGenerateV2, TokenResponseStatsCollector.ResponseStatus.BadSubscriptionId, siteProvider, platformType); return; } - rc.data().put("api-contact", siteProvider.getSite(clientSideKeypair.getSiteId()).getName()); + rc.data().put(AuthMiddleware.API_CONTACT_PROP, siteProvider.getSite(clientSideKeypair.getSiteId()).getName()); + rc.put(com.uid2.shared.Const.RoutingContextData.SiteId, clientSideKeypair.getSiteId()); if(clientSideKeypair.isDisabled()) { SendClientErrorResponseAndRecordStats(ResponseStatus.Unauthorized, 401, rc, "Unauthorized", From b2e30d957015b5ff639dc836ec93dcf88e949d84 Mon Sep 17 00:00:00 2001 From: Cody Constine Date: Tue, 19 Nov 2024 10:13:54 -0700 Subject: [PATCH 136/431] Removing API contact --- src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index 7cba5a866..c3784ae38 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -357,7 +357,6 @@ private void handleClientSideTokenGenerateImpl(RoutingContext rc) throws NoSuchA null, TokenResponseStatsCollector.Endpoint.ClientSideTokenGenerateV2, TokenResponseStatsCollector.ResponseStatus.BadSubscriptionId, siteProvider, platformType); return; } - rc.data().put(AuthMiddleware.API_CONTACT_PROP, siteProvider.getSite(clientSideKeypair.getSiteId()).getName()); rc.put(com.uid2.shared.Const.RoutingContextData.SiteId, clientSideKeypair.getSiteId()); if(clientSideKeypair.isDisabled()) { From 07f4da8a13e7c0ec736c8a9cd40ea1cfd73ec1d9 Mon Sep 17 00:00:00 2001 From: Cody Constine Date: Wed, 20 Nov 2024 12:20:32 -0700 Subject: [PATCH 137/431] Catching up to rename changes --- conf/default-config.json | 2 +- conf/docker-config.json | 2 +- conf/integ-config.json | 2 +- conf/local-config.json | 2 +- conf/local-e2e-docker-private-config.json | 2 +- conf/local-e2e-docker-public-config.json | 2 +- conf/local-e2e-private-config.json | 2 +- conf/local-e2e-public-config.json | 2 +- ...dator-latest-e2e-docker-public-config.json | 2 +- src/main/java/com/uid2/operator/Main.java | 8 ++--- ...otatingCloudEncryptionKeyApiProvider.java} | 23 +++++++------- ...ingCloudEncryptionKeyApiProviderTest.java} | 30 +++++++++---------- 12 files changed, 38 insertions(+), 41 deletions(-) rename src/main/java/com/uid2/operator/reader/{RotatingS3KeyOperatorProvider.java => RotatingCloudEncryptionKeyApiProvider.java} (59%) rename src/test/java/com/uid2/operator/{RotatingS3KeyOperatorProviderTest.java => RotatingCloudEncryptionKeyApiProviderTest.java} (67%) diff --git a/conf/default-config.json b/conf/default-config.json index 70364ed8f..87aaef3e5 100644 --- a/conf/default-config.json +++ b/conf/default-config.json @@ -30,7 +30,7 @@ "salts_metadata_path": "salts/metadata.json", "services_metadata_path": "services/metadata.json", "service_links_metadata_path": "service_links/metadata.json", - "s3_keys_metadata_path": "s3encryption_keys/metadata.json", + "cloud_encryption_keys_metadata_path": "s3encryption_keys/metadata.json", "optout_metadata_path": null, "optout_inmem_cache": false, "enclave_platform": null, diff --git a/conf/docker-config.json b/conf/docker-config.json index 3f59e8263..079432060 100644 --- a/conf/docker-config.json +++ b/conf/docker-config.json @@ -32,7 +32,7 @@ "salts_metadata_path": "/com.uid2.core/test/salts/metadata.json", "services_metadata_path": "/com.uid2.core/test/services/metadata.json", "service_links_metadata_path": "/com.uid2.core/test/service_links/metadata.json", - "s3_keys_metadata_path": "/com.uid2.core/test/s3encryption_keys/metadata.json", + "cloud_encryption_keys_metadata_path": "/com.uid2.core/test/s3encryption_keys/metadata.json", "identity_token_expires_after_seconds": 3600, "optout_metadata_path": null, "optout_inmem_cache": false, diff --git a/conf/integ-config.json b/conf/integ-config.json index d79f97cc0..b9fea6ba1 100644 --- a/conf/integ-config.json +++ b/conf/integ-config.json @@ -14,6 +14,6 @@ "optout_api_token": "test-operator-key", "optout_api_uri": "http://localhost:8081/optout/replicate", "salts_expired_shutdown_hours": 12, - "s3_keys_metadata_path": "http://localhost:8088/s3encryption_keys/retrieve", + "cloud_encryption_keys_metadata_path": "http://localhost:8088/s3encryption_keys/retrieve", "operator_type": "public" } \ No newline at end of file diff --git a/conf/local-config.json b/conf/local-config.json index 5ba76a923..870a40bc0 100644 --- a/conf/local-config.json +++ b/conf/local-config.json @@ -9,7 +9,7 @@ "salts_metadata_path": "/com.uid2.core/test/salts/metadata.json", "services_metadata_path": "/com.uid2.core/test/services/metadata.json", "service_links_metadata_path": "/com.uid2.core/test/service_links/metadata.json", - "s3_keys_metadata_path":"/com.uid2.core/test/s3encryption_keys/metadata.json", + "cloud_encryption_keys_metadata_path":"/com.uid2.core/test/s3encryption_keys/metadata.json", "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, diff --git a/conf/local-e2e-docker-private-config.json b/conf/local-e2e-docker-private-config.json index abb93d899..4ea69a06c 100644 --- a/conf/local-e2e-docker-private-config.json +++ b/conf/local-e2e-docker-private-config.json @@ -11,7 +11,7 @@ "keysets_metadata_path": "http://core:8088/key/keyset/refresh", "keyset_keys_metadata_path": "http://core:8088/key/keyset-keys/refresh", "salts_metadata_path": "http://core:8088/salt/refresh", - "s3_keys_metadata_path": "http://core:8088/s3encryption_keys/retrieve", + "cloud_encryption_keys_metadata_path": "http://core:8088/s3encryption_keys/retrieve", "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, diff --git a/conf/local-e2e-docker-public-config.json b/conf/local-e2e-docker-public-config.json index 0d2389151..36784049e 100644 --- a/conf/local-e2e-docker-public-config.json +++ b/conf/local-e2e-docker-public-config.json @@ -13,7 +13,7 @@ "salts_metadata_path": "http://core:8088/salt/refresh", "services_metadata_path": "http://core:8088/services/refresh", "service_links_metadata_path": "http://core:8088/service_links/refresh", - "s3_keys_metadata_path": "http://core:8088/s3encryption_keys/retrieve", + "cloud_encryption_keys_metadata_path": "http://core:8088/s3encryption_keys/retrieve", "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, diff --git a/conf/local-e2e-private-config.json b/conf/local-e2e-private-config.json index 96a1ac92d..8c8b2b4e8 100644 --- a/conf/local-e2e-private-config.json +++ b/conf/local-e2e-private-config.json @@ -13,7 +13,7 @@ "salts_metadata_path": "http://localhost:8088/salt/refresh", "services_metadata_path": "http://localhost:8088/services/refresh", "service_links_metadata_path": "http://localhost:8088/service_links/refresh", - "s3_keys_metadata_path": "http://core:8088/s3encryption_keys/retrieve", + "cloud_encryption_keys_metadata_path": "http://core:8088/s3encryption_keys/retrieve", "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, diff --git a/conf/local-e2e-public-config.json b/conf/local-e2e-public-config.json index 179b8cb34..56bff716d 100644 --- a/conf/local-e2e-public-config.json +++ b/conf/local-e2e-public-config.json @@ -13,7 +13,7 @@ "salts_metadata_path": "http://localhost:8088/salt/refresh", "services_metadata_path": "http://localhost:8088/services/refresh", "service_links_metadata_path": "http://localhost:8088/service_links/refresh", - "s3_keys_metadata_path": "http://core:8088/s3encryption_keys/retrieve", + "cloud_encryption_keys_metadata_path": "http://core:8088/s3encryption_keys/retrieve", "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, diff --git a/conf/validator-latest-e2e-docker-public-config.json b/conf/validator-latest-e2e-docker-public-config.json index 66db86a25..ebcd22540 100644 --- a/conf/validator-latest-e2e-docker-public-config.json +++ b/conf/validator-latest-e2e-docker-public-config.json @@ -14,7 +14,7 @@ "salts_metadata_path": "http://core:8088/salt/refresh", "services_metadata_path": "http://core:8088/services/refresh", "service_links_metadata_path": "http://core:8088/service_links/refresh", - "s3_keys_metadata_path": "https://core:8088/s3encryption_keys/retrieve", + "cloud_encryption_keys_metadata_path": "https://core:8088/s3encryption_keys/retrieve", "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index 86ac035c6..272dae2aa 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -8,7 +8,7 @@ import com.uid2.operator.monitoring.IStatsCollectorQueue; import com.uid2.operator.monitoring.OperatorMetrics; import com.uid2.operator.monitoring.StatsCollectorVerticle; -import com.uid2.operator.reader.RotatingS3KeyOperatorProvider; +import com.uid2.operator.reader.RotatingCloudEncryptionKeyApiProvider; import com.uid2.operator.service.SecureLinkValidatorService; import com.uid2.operator.service.ShutdownService; import com.uid2.operator.vertx.Endpoints; @@ -82,7 +82,7 @@ public class Main { private IStatsCollectorQueue _statsCollectorQueue; private RotatingServiceStore serviceProvider; private RotatingServiceLinkStore serviceLinkProvider; - private RotatingS3KeyOperatorProvider s3KeyProvider; + private RotatingCloudEncryptionKeyApiProvider s3KeyProvider; public Main(Vertx vertx, JsonObject config) throws Exception { this.vertx = vertx; @@ -134,8 +134,8 @@ public Main(Vertx vertx, JsonObject config) throws Exception { this.fsOptOut = configureCloudOptOutStore(); } - String s3KeyMdPath = this.config.getString(Const.Config.S3keysMetadataPathProp); - this.s3KeyProvider = new RotatingS3KeyOperatorProvider(fsStores, new GlobalScope(new CloudPath(s3KeyMdPath))); + String s3KeyMdPath = this.config.getString(Const.Config.CloudEncryptionKeysMetadataPathProp); + this.s3KeyProvider = new RotatingCloudEncryptionKeyApiProvider(fsStores, new GlobalScope(new CloudPath(s3KeyMdPath))); String sitesMdPath = this.config.getString(Const.Config.SitesMetadataPathProp); String keypairMdPath = this.config.getString(Const.Config.ClientSideKeypairsMetadataPathProp); this.clientSideKeypairProvider = new RotatingClientSideKeypairStore(fsStores, new GlobalScope(new CloudPath(keypairMdPath))); diff --git a/src/main/java/com/uid2/operator/reader/RotatingS3KeyOperatorProvider.java b/src/main/java/com/uid2/operator/reader/RotatingCloudEncryptionKeyApiProvider.java similarity index 59% rename from src/main/java/com/uid2/operator/reader/RotatingS3KeyOperatorProvider.java rename to src/main/java/com/uid2/operator/reader/RotatingCloudEncryptionKeyApiProvider.java index df73b790a..26242229f 100644 --- a/src/main/java/com/uid2/operator/reader/RotatingS3KeyOperatorProvider.java +++ b/src/main/java/com/uid2/operator/reader/RotatingCloudEncryptionKeyApiProvider.java @@ -1,28 +1,25 @@ package com.uid2.operator.reader; -import com.uid2.operator.reader.ApiStoreReader; import com.uid2.shared.cloud.DownloadCloudStorage; -import com.uid2.shared.model.S3Key; +import com.uid2.shared.model.CloudEncryptionKey; import com.uid2.shared.store.CloudPath; -import com.uid2.shared.store.parser.S3KeyParser; -import com.uid2.shared.store.reader.RotatingS3KeyProvider; +import com.uid2.shared.store.parser.CloudEncryptionKeyParser; +import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider; import com.uid2.shared.store.scope.StoreScope; import io.vertx.core.json.JsonObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.time.Instant; import java.util.*; -import java.util.stream.Collectors; -public class RotatingS3KeyOperatorProvider extends RotatingS3KeyProvider { - private static final Logger LOGGER = LoggerFactory.getLogger(RotatingS3KeyOperatorProvider.class); +public class RotatingCloudEncryptionKeyApiProvider extends RotatingCloudEncryptionKeyProvider { + private static final Logger LOGGER = LoggerFactory.getLogger(RotatingCloudEncryptionKeyApiProvider.class); - public ApiStoreReader> apiStoreReader; + public ApiStoreReader> apiStoreReader; - public RotatingS3KeyOperatorProvider(DownloadCloudStorage fileStreamProvider, StoreScope scope) { + public RotatingCloudEncryptionKeyApiProvider(DownloadCloudStorage fileStreamProvider, StoreScope scope) { super(fileStreamProvider, scope); - this.apiStoreReader = new ApiStoreReader<>(fileStreamProvider, scope, new S3KeyParser(), "s3encryption_keys"); + this.apiStoreReader = new ApiStoreReader<>(fileStreamProvider, scope, new CloudEncryptionKeyParser(), "s3encryption_keys"); } @Override @@ -41,8 +38,8 @@ public long loadContent(JsonObject metadata) throws Exception { } @Override - public Map getAll() { - Map keys = apiStoreReader.getSnapshot(); + public Map getAll() { + Map keys = apiStoreReader.getSnapshot(); return keys != null ? keys : new HashMap<>(); } diff --git a/src/test/java/com/uid2/operator/RotatingS3KeyOperatorProviderTest.java b/src/test/java/com/uid2/operator/RotatingCloudEncryptionKeyApiProviderTest.java similarity index 67% rename from src/test/java/com/uid2/operator/RotatingS3KeyOperatorProviderTest.java rename to src/test/java/com/uid2/operator/RotatingCloudEncryptionKeyApiProviderTest.java index ee53d3ab5..3c8776040 100644 --- a/src/test/java/com/uid2/operator/RotatingS3KeyOperatorProviderTest.java +++ b/src/test/java/com/uid2/operator/RotatingCloudEncryptionKeyApiProviderTest.java @@ -1,9 +1,9 @@ package com.uid2.operator; import com.uid2.operator.reader.ApiStoreReader; -import com.uid2.operator.reader.RotatingS3KeyOperatorProvider; +import com.uid2.operator.reader.RotatingCloudEncryptionKeyApiProvider; import com.uid2.shared.cloud.DownloadCloudStorage; -import com.uid2.shared.model.S3Key; +import com.uid2.shared.model.CloudEncryptionKey; import com.uid2.shared.store.CloudPath; import com.uid2.shared.store.scope.StoreScope; import io.vertx.core.json.JsonObject; @@ -18,7 +18,7 @@ import static org.junit.jupiter.api.Assertions.*; import static org.mockito.Mockito.*; -class RotatingS3KeyOperatorProviderTest { +class RotatingCloudEncryptionKeyApiProviderTest { @Mock private DownloadCloudStorage mockFileStreamProvider; @@ -27,15 +27,15 @@ class RotatingS3KeyOperatorProviderTest { private StoreScope mockScope; @Mock - private ApiStoreReader> mockApiStoreReader; + private ApiStoreReader> mockApiStoreReader; - private RotatingS3KeyOperatorProvider rotatingS3KeyOperatorProvider; + private RotatingCloudEncryptionKeyApiProvider rotatingCloudEncryptionKeyApiProvider; @BeforeEach void setUp() { MockitoAnnotations.openMocks(this); - rotatingS3KeyOperatorProvider = new RotatingS3KeyOperatorProvider(mockFileStreamProvider, mockScope); - rotatingS3KeyOperatorProvider.apiStoreReader = mockApiStoreReader; + rotatingCloudEncryptionKeyApiProvider = new RotatingCloudEncryptionKeyApiProvider(mockFileStreamProvider, mockScope); + rotatingCloudEncryptionKeyApiProvider.apiStoreReader = mockApiStoreReader; } @Test @@ -43,7 +43,7 @@ void testGetMetadata() throws Exception { JsonObject expectedMetadata = new JsonObject().put("version", 1L); when(mockApiStoreReader.getMetadata()).thenReturn(expectedMetadata); - JsonObject metadata = rotatingS3KeyOperatorProvider.getMetadata(); + JsonObject metadata = rotatingCloudEncryptionKeyApiProvider.getMetadata(); assertEquals(expectedMetadata, metadata); verify(mockApiStoreReader).getMetadata(); } @@ -53,7 +53,7 @@ void testGetMetadataPath() { CloudPath expectedPath = new CloudPath("test/path"); when(mockApiStoreReader.getMetadataPath()).thenReturn(expectedPath); - CloudPath path = rotatingS3KeyOperatorProvider.getMetadataPath(); + CloudPath path = rotatingCloudEncryptionKeyApiProvider.getMetadataPath(); assertEquals(expectedPath, path); verify(mockApiStoreReader).getMetadataPath(); } @@ -63,19 +63,19 @@ void testLoadContentWithMetadata() throws Exception { JsonObject metadata = new JsonObject(); when(mockApiStoreReader.loadContent(metadata, "s3Keys")).thenReturn(1L); - long version = rotatingS3KeyOperatorProvider.loadContent(metadata); + long version = rotatingCloudEncryptionKeyApiProvider.loadContent(metadata); assertEquals(1L, version); verify(mockApiStoreReader).loadContent(metadata, "s3Keys"); } @Test void testGetAll() { - Map expectedKeys = new HashMap<>(); - S3Key key = new S3Key(1, 123, 1687635529, 1687808329, "secret"); + Map expectedKeys = new HashMap<>(); + CloudEncryptionKey key = new CloudEncryptionKey(1, 123, 1687635529, 1687808329, "secret"); expectedKeys.put(1, key); when(mockApiStoreReader.getSnapshot()).thenReturn(expectedKeys); - Map keys = rotatingS3KeyOperatorProvider.getAll(); + Map keys = rotatingCloudEncryptionKeyApiProvider.getAll(); assertEquals(expectedKeys, keys); verify(mockApiStoreReader).getSnapshot(); } @@ -84,7 +84,7 @@ void testGetAll() { void testGetAllWithNullSnapshot() { when(mockApiStoreReader.getSnapshot()).thenReturn(null); - Map keys = rotatingS3KeyOperatorProvider.getAll(); + Map keys = rotatingCloudEncryptionKeyApiProvider.getAll(); assertNotNull(keys); assertTrue(keys.isEmpty()); verify(mockApiStoreReader).getSnapshot(); @@ -96,7 +96,7 @@ void testLoadContent() throws Exception { when(mockApiStoreReader.getMetadata()).thenReturn(metadata); when(mockApiStoreReader.loadContent(metadata, "s3Keys")).thenReturn(1L); - rotatingS3KeyOperatorProvider.loadContent(); + rotatingCloudEncryptionKeyApiProvider.loadContent(); verify(mockApiStoreReader).getMetadata(); verify(mockApiStoreReader).loadContent(metadata, "s3Keys"); } From eb0fd116c77bd92c182a7624acbb29f9c7ae58c4 Mon Sep 17 00:00:00 2001 From: Cody Constine Date: Fri, 22 Nov 2024 10:30:20 -0700 Subject: [PATCH 138/431] Renaming + adding get version for api key class --- conf/default-config.json | 2 +- conf/docker-config.json | 2 +- conf/integ-config.json | 2 +- conf/local-config.json | 2 +- conf/local-e2e-docker-private-config.json | 2 +- conf/local-e2e-docker-public-config.json | 2 +- conf/local-e2e-private-config.json | 2 +- conf/local-e2e-public-config.json | 2 +- conf/validator-latest-e2e-docker-public-config.json | 2 +- pom.xml | 2 +- src/main/java/com/uid2/operator/Main.java | 2 +- .../reader/RotatingCloudEncryptionKeyApiProvider.java | 8 +++++++- .../cloud_encryption_keys.json} | 0 .../test/cloud_encryption_keys/metadata.json | 7 +++++++ .../com.uid2.core/test/s3encryption_keys/metadata.json | 7 ------- 15 files changed, 25 insertions(+), 19 deletions(-) rename src/main/resources/com.uid2.core/test/{s3encryption_keys/s3encryption_keys.json => cloud_encryption_keys/cloud_encryption_keys.json} (100%) create mode 100644 src/main/resources/com.uid2.core/test/cloud_encryption_keys/metadata.json delete mode 100644 src/main/resources/com.uid2.core/test/s3encryption_keys/metadata.json diff --git a/conf/default-config.json b/conf/default-config.json index 87aaef3e5..372f7ccb2 100644 --- a/conf/default-config.json +++ b/conf/default-config.json @@ -30,7 +30,7 @@ "salts_metadata_path": "salts/metadata.json", "services_metadata_path": "services/metadata.json", "service_links_metadata_path": "service_links/metadata.json", - "cloud_encryption_keys_metadata_path": "s3encryption_keys/metadata.json", + "cloud_encryption_keys_metadata_path": "cloud_encryption_keys/metadata.json", "optout_metadata_path": null, "optout_inmem_cache": false, "enclave_platform": null, diff --git a/conf/docker-config.json b/conf/docker-config.json index 079432060..80f0a2a45 100644 --- a/conf/docker-config.json +++ b/conf/docker-config.json @@ -32,7 +32,7 @@ "salts_metadata_path": "/com.uid2.core/test/salts/metadata.json", "services_metadata_path": "/com.uid2.core/test/services/metadata.json", "service_links_metadata_path": "/com.uid2.core/test/service_links/metadata.json", - "cloud_encryption_keys_metadata_path": "/com.uid2.core/test/s3encryption_keys/metadata.json", + "cloud_encryption_keys_metadata_path": "/com.uid2.core/test/cloud_encryption_keys/metadata.json", "identity_token_expires_after_seconds": 3600, "optout_metadata_path": null, "optout_inmem_cache": false, diff --git a/conf/integ-config.json b/conf/integ-config.json index b9fea6ba1..3efa3fc7a 100644 --- a/conf/integ-config.json +++ b/conf/integ-config.json @@ -14,6 +14,6 @@ "optout_api_token": "test-operator-key", "optout_api_uri": "http://localhost:8081/optout/replicate", "salts_expired_shutdown_hours": 12, - "cloud_encryption_keys_metadata_path": "http://localhost:8088/s3encryption_keys/retrieve", + "cloud_encryption_keys_metadata_path": "http://localhost:8088/cloud_encryption_keys/retrieve", "operator_type": "public" } \ No newline at end of file diff --git a/conf/local-config.json b/conf/local-config.json index 870a40bc0..77551f3e2 100644 --- a/conf/local-config.json +++ b/conf/local-config.json @@ -9,7 +9,7 @@ "salts_metadata_path": "/com.uid2.core/test/salts/metadata.json", "services_metadata_path": "/com.uid2.core/test/services/metadata.json", "service_links_metadata_path": "/com.uid2.core/test/service_links/metadata.json", - "cloud_encryption_keys_metadata_path":"/com.uid2.core/test/s3encryption_keys/metadata.json", + "cloud_encryption_keys_metadata_path":"/com.uid2.core/test/cloud_encryption_keys/metadata.json", "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, diff --git a/conf/local-e2e-docker-private-config.json b/conf/local-e2e-docker-private-config.json index 4ea69a06c..7e1c3201c 100644 --- a/conf/local-e2e-docker-private-config.json +++ b/conf/local-e2e-docker-private-config.json @@ -11,7 +11,7 @@ "keysets_metadata_path": "http://core:8088/key/keyset/refresh", "keyset_keys_metadata_path": "http://core:8088/key/keyset-keys/refresh", "salts_metadata_path": "http://core:8088/salt/refresh", - "cloud_encryption_keys_metadata_path": "http://core:8088/s3encryption_keys/retrieve", + "cloud_encryption_keys_metadata_path": "http://core:8088/cloud_encryption_keys/retrieve", "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, diff --git a/conf/local-e2e-docker-public-config.json b/conf/local-e2e-docker-public-config.json index 36784049e..373d04269 100644 --- a/conf/local-e2e-docker-public-config.json +++ b/conf/local-e2e-docker-public-config.json @@ -13,7 +13,7 @@ "salts_metadata_path": "http://core:8088/salt/refresh", "services_metadata_path": "http://core:8088/services/refresh", "service_links_metadata_path": "http://core:8088/service_links/refresh", - "cloud_encryption_keys_metadata_path": "http://core:8088/s3encryption_keys/retrieve", + "cloud_encryption_keys_metadata_path": "http://core:8088/cloud_encryption_keys/retrieve", "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, diff --git a/conf/local-e2e-private-config.json b/conf/local-e2e-private-config.json index 8c8b2b4e8..e4b7580d8 100644 --- a/conf/local-e2e-private-config.json +++ b/conf/local-e2e-private-config.json @@ -13,7 +13,7 @@ "salts_metadata_path": "http://localhost:8088/salt/refresh", "services_metadata_path": "http://localhost:8088/services/refresh", "service_links_metadata_path": "http://localhost:8088/service_links/refresh", - "cloud_encryption_keys_metadata_path": "http://core:8088/s3encryption_keys/retrieve", + "cloud_encryption_keys_metadata_path": "http://core:8088/cloud_encryption_keys/retrieve", "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, diff --git a/conf/local-e2e-public-config.json b/conf/local-e2e-public-config.json index 56bff716d..2f5bc4ffe 100644 --- a/conf/local-e2e-public-config.json +++ b/conf/local-e2e-public-config.json @@ -13,7 +13,7 @@ "salts_metadata_path": "http://localhost:8088/salt/refresh", "services_metadata_path": "http://localhost:8088/services/refresh", "service_links_metadata_path": "http://localhost:8088/service_links/refresh", - "cloud_encryption_keys_metadata_path": "http://core:8088/s3encryption_keys/retrieve", + "cloud_encryption_keys_metadata_path": "http://core:8088/cloud_encryption_keys/retrieve", "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, diff --git a/conf/validator-latest-e2e-docker-public-config.json b/conf/validator-latest-e2e-docker-public-config.json index ebcd22540..05bcf0db1 100644 --- a/conf/validator-latest-e2e-docker-public-config.json +++ b/conf/validator-latest-e2e-docker-public-config.json @@ -14,7 +14,7 @@ "salts_metadata_path": "http://core:8088/salt/refresh", "services_metadata_path": "http://core:8088/services/refresh", "service_links_metadata_path": "http://core:8088/service_links/refresh", - "cloud_encryption_keys_metadata_path": "https://core:8088/s3encryption_keys/retrieve", + "cloud_encryption_keys_metadata_path": "https://core:8088/cloud_encryption_keys/retrieve", "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, diff --git a/pom.xml b/pom.xml index 6590ce33c..cc5d76f08 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ 2.1.0 2.1.0 2.1.0 - 7.21.12-alpha-165-SNAPSHOT + 8.0.0 ${project.version} 21 21 diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index 272dae2aa..d7e8d3f9a 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -335,7 +335,7 @@ private Future createStoreVerticles() throws Exception { fs.add(createAndDeployRotatingStoreVerticle("auth", clientKeyProvider, "auth_refresh_ms")); fs.add(createAndDeployRotatingStoreVerticle("keyset", keysetProvider, "keyset_refresh_ms")); fs.add(createAndDeployRotatingStoreVerticle("keysetkey", keysetKeyStore, "keysetkey_refresh_ms")); - fs.add(createAndDeployRotatingStoreVerticle("s3encryption_keys", s3KeyProvider, "s3keys_refresh_ms")); + fs.add(createAndDeployRotatingStoreVerticle("cloud_encryption_keys", s3KeyProvider, "s3keys_refresh_ms")); fs.add(createAndDeployRotatingStoreVerticle("salt", saltProvider, "salt_refresh_ms")); fs.add(createAndDeployCloudSyncStoreVerticle("optout", fsOptOut, optOutCloudSync)); CompositeFuture.all(fs).onComplete(ar -> { diff --git a/src/main/java/com/uid2/operator/reader/RotatingCloudEncryptionKeyApiProvider.java b/src/main/java/com/uid2/operator/reader/RotatingCloudEncryptionKeyApiProvider.java index 26242229f..aa2a3fad6 100644 --- a/src/main/java/com/uid2/operator/reader/RotatingCloudEncryptionKeyApiProvider.java +++ b/src/main/java/com/uid2/operator/reader/RotatingCloudEncryptionKeyApiProvider.java @@ -10,6 +10,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.time.Instant; import java.util.*; public class RotatingCloudEncryptionKeyApiProvider extends RotatingCloudEncryptionKeyProvider { @@ -19,7 +20,7 @@ public class RotatingCloudEncryptionKeyApiProvider extends RotatingCloudEncrypti public RotatingCloudEncryptionKeyApiProvider(DownloadCloudStorage fileStreamProvider, StoreScope scope) { super(fileStreamProvider, scope); - this.apiStoreReader = new ApiStoreReader<>(fileStreamProvider, scope, new CloudEncryptionKeyParser(), "s3encryption_keys"); + this.apiStoreReader = new ApiStoreReader<>(fileStreamProvider, scope, new CloudEncryptionKeyParser(), "cloud_encryption_keys"); } @Override @@ -37,6 +38,11 @@ public long loadContent(JsonObject metadata) throws Exception { return apiStoreReader.loadContent(metadata, "s3Keys"); } + @Override + public long getVersion(JsonObject metadata) { + return Instant.now().getEpochSecond(); + } + @Override public Map getAll() { Map keys = apiStoreReader.getSnapshot(); diff --git a/src/main/resources/com.uid2.core/test/s3encryption_keys/s3encryption_keys.json b/src/main/resources/com.uid2.core/test/cloud_encryption_keys/cloud_encryption_keys.json similarity index 100% rename from src/main/resources/com.uid2.core/test/s3encryption_keys/s3encryption_keys.json rename to src/main/resources/com.uid2.core/test/cloud_encryption_keys/cloud_encryption_keys.json diff --git a/src/main/resources/com.uid2.core/test/cloud_encryption_keys/metadata.json b/src/main/resources/com.uid2.core/test/cloud_encryption_keys/metadata.json new file mode 100644 index 000000000..af9de38c2 --- /dev/null +++ b/src/main/resources/com.uid2.core/test/cloud_encryption_keys/metadata.json @@ -0,0 +1,7 @@ +{ + "version": 1, + "generated": 1620253519, + "cloud_encryption_keys": { + "location": "/com.uid2.core/test/cloud_encryption_keys/cloud_encryption_keys.json" + } +} \ No newline at end of file diff --git a/src/main/resources/com.uid2.core/test/s3encryption_keys/metadata.json b/src/main/resources/com.uid2.core/test/s3encryption_keys/metadata.json deleted file mode 100644 index 4a667ec4c..000000000 --- a/src/main/resources/com.uid2.core/test/s3encryption_keys/metadata.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "version": 1, - "generated": 1620253519, - "s3encryption_keys": { - "location": "/com.uid2.core/test/s3encryption_keys/s3encryption_keys.json" - } -} \ No newline at end of file From 4359b3dfeb672f2c9e66dc4e597180e6decef820 Mon Sep 17 00:00:00 2001 From: Cody Constine Date: Fri, 22 Nov 2024 10:44:23 -0700 Subject: [PATCH 139/431] Upgrading vertx version --- .trivyignore | 3 --- pom.xml | 4 ++-- src/main/java/com/uid2/operator/Main.java | 2 +- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/.trivyignore b/.trivyignore index f3b81e9c6..be4a48a9d 100644 --- a/.trivyignore +++ b/.trivyignore @@ -5,6 +5,3 @@ # https://thetradedesk.atlassian.net/browse/UID2-4460 CVE-2024-47535 - -# https://thetradedesk.atlassian.net/browse/UID2-4461 -CVE-2024-7254 diff --git a/pom.xml b/pom.xml index a1311d45d..99584430b 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ UTF-8 - 4.5.3 + 4.5.11 1.0.22 5.11.2 5.11.2 @@ -22,7 +22,7 @@ 2.1.0 2.1.0 2.1.0 - 7.20.4 + 8.0.1-alpha-167-SNAPSHOT ${project.version} 21 21 diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index 6aa069604..6b88d715b 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -414,7 +414,7 @@ private static Vertx createVertx() { } private static void setupMetrics(MicrometerMetricsOptions metricOptions) { - BackendRegistries.setupBackend(metricOptions); + BackendRegistries.setupBackend(metricOptions, null); MeterRegistry backendRegistry = BackendRegistries.getDefaultNow(); if (backendRegistry instanceof PrometheusMeterRegistry) { From b23dcdc835a0fbdda462c6c8f1d6535bcb71bf31 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Fri, 22 Nov 2024 17:47:44 +0000 Subject: [PATCH 140/431] [CI Pipeline] Released Snapshot version: 5.42.7-alpha-139-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 99584430b..d64fb772a 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.42.6 + 5.42.7-alpha-139-SNAPSHOT UTF-8 From 3768f821402215f6497da1e28984538af8157a7e Mon Sep 17 00:00:00 2001 From: Cody Constine Date: Fri, 22 Nov 2024 11:06:42 -0700 Subject: [PATCH 141/431] Adding new shared version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 99584430b..f81efef11 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ 2.1.0 2.1.0 2.1.0 - 8.0.1-alpha-167-SNAPSHOT + 8.0.6 ${project.version} 21 21 From aad45925407a5683163b6c3c2901d75a5fb59334 Mon Sep 17 00:00:00 2001 From: Cody Constine Date: Mon, 25 Nov 2024 16:02:27 -0700 Subject: [PATCH 142/431] Renaming and adding small fixes --- pom.xml | 2 +- src/main/java/com/uid2/operator/Main.java | 20 ++++++++++--------- .../uid2/operator/reader/ApiStoreReader.java | 6 +++--- ...RotatingCloudEncryptionKeyApiProvider.java | 2 +- ...tingCloudEncryptionKeyApiProviderTest.java | 8 ++++---- 5 files changed, 20 insertions(+), 18 deletions(-) diff --git a/pom.xml b/pom.xml index cc5d76f08..b883ef9fa 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.42.6 + 6.0.0 UTF-8 diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index d7e8d3f9a..fb69d9c9d 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -82,7 +82,7 @@ public class Main { private IStatsCollectorQueue _statsCollectorQueue; private RotatingServiceStore serviceProvider; private RotatingServiceLinkStore serviceLinkProvider; - private RotatingCloudEncryptionKeyApiProvider s3KeyProvider; + private RotatingCloudEncryptionKeyApiProvider cloudEncryptionKeyProvider; public Main(Vertx vertx, JsonObject config) throws Exception { this.vertx = vertx; @@ -134,17 +134,17 @@ public Main(Vertx vertx, JsonObject config) throws Exception { this.fsOptOut = configureCloudOptOutStore(); } - String s3KeyMdPath = this.config.getString(Const.Config.CloudEncryptionKeysMetadataPathProp); - this.s3KeyProvider = new RotatingCloudEncryptionKeyApiProvider(fsStores, new GlobalScope(new CloudPath(s3KeyMdPath))); + String cloudEncryptionKeyMdPath = this.config.getString(Const.Config.CloudEncryptionKeysMetadataPathProp); + this.cloudEncryptionKeyProvider = new RotatingCloudEncryptionKeyApiProvider(fsStores, new GlobalScope(new CloudPath(cloudEncryptionKeyMdPath))); String sitesMdPath = this.config.getString(Const.Config.SitesMetadataPathProp); String keypairMdPath = this.config.getString(Const.Config.ClientSideKeypairsMetadataPathProp); this.clientSideKeypairProvider = new RotatingClientSideKeypairStore(fsStores, new GlobalScope(new CloudPath(keypairMdPath))); String clientsMdPath = this.config.getString(Const.Config.ClientsMetadataPathProp); - this.clientKeyProvider = new RotatingClientKeyProvider(fsStores, new GlobalScope(new CloudPath(clientsMdPath)),s3KeyProvider); + this.clientKeyProvider = new RotatingClientKeyProvider(fsStores, new GlobalScope(new CloudPath(clientsMdPath)), cloudEncryptionKeyProvider); String keysetKeysMdPath = this.config.getString(Const.Config.KeysetKeysMetadataPathProp); - this.keysetKeyStore = new RotatingKeysetKeyStore(fsStores, new GlobalScope(new CloudPath(keysetKeysMdPath)),s3KeyProvider); + this.keysetKeyStore = new RotatingKeysetKeyStore(fsStores, new GlobalScope(new CloudPath(keysetKeysMdPath)), cloudEncryptionKeyProvider); String keysetMdPath = this.config.getString(Const.Config.KeysetsMetadataPathProp); - this.keysetProvider = new RotatingKeysetProvider(fsStores, new GlobalScope(new CloudPath(keysetMdPath)),s3KeyProvider); + this.keysetProvider = new RotatingKeysetProvider(fsStores, new GlobalScope(new CloudPath(keysetMdPath)), cloudEncryptionKeyProvider); String saltsMdPath = this.config.getString(Const.Config.SaltsMetadataPathProp); this.saltProvider = new RotatingSaltProvider(fsStores, saltsMdPath); this.optOutStore = new CloudSyncOptOutStore(vertx, fsLocal, this.config, operatorKey, Clock.systemUTC()); @@ -156,7 +156,7 @@ public Main(Vertx vertx, JsonObject config) throws Exception { this.serviceLinkProvider = new RotatingServiceLinkStore(fsStores, new GlobalScope(new CloudPath(serviceLinkMdPath))); } - this.siteProvider = clientSideTokenGenerate ? new RotatingSiteStore(fsStores, new GlobalScope(new CloudPath(sitesMdPath)), s3KeyProvider) : null; + this.siteProvider = clientSideTokenGenerate ? new RotatingSiteStore(fsStores, new GlobalScope(new CloudPath(sitesMdPath)), cloudEncryptionKeyProvider) : null; if (useStorageMock && coreAttestUrl == null) { if (clientSideTokenGenerate) { @@ -167,7 +167,7 @@ public Main(Vertx vertx, JsonObject config) throws Exception { this.saltProvider.loadContent(); this.keysetProvider.loadContent(); this.keysetKeyStore.loadContent(); - this.s3KeyProvider.loadContent(); + this.cloudEncryptionKeyProvider.loadContent(); if (this.validateServiceLinks) { this.serviceProvider.loadContent(); @@ -307,6 +307,8 @@ private void run() throws Exception { private Future createStoreVerticles() throws Exception { // load metadatas for the first time + cloudEncryptionKeyProvider.loadContent(); + if (clientSideTokenGenerate) { siteProvider.getMetadata(); clientSideKeypairProvider.getMetadata(); @@ -335,7 +337,7 @@ private Future createStoreVerticles() throws Exception { fs.add(createAndDeployRotatingStoreVerticle("auth", clientKeyProvider, "auth_refresh_ms")); fs.add(createAndDeployRotatingStoreVerticle("keyset", keysetProvider, "keyset_refresh_ms")); fs.add(createAndDeployRotatingStoreVerticle("keysetkey", keysetKeyStore, "keysetkey_refresh_ms")); - fs.add(createAndDeployRotatingStoreVerticle("cloud_encryption_keys", s3KeyProvider, "s3keys_refresh_ms")); + fs.add(createAndDeployRotatingStoreVerticle("cloud_encryption_keys", cloudEncryptionKeyProvider, "cloud_encryption_keys_refresh_ms")); fs.add(createAndDeployRotatingStoreVerticle("salt", saltProvider, "salt_refresh_ms")); fs.add(createAndDeployCloudSyncStoreVerticle("optout", fsOptOut, optOutCloudSync)); CompositeFuture.all(fs).onComplete(ar -> { diff --git a/src/main/java/com/uid2/operator/reader/ApiStoreReader.java b/src/main/java/com/uid2/operator/reader/ApiStoreReader.java index f34da3513..d3d75c5d5 100644 --- a/src/main/java/com/uid2/operator/reader/ApiStoreReader.java +++ b/src/main/java/com/uid2/operator/reader/ApiStoreReader.java @@ -28,12 +28,12 @@ public long loadContent(JsonObject contents, String dataType) throws Exception { } try { - JsonArray s3KeysArray = contents.getJsonArray(dataType); - if (s3KeysArray == null) { + JsonArray dataArray = contents.getJsonArray(dataType); + if (dataArray == null) { throw new IllegalArgumentException("No array found in the contents"); } - String jsonString = s3KeysArray.toString(); + String jsonString = dataArray.toString(); InputStream inputStream = new ByteArrayInputStream(jsonString.getBytes(StandardCharsets.UTF_8)); ParsingResult parsed = parser.deserialize(inputStream); diff --git a/src/main/java/com/uid2/operator/reader/RotatingCloudEncryptionKeyApiProvider.java b/src/main/java/com/uid2/operator/reader/RotatingCloudEncryptionKeyApiProvider.java index aa2a3fad6..594cfb7b6 100644 --- a/src/main/java/com/uid2/operator/reader/RotatingCloudEncryptionKeyApiProvider.java +++ b/src/main/java/com/uid2/operator/reader/RotatingCloudEncryptionKeyApiProvider.java @@ -35,7 +35,7 @@ public CloudPath getMetadataPath() { @Override public long loadContent(JsonObject metadata) throws Exception { - return apiStoreReader.loadContent(metadata, "s3Keys"); + return apiStoreReader.loadContent(metadata, "cloudEncryptionKeys"); } @Override diff --git a/src/test/java/com/uid2/operator/RotatingCloudEncryptionKeyApiProviderTest.java b/src/test/java/com/uid2/operator/RotatingCloudEncryptionKeyApiProviderTest.java index 3c8776040..e07489be5 100644 --- a/src/test/java/com/uid2/operator/RotatingCloudEncryptionKeyApiProviderTest.java +++ b/src/test/java/com/uid2/operator/RotatingCloudEncryptionKeyApiProviderTest.java @@ -61,11 +61,11 @@ void testGetMetadataPath() { @Test void testLoadContentWithMetadata() throws Exception { JsonObject metadata = new JsonObject(); - when(mockApiStoreReader.loadContent(metadata, "s3Keys")).thenReturn(1L); + when(mockApiStoreReader.loadContent(metadata, "cloudEncryptionKeys")).thenReturn(1L); long version = rotatingCloudEncryptionKeyApiProvider.loadContent(metadata); assertEquals(1L, version); - verify(mockApiStoreReader).loadContent(metadata, "s3Keys"); + verify(mockApiStoreReader).loadContent(metadata, "cloudEncryptionKeys"); } @Test @@ -94,10 +94,10 @@ void testGetAllWithNullSnapshot() { void testLoadContent() throws Exception { JsonObject metadata = new JsonObject().put("version", 1L); when(mockApiStoreReader.getMetadata()).thenReturn(metadata); - when(mockApiStoreReader.loadContent(metadata, "s3Keys")).thenReturn(1L); + when(mockApiStoreReader.loadContent(metadata, "cloudEncryptionKeys")).thenReturn(1L); rotatingCloudEncryptionKeyApiProvider.loadContent(); verify(mockApiStoreReader).getMetadata(); - verify(mockApiStoreReader).loadContent(metadata, "s3Keys"); + verify(mockApiStoreReader).loadContent(metadata, "cloudEncryptionKeys"); } } \ No newline at end of file From 39540466f3fc87980ac211e1bc9ef0de547243c9 Mon Sep 17 00:00:00 2001 From: Katherine Chen Date: Tue, 26 Nov 2024 14:24:24 +1100 Subject: [PATCH 143/431] Remove siteIdsUsingV4Tokens (#1178) * Remove siteIdsUsingV4Tokens * Remove site_ids_using_v4_tokens * Remove v4 variable * Remove test case which generate v2 token * Set advertising_token_v4_percentage to be 100 * Remove siteId param --- conf/local-config.json | 1 - scripts/aws/conf/default-config.json | 1 - scripts/azure-cc/conf/default-config.json | 1 - scripts/gcp-oidc/conf/default-config.json | 1 - .../com/uid2/operator/service/TokenUtils.java | 17 ------- .../operator/service/UIDOperatorService.java | 21 +++------ .../uid2/operator/UIDOperatorServiceTest.java | 12 ++--- .../operator/UIDOperatorVerticleTest.java | 1 - .../uid2/operator/service/TokenUtilsTest.java | 46 ------------------- 9 files changed, 11 insertions(+), 90 deletions(-) delete mode 100644 src/test/java/com/uid2/operator/service/TokenUtilsTest.java diff --git a/conf/local-config.json b/conf/local-config.json index f19a4357d..7a3200f12 100644 --- a/conf/local-config.json +++ b/conf/local-config.json @@ -14,7 +14,6 @@ "refresh_identity_token_after_seconds": 900, "advertising_token_v3": false, "advertising_token_v4_percentage": 0, - "site_ids_using_v4_tokens": "", "refresh_token_v3": false, "identity_v3": false, "identity_scope": "uid2", diff --git a/scripts/aws/conf/default-config.json b/scripts/aws/conf/default-config.json index 6db89fd29..1836f1aa7 100644 --- a/scripts/aws/conf/default-config.json +++ b/scripts/aws/conf/default-config.json @@ -35,6 +35,5 @@ "sharing_token_expiry_seconds": 2592000, "validate_service_links": false, "advertising_token_v4_percentage": 100, - "site_ids_using_v4_tokens": "", "operator_type": "private" } diff --git a/scripts/azure-cc/conf/default-config.json b/scripts/azure-cc/conf/default-config.json index fbe3e7184..c47eef8f6 100644 --- a/scripts/azure-cc/conf/default-config.json +++ b/scripts/azure-cc/conf/default-config.json @@ -39,6 +39,5 @@ "sharing_token_expiry_seconds": 2592000, "validate_service_links": false, "advertising_token_v4_percentage": 100, - "site_ids_using_v4_tokens": "", "operator_type": "private" } diff --git a/scripts/gcp-oidc/conf/default-config.json b/scripts/gcp-oidc/conf/default-config.json index 302a8c3c3..aaea43364 100644 --- a/scripts/gcp-oidc/conf/default-config.json +++ b/scripts/gcp-oidc/conf/default-config.json @@ -39,6 +39,5 @@ "sharing_token_expiry_seconds": 2592000, "validate_service_links": false, "advertising_token_v4_percentage": 100, - "site_ids_using_v4_tokens": "", "operator_type": "private" } diff --git a/src/main/java/com/uid2/operator/service/TokenUtils.java b/src/main/java/com/uid2/operator/service/TokenUtils.java index 4a13145c6..2cabc641b 100644 --- a/src/main/java/com/uid2/operator/service/TokenUtils.java +++ b/src/main/java/com/uid2/operator/service/TokenUtils.java @@ -62,21 +62,4 @@ public static byte encodeIdentityScope(IdentityScope identityScope) { public static byte encodeIdentityType(IdentityType identityType) { return (byte) (identityType.value << 2); } - - public static Set getSiteIdsUsingV4Tokens(String siteIdsUsingV4TokensInString) { - String[] siteIdsV4TokensList = siteIdsUsingV4TokensInString.split(","); - - Set siteIdsV4TokensSet = new HashSet<>(); - try { - for (String siteId : siteIdsV4TokensList) { - String siteIdTrimmed = siteId.trim(); - if (!siteIdTrimmed.isEmpty()) { - siteIdsV4TokensSet.add(Integer.parseInt(siteIdTrimmed)); - } - } - } catch (NumberFormatException ex) { - throw new IllegalArgumentException(String.format("Invalid integer format found in site_ids_using_v4_tokens: %s", siteIdsUsingV4TokensInString)); - } - return siteIdsV4TokensSet; - } } diff --git a/src/main/java/com/uid2/operator/service/UIDOperatorService.java b/src/main/java/com/uid2/operator/service/UIDOperatorService.java index 672cec238..7e6450829 100644 --- a/src/main/java/com/uid2/operator/service/UIDOperatorService.java +++ b/src/main/java/com/uid2/operator/service/UIDOperatorService.java @@ -22,7 +22,6 @@ import java.util.*; import static com.uid2.operator.IdentityConst.*; -import static com.uid2.operator.service.TokenUtils.getSiteIdsUsingV4Tokens; public class UIDOperatorService implements IUIDOperatorService { public static final String IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS = "identity_token_expires_after_seconds"; @@ -49,7 +48,6 @@ public class UIDOperatorService implements IUIDOperatorService { private final OperatorIdentity operatorIdentity; protected final TokenVersion tokenVersionToUseIfNotV4; protected final int advertisingTokenV4Percentage; - protected final Set siteIdsUsingV4Tokens; private final TokenVersion refreshTokenVersion; private final boolean identityV3Enabled; @@ -94,7 +92,6 @@ public UIDOperatorService(JsonObject config, IOptOutStore optOutStore, ISaltProv } this.advertisingTokenV4Percentage = config.getInteger("advertising_token_v4_percentage", 0); //0 indicates token v4 will not be used - this.siteIdsUsingV4Tokens = getSiteIdsUsingV4Tokens(config.getString("site_ids_using_v4_tokens", "")); this.tokenVersionToUseIfNotV4 = config.getBoolean("advertising_token_v3", false) ? TokenVersion.V3 : TokenVersion.V2; this.refreshTokenVersion = TokenVersion.V3; @@ -271,18 +268,14 @@ private RefreshToken createRefreshToken(PublisherIdentity publisherIdentity, Use private AdvertisingToken createAdvertisingToken(PublisherIdentity publisherIdentity, UserIdentity userIdentity, Instant now) { TokenVersion tokenVersion; - if (siteIdsUsingV4Tokens.contains(publisherIdentity.siteId)) { - tokenVersion = TokenVersion.V4; - } else { - int pseudoRandomNumber = 1; - final var rawUid = userIdentity.id; - if (rawUid.length > 2) - { - int hash = ((rawUid[0] & 0xFF) << 12) | ((rawUid[1] & 0xFF) << 4) | ((rawUid[2] & 0xFF) & 0xF); //using same logic as ModBasedSaltEntryIndexer.getIndex() in uid2-shared - pseudoRandomNumber = (hash % 100) + 1; //1 to 100 - } - tokenVersion = (pseudoRandomNumber <= this.advertisingTokenV4Percentage) ? TokenVersion.V4 : this.tokenVersionToUseIfNotV4; + int pseudoRandomNumber = 1; + final var rawUid = userIdentity.id; + if (rawUid.length > 2) + { + int hash = ((rawUid[0] & 0xFF) << 12) | ((rawUid[1] & 0xFF) << 4) | ((rawUid[2] & 0xFF) & 0xF); //using same logic as ModBasedSaltEntryIndexer.getIndex() in uid2-shared + pseudoRandomNumber = (hash % 100) + 1; //1 to 100 } + tokenVersion = (pseudoRandomNumber <= this.advertisingTokenV4Percentage) ? TokenVersion.V4 : this.tokenVersionToUseIfNotV4; return new AdvertisingToken(tokenVersion, now, now.plusMillis(identityExpiresAfter.toMillis()), this.operatorIdentity, publisherIdentity, userIdentity); } diff --git a/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java b/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java index bc1d49bb2..4a8aef9d1 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java @@ -58,11 +58,8 @@ public ExtendedUIDOperatorService(JsonObject config, IOptOutStore optOutStore, I super(config, optOutStore, saltProvider, encoder, clock, identityScope, saltRetrievalResponseHandler); } - public TokenVersion getAdvertisingTokenVersionForTests(int siteId) { + public TokenVersion getAdvertisingTokenVersionForTests() { assert this.advertisingTokenV4Percentage == 0 || this.advertisingTokenV4Percentage == 100; //we want tests to be deterministic - if (this.siteIdsUsingV4Tokens.contains(siteId)) { - return TokenVersion.V4; - } return this.advertisingTokenV4Percentage == 100 ? TokenVersion.V4 : this.tokenVersionToUseIfNotV4; } } @@ -96,8 +93,7 @@ void setup() throws Exception { uid2Config.put(UIDOperatorService.IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS); uid2Config.put(UIDOperatorService.REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, REFRESH_TOKEN_EXPIRES_AFTER_SECONDS); uid2Config.put(UIDOperatorService.REFRESH_IDENTITY_TOKEN_AFTER_SECONDS, REFRESH_IDENTITY_TOKEN_AFTER_SECONDS); - uid2Config.put("advertising_token_v4_percentage", 0); - uid2Config.put("site_ids_using_v4_tokens", "127,128"); + uid2Config.put("advertising_token_v4_percentage", 100); uid2Config.put("advertising_token_v3", false); // prod is using v2 token version for now uid2Config.put("identity_v3", false); @@ -152,7 +148,7 @@ private UserIdentity createUserIdentity(String rawIdentityHash, IdentityScope sc } private AdvertisingToken validateAndGetToken(EncryptedTokenEncoder tokenEncoder, String advertisingTokenString, IdentityScope scope, IdentityType type, int siteId) { - TokenVersion tokenVersion = (scope == IdentityScope.UID2) ? uid2Service.getAdvertisingTokenVersionForTests(siteId) : euidService.getAdvertisingTokenVersionForTests(siteId); + TokenVersion tokenVersion = (scope == IdentityScope.UID2) ? uid2Service.getAdvertisingTokenVersionForTests() : euidService.getAdvertisingTokenVersionForTests(); UIDOperatorVerticleTest.validateAdvertisingToken(advertisingTokenString, tokenVersion, scope, type); return tokenEncoder.decodeAdvertisingToken(advertisingTokenString); } @@ -164,7 +160,7 @@ private void assertIdentityScopeIdentityTypeAndEstablishedAt(UserIdentity expcte } @ParameterizedTest - @CsvSource({"123, V2","127, V4","128, V4"}) //site id 127 and 128 is for testing "site_ids_using_v4_tokens" + @CsvSource({"123, V4","127, V4","128, V4"}) public void testGenerateAndRefresh(int siteId, TokenVersion tokenVersion) { final IdentityRequest identityRequest = new IdentityRequest( new PublisherIdentity(siteId, 124, 125), diff --git a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java index eafa14f9a..6ef9ba989 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java @@ -160,7 +160,6 @@ private void setupConfig(JsonObject config) { config.put("identity_scope", getIdentityScope().toString()); config.put("advertising_token_v3", getTokenVersion() == TokenVersion.V3); config.put("advertising_token_v4_percentage", getTokenVersion() == TokenVersion.V4 ? 100 : 0); - config.put("site_ids_using_v4_tokens", ""); config.put("identity_v3", useIdentityV3()); config.put("client_side_token_generate", true); config.put("key_sharing_endpoint_provide_app_names", true); diff --git a/src/test/java/com/uid2/operator/service/TokenUtilsTest.java b/src/test/java/com/uid2/operator/service/TokenUtilsTest.java deleted file mode 100644 index 2fb7af1fd..000000000 --- a/src/test/java/com/uid2/operator/service/TokenUtilsTest.java +++ /dev/null @@ -1,46 +0,0 @@ -package com.uid2.operator.service; - -import com.uid2.shared.cloud.CloudStorageException; -import org.junit.jupiter.api.Test; - -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import static com.uid2.operator.service.TokenUtils.getSiteIdsUsingV4Tokens; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; - -public class TokenUtilsTest { - Set siteIdsV4TokensSet = new HashSet<>(Arrays.asList(127, 128)); - @Test - void getSiteIdsUsingV4Tokens_multipleSiteIds() { - Set actualSiteIdsV4TokensSet = getSiteIdsUsingV4Tokens("127, 128"); - assertEquals(siteIdsV4TokensSet, actualSiteIdsV4TokensSet); - } - - @Test - void getSiteIdsUsingV4Tokens_oneSiteIds() { - Set actualSiteIdsV4TokensSet = getSiteIdsUsingV4Tokens("127"); - assertEquals(new HashSet<>(List.of(127)), actualSiteIdsV4TokensSet); - } - - @Test - void getSiteIdsUsingV4Tokens_emptyInput() { - Set actualSiteIdsV4TokensSet = getSiteIdsUsingV4Tokens(""); - assertEquals(new HashSet<>(), actualSiteIdsV4TokensSet); - } - - @Test - void getSiteIdsUsingV4Tokens_inputContainsSpaces() { - Set actualSiteIdsV4TokensSet = getSiteIdsUsingV4Tokens(" 127 ,128 "); - assertEquals(siteIdsV4TokensSet, actualSiteIdsV4TokensSet); - } - - @Test - void getSiteIdsUsingV4Tokens_inputContainsInvalidInteger() { - assertThrows(IllegalArgumentException.class, - () -> getSiteIdsUsingV4Tokens(" 1 27 ,128 ")); - } -} From 9f0f28b58612bc74206fb667e3df09ad1cc8e840 Mon Sep 17 00:00:00 2001 From: Katherine Chen Date: Tue, 3 Dec 2024 16:25:13 +1100 Subject: [PATCH 144/431] UID2-3804 Remove `advertising_token_v4_percentage` env var (#1180) * Remove `advertising_token_v4_percentage` env var * Always return advertising token v4 in tests * Return token version v4 for EUID as well * Remove duplicated code * Remove PRNG * Remove advertising_token_v3 --- conf/docker-config.json | 1 - conf/local-config.json | 2 -- conf/local-e2e-docker-private-config.json | 1 - conf/local-e2e-docker-public-config.json | 1 - conf/local-e2e-private-config.json | 1 - conf/local-e2e-public-config.json | 1 - ...alidator-latest-e2e-docker-public-config.json | 1 - scripts/aws/conf/default-config.json | 1 - scripts/aws/conf/prod-euid-config.json | 1 - scripts/azure-cc/conf/default-config.json | 1 - scripts/gcp-oidc/conf/default-config.json | 1 - .../operator/service/UIDOperatorService.java | 16 +--------------- .../uid2/operator/EUIDOperatorVerticleTest.java | 3 --- .../operator/EUIDOperatorVerticleV4Test.java | 15 --------------- .../uid2/operator/UIDOperatorServiceTest.java | 12 +----------- .../uid2/operator/UIDOperatorVerticleTest.java | 4 +--- .../uid2/operator/UidOperatorVerticleV4Test.java | 14 -------------- 17 files changed, 3 insertions(+), 73 deletions(-) delete mode 100644 src/test/java/com/uid2/operator/EUIDOperatorVerticleV4Test.java delete mode 100644 src/test/java/com/uid2/operator/UidOperatorVerticleV4Test.java diff --git a/conf/docker-config.json b/conf/docker-config.json index 648b922a8..25f38e6ae 100644 --- a/conf/docker-config.json +++ b/conf/docker-config.json @@ -4,7 +4,6 @@ "storage_mock": true, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, - "advertising_token_v3": false, "refresh_token_v3": false, "identity_v3": false, "identity_scope": "uid2", diff --git a/conf/local-config.json b/conf/local-config.json index 7a3200f12..6a357dba1 100644 --- a/conf/local-config.json +++ b/conf/local-config.json @@ -12,8 +12,6 @@ "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, - "advertising_token_v3": false, - "advertising_token_v4_percentage": 0, "refresh_token_v3": false, "identity_v3": false, "identity_scope": "uid2", diff --git a/conf/local-e2e-docker-private-config.json b/conf/local-e2e-docker-private-config.json index ef05b8772..8637e6da3 100644 --- a/conf/local-e2e-docker-private-config.json +++ b/conf/local-e2e-docker-private-config.json @@ -14,7 +14,6 @@ "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, - "advertising_token_v3": false, "refresh_token_v3": true, "identity_v3": false, "identity_scope": "uid2", diff --git a/conf/local-e2e-docker-public-config.json b/conf/local-e2e-docker-public-config.json index 60f0abd92..a145c4d17 100644 --- a/conf/local-e2e-docker-public-config.json +++ b/conf/local-e2e-docker-public-config.json @@ -16,7 +16,6 @@ "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, - "advertising_token_v3": false, "refresh_token_v3": true, "identity_v3": false, "identity_scope": "uid2", diff --git a/conf/local-e2e-private-config.json b/conf/local-e2e-private-config.json index e9d3f8b53..4ab52330f 100644 --- a/conf/local-e2e-private-config.json +++ b/conf/local-e2e-private-config.json @@ -16,7 +16,6 @@ "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, - "advertising_token_v3": false, "refresh_token_v3": true, "identity_v3": false, "identity_scope": "uid2", diff --git a/conf/local-e2e-public-config.json b/conf/local-e2e-public-config.json index cb635b103..bfdc8e394 100644 --- a/conf/local-e2e-public-config.json +++ b/conf/local-e2e-public-config.json @@ -16,7 +16,6 @@ "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, - "advertising_token_v3": false, "refresh_token_v3": true, "identity_v3": false, "identity_scope": "uid2", diff --git a/conf/validator-latest-e2e-docker-public-config.json b/conf/validator-latest-e2e-docker-public-config.json index cabf23380..8f82b01a4 100644 --- a/conf/validator-latest-e2e-docker-public-config.json +++ b/conf/validator-latest-e2e-docker-public-config.json @@ -17,7 +17,6 @@ "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, - "advertising_token_v3": false, "refresh_token_v3": true, "identity_v3": false, "identity_scope": "uid2", diff --git a/scripts/aws/conf/default-config.json b/scripts/aws/conf/default-config.json index 1836f1aa7..35c3be58c 100644 --- a/scripts/aws/conf/default-config.json +++ b/scripts/aws/conf/default-config.json @@ -34,6 +34,5 @@ "failure_shutdown_wait_hours": 120, "sharing_token_expiry_seconds": 2592000, "validate_service_links": false, - "advertising_token_v4_percentage": 100, "operator_type": "private" } diff --git a/scripts/aws/conf/prod-euid-config.json b/scripts/aws/conf/prod-euid-config.json index 581ad0ae9..0fbf5d69c 100644 --- a/scripts/aws/conf/prod-euid-config.json +++ b/scripts/aws/conf/prod-euid-config.json @@ -24,7 +24,6 @@ "refresh_identity_token_after_seconds": 3600, "allow_legacy_api": false, "identity_scope": "euid", - "advertising_token_v3": true, "refresh_token_v3": true, "enable_phone_support": true, "enable_v1_phone_support": false, diff --git a/scripts/azure-cc/conf/default-config.json b/scripts/azure-cc/conf/default-config.json index c47eef8f6..4870b2fda 100644 --- a/scripts/azure-cc/conf/default-config.json +++ b/scripts/azure-cc/conf/default-config.json @@ -38,6 +38,5 @@ "failure_shutdown_wait_hours": 120, "sharing_token_expiry_seconds": 2592000, "validate_service_links": false, - "advertising_token_v4_percentage": 100, "operator_type": "private" } diff --git a/scripts/gcp-oidc/conf/default-config.json b/scripts/gcp-oidc/conf/default-config.json index aaea43364..6a65ee2d0 100644 --- a/scripts/gcp-oidc/conf/default-config.json +++ b/scripts/gcp-oidc/conf/default-config.json @@ -38,6 +38,5 @@ "failure_shutdown_wait_hours": 120, "sharing_token_expiry_seconds": 2592000, "validate_service_links": false, - "advertising_token_v4_percentage": 100, "operator_type": "private" } diff --git a/src/main/java/com/uid2/operator/service/UIDOperatorService.java b/src/main/java/com/uid2/operator/service/UIDOperatorService.java index 7e6450829..6d4ff86d0 100644 --- a/src/main/java/com/uid2/operator/service/UIDOperatorService.java +++ b/src/main/java/com/uid2/operator/service/UIDOperatorService.java @@ -46,8 +46,6 @@ public class UIDOperatorService implements IUIDOperatorService { private final Duration refreshIdentityAfter; private final OperatorIdentity operatorIdentity; - protected final TokenVersion tokenVersionToUseIfNotV4; - protected final int advertisingTokenV4Percentage; private final TokenVersion refreshTokenVersion; private final boolean identityV3Enabled; @@ -91,9 +89,6 @@ public UIDOperatorService(JsonObject config, IOptOutStore optOutStore, ISaltProv throw new IllegalStateException(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS + " must be >= " + REFRESH_IDENTITY_TOKEN_AFTER_SECONDS); } - this.advertisingTokenV4Percentage = config.getInteger("advertising_token_v4_percentage", 0); //0 indicates token v4 will not be used - this.tokenVersionToUseIfNotV4 = config.getBoolean("advertising_token_v3", false) ? TokenVersion.V3 : TokenVersion.V2; - this.refreshTokenVersion = TokenVersion.V3; this.identityV3Enabled = config.getBoolean("identity_v3", false); } @@ -267,16 +262,7 @@ private RefreshToken createRefreshToken(PublisherIdentity publisherIdentity, Use } private AdvertisingToken createAdvertisingToken(PublisherIdentity publisherIdentity, UserIdentity userIdentity, Instant now) { - TokenVersion tokenVersion; - int pseudoRandomNumber = 1; - final var rawUid = userIdentity.id; - if (rawUid.length > 2) - { - int hash = ((rawUid[0] & 0xFF) << 12) | ((rawUid[1] & 0xFF) << 4) | ((rawUid[2] & 0xFF) & 0xF); //using same logic as ModBasedSaltEntryIndexer.getIndex() in uid2-shared - pseudoRandomNumber = (hash % 100) + 1; //1 to 100 - } - tokenVersion = (pseudoRandomNumber <= this.advertisingTokenV4Percentage) ? TokenVersion.V4 : this.tokenVersionToUseIfNotV4; - return new AdvertisingToken(tokenVersion, now, now.plusMillis(identityExpiresAfter.toMillis()), this.operatorIdentity, publisherIdentity, userIdentity); + return new AdvertisingToken(TokenVersion.V4, now, now.plusMillis(identityExpiresAfter.toMillis()), this.operatorIdentity, publisherIdentity, userIdentity); } static protected class GlobalOptoutResult { diff --git a/src/test/java/com/uid2/operator/EUIDOperatorVerticleTest.java b/src/test/java/com/uid2/operator/EUIDOperatorVerticleTest.java index 8d168d42e..138e17777 100644 --- a/src/test/java/com/uid2/operator/EUIDOperatorVerticleTest.java +++ b/src/test/java/com/uid2/operator/EUIDOperatorVerticleTest.java @@ -18,9 +18,6 @@ public class EUIDOperatorVerticleTest extends UIDOperatorVerticleTest { public EUIDOperatorVerticleTest() throws IOException { } - @Override - protected TokenVersion getTokenVersion() {return TokenVersion.V3;} - @Override protected IdentityScope getIdentityScope() { return IdentityScope.EUID; } @Override diff --git a/src/test/java/com/uid2/operator/EUIDOperatorVerticleV4Test.java b/src/test/java/com/uid2/operator/EUIDOperatorVerticleV4Test.java deleted file mode 100644 index fb5ff985b..000000000 --- a/src/test/java/com/uid2/operator/EUIDOperatorVerticleV4Test.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.uid2.operator; - -import com.uid2.shared.model.TokenVersion; - -import java.io.IOException; - -public class EUIDOperatorVerticleV4Test extends EUIDOperatorVerticleTest { - public EUIDOperatorVerticleV4Test() throws IOException { - } - - @Override - protected TokenVersion getTokenVersion() { - return TokenVersion.V4; - } -} diff --git a/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java b/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java index 4a8aef9d1..37eeef36f 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java @@ -57,11 +57,6 @@ class ExtendedUIDOperatorService extends UIDOperatorService { public ExtendedUIDOperatorService(JsonObject config, IOptOutStore optOutStore, ISaltProvider saltProvider, ITokenEncoder encoder, Clock clock, IdentityScope identityScope, Handler saltRetrievalResponseHandler) { super(config, optOutStore, saltProvider, encoder, clock, identityScope, saltRetrievalResponseHandler); } - - public TokenVersion getAdvertisingTokenVersionForTests() { - assert this.advertisingTokenV4Percentage == 0 || this.advertisingTokenV4Percentage == 100; //we want tests to be deterministic - return this.advertisingTokenV4Percentage == 100 ? TokenVersion.V4 : this.tokenVersionToUseIfNotV4; - } } @BeforeEach @@ -93,8 +88,6 @@ void setup() throws Exception { uid2Config.put(UIDOperatorService.IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS); uid2Config.put(UIDOperatorService.REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, REFRESH_TOKEN_EXPIRES_AFTER_SECONDS); uid2Config.put(UIDOperatorService.REFRESH_IDENTITY_TOKEN_AFTER_SECONDS, REFRESH_IDENTITY_TOKEN_AFTER_SECONDS); - uid2Config.put("advertising_token_v4_percentage", 100); - uid2Config.put("advertising_token_v3", false); // prod is using v2 token version for now uid2Config.put("identity_v3", false); uid2Service = new ExtendedUIDOperatorService( @@ -111,8 +104,6 @@ void setup() throws Exception { euidConfig.put(UIDOperatorService.IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS); euidConfig.put(UIDOperatorService.REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, REFRESH_TOKEN_EXPIRES_AFTER_SECONDS); euidConfig.put(UIDOperatorService.REFRESH_IDENTITY_TOKEN_AFTER_SECONDS, REFRESH_IDENTITY_TOKEN_AFTER_SECONDS); - euidConfig.put("advertising_token_v4_percentage", 0); - euidConfig.put("advertising_token_v3", true); euidConfig.put("identity_v3", true); euidService = new ExtendedUIDOperatorService( @@ -148,8 +139,7 @@ private UserIdentity createUserIdentity(String rawIdentityHash, IdentityScope sc } private AdvertisingToken validateAndGetToken(EncryptedTokenEncoder tokenEncoder, String advertisingTokenString, IdentityScope scope, IdentityType type, int siteId) { - TokenVersion tokenVersion = (scope == IdentityScope.UID2) ? uid2Service.getAdvertisingTokenVersionForTests() : euidService.getAdvertisingTokenVersionForTests(); - UIDOperatorVerticleTest.validateAdvertisingToken(advertisingTokenString, tokenVersion, scope, type); + UIDOperatorVerticleTest.validateAdvertisingToken(advertisingTokenString, TokenVersion.V4, scope, type); return tokenEncoder.decodeAdvertisingToken(advertisingTokenString); } diff --git a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java index 6ef9ba989..d9a91ae01 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java @@ -158,8 +158,6 @@ private void setupConfig(JsonObject config) { config.put(Const.Config.SharingTokenExpiryProp, 60 * 60 * 24 * 30); config.put("identity_scope", getIdentityScope().toString()); - config.put("advertising_token_v3", getTokenVersion() == TokenVersion.V3); - config.put("advertising_token_v4_percentage", getTokenVersion() == TokenVersion.V4 ? 100 : 0); config.put("identity_v3", useIdentityV3()); config.put("client_side_token_generate", true); config.put("key_sharing_endpoint_provide_app_names", true); @@ -665,7 +663,7 @@ private JsonObject setupIdentityMapServiceLinkTest() { return req; } - protected TokenVersion getTokenVersion() {return TokenVersion.V2;} + protected TokenVersion getTokenVersion() {return TokenVersion.V4;} final boolean useIdentityV3() { return getTokenVersion() != TokenVersion.V2; } protected IdentityScope getIdentityScope() { return IdentityScope.UID2; } diff --git a/src/test/java/com/uid2/operator/UidOperatorVerticleV4Test.java b/src/test/java/com/uid2/operator/UidOperatorVerticleV4Test.java deleted file mode 100644 index 7a040427e..000000000 --- a/src/test/java/com/uid2/operator/UidOperatorVerticleV4Test.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.uid2.operator; - -import com.uid2.shared.model.TokenVersion; - -import java.io.IOException; - -public class UidOperatorVerticleV4Test extends UIDOperatorVerticleTest { - public UidOperatorVerticleV4Test() throws IOException { - } - - @Override - protected TokenVersion getTokenVersion() {return TokenVersion.V4;} - -} From 7d2ad652270ea14737ec98782b7947111788b8e2 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Tue, 3 Dec 2024 05:27:48 +0000 Subject: [PATCH 145/431] [CI Pipeline] Released Patch version: 5.42.17 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 13a9c3676..f5fb728c5 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.42.7-alpha-139-SNAPSHOT + 5.42.17 UTF-8 From a002ad1b0374fa76c663bd8ec0b0810d15626433 Mon Sep 17 00:00:00 2001 From: Sunny Wu Date: Thu, 5 Dec 2024 13:20:55 +1100 Subject: [PATCH 146/431] Code review feedback, Mainly renaming RefreshTokenRequest/RefreshResponse to TokenRefreshRequest/TokenRefreshResponse --- .../java/com/uid2/operator/IdentityConst.java | 2 +- .../model/AdvertisingTokenRequest.java | 4 +- .../uid2/operator/model/IdentityResponse.java | 2 +- .../uid2/operator/model/SourcePublisher.java | 4 + ...nRequest.java => TokenRefreshRequest.java} | 4 +- ...esponse.java => TokenRefreshResponse.java} | 24 ++-- ...dToken.java => VersionedTokenRequest.java} | 6 +- .../TokenResponseStatsCollector.java | 4 +- .../service/EncryptedTokenEncoder.java | 26 ++-- .../uid2/operator/service/ITokenEncoder.java | 6 +- .../operator/service/IUIDOperatorService.java | 2 +- .../operator/service/UIDOperatorService.java | 24 ++-- .../operator/vertx/UIDOperatorVerticle.java | 22 ++-- .../com/uid2/operator/TokenEncodingTest.java | 20 +-- .../uid2/operator/UIDOperatorServiceTest.java | 74 +++++------ .../operator/UIDOperatorVerticleTest.java | 122 +++++++++--------- .../benchmark/TokenEndecBenchmark.java | 3 +- .../utilTests/IdentityResponseTest.java | 4 +- 18 files changed, 178 insertions(+), 175 deletions(-) rename src/main/java/com/uid2/operator/model/{RefreshTokenRequest.java => TokenRefreshRequest.java} (88%) rename src/main/java/com/uid2/operator/model/{RefreshResponse.java => TokenRefreshResponse.java} (55%) rename src/main/java/com/uid2/operator/model/{VersionedToken.java => VersionedTokenRequest.java} (68%) diff --git a/src/main/java/com/uid2/operator/IdentityConst.java b/src/main/java/com/uid2/operator/IdentityConst.java index 9dad70a0c..402a99d88 100644 --- a/src/main/java/com/uid2/operator/IdentityConst.java +++ b/src/main/java/com/uid2/operator/IdentityConst.java @@ -13,7 +13,7 @@ public class IdentityConst { public static final byte[] ValidateIdentityForEmailHash = EncodingUtils.getSha256Bytes(IdentityConst.ValidateIdentityForEmail); public static final byte[] ValidateIdentityForPhoneHash = EncodingUtils.getSha256Bytes(IdentityConst.ValidateIdentityForPhone); - // DIIs to use when you want to generate a optout response in token generation or identity map + // DIIs to use when you want to generate an optout response in token generation or identity map public static final String OptOutIdentityForEmail = "optout@example.com"; public static final String OptOutIdentityForPhone = "+00000000000"; diff --git a/src/main/java/com/uid2/operator/model/AdvertisingTokenRequest.java b/src/main/java/com/uid2/operator/model/AdvertisingTokenRequest.java index 226b0e122..f9a41caf6 100644 --- a/src/main/java/com/uid2/operator/model/AdvertisingTokenRequest.java +++ b/src/main/java/com/uid2/operator/model/AdvertisingTokenRequest.java @@ -6,8 +6,8 @@ import com.uid2.operator.util.PrivacyBits; import com.uid2.shared.model.TokenVersion; -// class containing enough data to create a new uid or advertising token -public class AdvertisingTokenRequest extends VersionedToken { +// class containing enough information to create a new uid token (aka advertising token) +public class AdvertisingTokenRequest extends VersionedTokenRequest { public final OperatorIdentity operatorIdentity; public final SourcePublisher sourcePublisher; public final RawUidIdentity rawUidIdentity; diff --git a/src/main/java/com/uid2/operator/model/IdentityResponse.java b/src/main/java/com/uid2/operator/model/IdentityResponse.java index 11e2f4c6e..746294064 100644 --- a/src/main/java/com/uid2/operator/model/IdentityResponse.java +++ b/src/main/java/com/uid2/operator/model/IdentityResponse.java @@ -9,7 +9,7 @@ // jsonified // todo: can be converted to record later public class IdentityResponse { - public static IdentityResponse OptOutIdentityResponse = new IdentityResponse("", null, "", Instant.EPOCH, Instant.EPOCH, Instant.EPOCH); + public static final IdentityResponse OptOutResponse = new IdentityResponse("", null, "", Instant.EPOCH, Instant.EPOCH, Instant.EPOCH); //aka UID token private final String advertisingToken; diff --git a/src/main/java/com/uid2/operator/model/SourcePublisher.java b/src/main/java/com/uid2/operator/model/SourcePublisher.java index a6b4f49cd..bd19740a1 100644 --- a/src/main/java/com/uid2/operator/model/SourcePublisher.java +++ b/src/main/java/com/uid2/operator/model/SourcePublisher.java @@ -3,6 +3,10 @@ // The original publisher that requests to generate a UID token public class SourcePublisher { public final int siteId; + + // these 2 values are added into adverting/UID token and refresh token payload but + // are not really used for any real purposes currently so sometimes are set to 0 + // see the constructor below public final int clientKeyId; public final long publisherId; diff --git a/src/main/java/com/uid2/operator/model/RefreshTokenRequest.java b/src/main/java/com/uid2/operator/model/TokenRefreshRequest.java similarity index 88% rename from src/main/java/com/uid2/operator/model/RefreshTokenRequest.java rename to src/main/java/com/uid2/operator/model/TokenRefreshRequest.java index 770b6dff5..f5cc8b6eb 100644 --- a/src/main/java/com/uid2/operator/model/RefreshTokenRequest.java +++ b/src/main/java/com/uid2/operator/model/TokenRefreshRequest.java @@ -7,7 +7,7 @@ import com.uid2.shared.model.TokenVersion; // class containing enough data to create a new refresh token -public class RefreshTokenRequest extends VersionedToken { +public class TokenRefreshRequest extends VersionedTokenRequest { public final OperatorIdentity operatorIdentity; public final SourcePublisher sourcePublisher; public final FirstLevelHashIdentity firstLevelHashIdentity; @@ -15,7 +15,7 @@ public class RefreshTokenRequest extends VersionedToken { public final PrivacyBits privacyBits; - public RefreshTokenRequest(TokenVersion version, Instant createdAt, Instant expiresAt, OperatorIdentity operatorIdentity, + public TokenRefreshRequest(TokenVersion version, Instant createdAt, Instant expiresAt, OperatorIdentity operatorIdentity, SourcePublisher sourcePublisher, FirstLevelHashIdentity firstLevelHashIdentity, PrivacyBits privacyBits) { super(version, createdAt, expiresAt); this.operatorIdentity = operatorIdentity; diff --git a/src/main/java/com/uid2/operator/model/RefreshResponse.java b/src/main/java/com/uid2/operator/model/TokenRefreshResponse.java similarity index 55% rename from src/main/java/com/uid2/operator/model/RefreshResponse.java rename to src/main/java/com/uid2/operator/model/TokenRefreshResponse.java index 7fc556b1f..5a52fa571 100644 --- a/src/main/java/com/uid2/operator/model/RefreshResponse.java +++ b/src/main/java/com/uid2/operator/model/TokenRefreshResponse.java @@ -2,32 +2,32 @@ import java.time.Duration; -public class RefreshResponse { - - public static final RefreshResponse Invalid = new RefreshResponse(Status.Invalid, - IdentityResponse.OptOutIdentityResponse); - public static final RefreshResponse Optout = new RefreshResponse(Status.Optout, IdentityResponse.OptOutIdentityResponse); - public static final RefreshResponse Expired = new RefreshResponse(Status.Expired, IdentityResponse.OptOutIdentityResponse); - public static final RefreshResponse Deprecated = new RefreshResponse(Status.Deprecated, IdentityResponse.OptOutIdentityResponse); - public static final RefreshResponse NoActiveKey = new RefreshResponse(Status.NoActiveKey, IdentityResponse.OptOutIdentityResponse); +public class TokenRefreshResponse { + + public static final TokenRefreshResponse Invalid = new TokenRefreshResponse(Status.Invalid, + IdentityResponse.OptOutResponse); + public static final TokenRefreshResponse Optout = new TokenRefreshResponse(Status.Optout, IdentityResponse.OptOutResponse); + public static final TokenRefreshResponse Expired = new TokenRefreshResponse(Status.Expired, IdentityResponse.OptOutResponse); + public static final TokenRefreshResponse Deprecated = new TokenRefreshResponse(Status.Deprecated, IdentityResponse.OptOutResponse); + public static final TokenRefreshResponse NoActiveKey = new TokenRefreshResponse(Status.NoActiveKey, IdentityResponse.OptOutResponse); private final Status status; private final IdentityResponse identityResponse; private final Duration durationSinceLastRefresh; private final boolean isCstg; - private RefreshResponse(Status status, IdentityResponse identityResponse, Duration durationSinceLastRefresh, boolean isCstg) { + private TokenRefreshResponse(Status status, IdentityResponse identityResponse, Duration durationSinceLastRefresh, boolean isCstg) { this.status = status; this.identityResponse = identityResponse; this.durationSinceLastRefresh = durationSinceLastRefresh; this.isCstg = isCstg; } - private RefreshResponse(Status status, IdentityResponse identityResponse) { + private TokenRefreshResponse(Status status, IdentityResponse identityResponse) { this(status, identityResponse, null, false); } - public static RefreshResponse createRefreshedResponse(IdentityResponse identityResponse, Duration durationSinceLastRefresh, boolean isCstg) { - return new RefreshResponse(Status.Refreshed, identityResponse, durationSinceLastRefresh, isCstg); + public static TokenRefreshResponse createRefreshedResponse(IdentityResponse identityResponse, Duration durationSinceLastRefresh, boolean isCstg) { + return new TokenRefreshResponse(Status.Refreshed, identityResponse, durationSinceLastRefresh, isCstg); } public Status getStatus() { diff --git a/src/main/java/com/uid2/operator/model/VersionedToken.java b/src/main/java/com/uid2/operator/model/VersionedTokenRequest.java similarity index 68% rename from src/main/java/com/uid2/operator/model/VersionedToken.java rename to src/main/java/com/uid2/operator/model/VersionedTokenRequest.java index 5be86b80e..5cc9c5335 100644 --- a/src/main/java/com/uid2/operator/model/VersionedToken.java +++ b/src/main/java/com/uid2/operator/model/VersionedTokenRequest.java @@ -1,16 +1,16 @@ package com.uid2.operator.model; import java.time.Instant; -import java.util.Objects; + import com.uid2.shared.model.TokenVersion; -public abstract class VersionedToken { +public abstract class VersionedTokenRequest { public final TokenVersion version; public final Instant createdAt; public final Instant expiresAt; - public VersionedToken(TokenVersion version, Instant createdAt, Instant expiresAt) { + public VersionedTokenRequest(TokenVersion version, Instant createdAt, Instant expiresAt) { this.version = version; this.createdAt = createdAt; this.expiresAt = expiresAt; diff --git a/src/main/java/com/uid2/operator/monitoring/TokenResponseStatsCollector.java b/src/main/java/com/uid2/operator/monitoring/TokenResponseStatsCollector.java index fc28bba70..c5f46cc7e 100644 --- a/src/main/java/com/uid2/operator/monitoring/TokenResponseStatsCollector.java +++ b/src/main/java/com/uid2/operator/monitoring/TokenResponseStatsCollector.java @@ -1,6 +1,6 @@ package com.uid2.operator.monitoring; -import com.uid2.operator.model.RefreshResponse; +import com.uid2.operator.model.TokenRefreshResponse; import com.uid2.operator.vertx.UIDOperatorVerticle; import com.uid2.shared.model.TokenVersion; import com.uid2.shared.store.ISiteStore; @@ -69,7 +69,7 @@ private static void recordInternal(ISiteStore siteStore, Integer siteId, Endpoin builder.register(Metrics.globalRegistry).increment(); } - public static void recordRefresh(ISiteStore siteStore, Integer siteId, Endpoint endpoint, RefreshResponse refreshResponse, PlatformType platformType) { + public static void recordRefresh(ISiteStore siteStore, Integer siteId, Endpoint endpoint, TokenRefreshResponse refreshResponse, PlatformType platformType) { if (!refreshResponse.isRefreshed()) { if (refreshResponse.isOptOut() || refreshResponse.isDeprecated()) { recordInternal(siteStore, siteId, endpoint, ResponseStatus.OptOut, refreshResponse.getIdentityResponse().getAdvertisingTokenVersion(), refreshResponse.isCstg(), platformType); diff --git a/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java b/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java index 0ca19d803..26d019795 100644 --- a/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java +++ b/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java @@ -78,7 +78,7 @@ private byte[] encodeIntoAdvertisingTokenV3(AdvertisingTokenRequest t, KeysetKey } @Override - public RefreshTokenRequest decodeRefreshToken(String s) { + public TokenRefreshRequest decodeRefreshToken(String s) { if (s != null && !s.isEmpty()) { final byte[] bytes; try { @@ -97,7 +97,7 @@ public RefreshTokenRequest decodeRefreshToken(String s) { throw new ClientInputValidationException("Invalid refresh token version"); } - private RefreshTokenRequest decodeRefreshTokenV2(Buffer b) { + private TokenRefreshRequest decodeRefreshTokenV2(Buffer b) { final Instant createdAt = Instant.ofEpochMilli(b.getLong(1)); //final Instant expiresAt = Instant.ofEpochMilli(b.getLong(9)); final Instant validTill = Instant.ofEpochMilli(b.getLong(17)); @@ -125,7 +125,7 @@ private RefreshTokenRequest decodeRefreshTokenV2(Buffer b) { final PrivacyBits privacyBits = PrivacyBits.fromInt(b2.getInt(8 + length)); final long establishedMillis = b2.getLong(8 + length + 4); - return new RefreshTokenRequest( + return new TokenRefreshRequest( TokenVersion.V2, createdAt, validTill, new OperatorIdentity(0, OperatorType.Service, 0, 0), new SourcePublisher(siteId), @@ -134,7 +134,7 @@ private RefreshTokenRequest decodeRefreshTokenV2(Buffer b) { privacyBits); } - private RefreshTokenRequest decodeRefreshTokenV3(Buffer b, byte[] bytes) { + private TokenRefreshRequest decodeRefreshTokenV3(Buffer b, byte[] bytes) { final int keyId = b.getInt(2); final KeysetKey key = this.keyManager.getKey(keyId); @@ -162,7 +162,7 @@ private RefreshTokenRequest decodeRefreshTokenV3(Buffer b, byte[] bytes) { throw new ClientInputValidationException("Failed to decode refreshTokenV3: Identity type mismatch"); } - return new RefreshTokenRequest( + return new TokenRefreshRequest( TokenVersion.V3, createdAt, expiresAt, operatorIdentity, sourcePublisher, new FirstLevelHashIdentity(identityScope, identityType, firstLevelHash, establishedAt), privacyBits); @@ -289,7 +289,7 @@ private void recordRefreshTokenVersionCount(String siteId, TokenVersion tokenVer .register(Metrics.globalRegistry).increment(); } - public byte[] encodeIntoRefreshToken(RefreshTokenRequest t, Instant asOf) { + public byte[] encodeIntoRefreshToken(TokenRefreshRequest t, Instant asOf) { final KeysetKey serviceKey = this.keyManager.getRefreshKey(asOf); switch (t.version) { @@ -304,7 +304,7 @@ public byte[] encodeIntoRefreshToken(RefreshTokenRequest t, Instant asOf) { } } - public byte[] encodeIntoRefreshTokenV2(RefreshTokenRequest t, KeysetKey serviceKey) { + public byte[] encodeIntoRefreshTokenV2(TokenRefreshRequest t, KeysetKey serviceKey) { final Buffer b = Buffer.buffer(); b.appendByte((byte) t.version.rawVersion); b.appendLong(t.createdAt.toEpochMilli()); @@ -318,7 +318,7 @@ public byte[] encodeIntoRefreshTokenV2(RefreshTokenRequest t, KeysetKey serviceK return b.getBytes(); } - public byte[] encodeIntoRefreshTokenV3(RefreshTokenRequest t, KeysetKey serviceKey) { + public byte[] encodeIntoRefreshTokenV3(TokenRefreshRequest t, KeysetKey serviceKey) { final Buffer refreshPayload = Buffer.buffer(90); refreshPayload.appendLong(t.expiresAt.toEpochMilli()); refreshPayload.appendLong(t.createdAt.toEpochMilli()); @@ -351,21 +351,21 @@ public static String bytesToBase64Token(byte[] advertisingTokenBytes, TokenVersi } @Override - public IdentityResponse encodeIntoIdentityResponse(AdvertisingTokenRequest advertisingTokenRequest, RefreshTokenRequest refreshTokenRequest, Instant refreshFrom, Instant asOf) { + public IdentityResponse encodeIntoIdentityResponse(AdvertisingTokenRequest advertisingTokenRequest, TokenRefreshRequest tokenRefreshRequest, Instant refreshFrom, Instant asOf) { final String advertisingToken = generateAdvertisingTokenString(advertisingTokenRequest, asOf); - final String refreshToken = generateRefreshTokenString(refreshTokenRequest, asOf); + final String refreshToken = generateRefreshTokenString(tokenRefreshRequest, asOf); return new IdentityResponse( advertisingToken, advertisingTokenRequest.version, refreshToken, advertisingTokenRequest.expiresAt, - refreshTokenRequest.expiresAt, + tokenRefreshRequest.expiresAt, refreshFrom ); } - private String generateRefreshTokenString(RefreshTokenRequest refreshTokenRequest, Instant asOf) { - return EncodingUtils.toBase64String(encodeIntoRefreshToken(refreshTokenRequest, asOf)); + private String generateRefreshTokenString(TokenRefreshRequest tokenRefreshRequest, Instant asOf) { + return EncodingUtils.toBase64String(encodeIntoRefreshToken(tokenRefreshRequest, asOf)); } private String generateAdvertisingTokenString(AdvertisingTokenRequest advertisingTokenRequest, Instant asOf) { diff --git a/src/main/java/com/uid2/operator/service/ITokenEncoder.java b/src/main/java/com/uid2/operator/service/ITokenEncoder.java index cac7994ae..c8beddc85 100644 --- a/src/main/java/com/uid2/operator/service/ITokenEncoder.java +++ b/src/main/java/com/uid2/operator/service/ITokenEncoder.java @@ -2,14 +2,14 @@ import com.uid2.operator.model.AdvertisingTokenRequest; import com.uid2.operator.model.IdentityResponse; -import com.uid2.operator.model.RefreshTokenRequest; +import com.uid2.operator.model.TokenRefreshRequest; import java.time.Instant; public interface ITokenEncoder { - IdentityResponse encodeIntoIdentityResponse(AdvertisingTokenRequest advertisingTokenRequest, RefreshTokenRequest refreshTokenRequest, Instant refreshFrom, Instant asOf); + IdentityResponse encodeIntoIdentityResponse(AdvertisingTokenRequest advertisingTokenRequest, TokenRefreshRequest tokenRefreshRequest, Instant refreshFrom, Instant asOf); AdvertisingTokenRequest decodeAdvertisingToken(String base64String); - RefreshTokenRequest decodeRefreshToken(String base64String); + TokenRefreshRequest decodeRefreshToken(String base64String); } diff --git a/src/main/java/com/uid2/operator/service/IUIDOperatorService.java b/src/main/java/com/uid2/operator/service/IUIDOperatorService.java index eacb244a2..879d784bd 100644 --- a/src/main/java/com/uid2/operator/service/IUIDOperatorService.java +++ b/src/main/java/com/uid2/operator/service/IUIDOperatorService.java @@ -14,7 +14,7 @@ public interface IUIDOperatorService { IdentityResponse generateIdentity(IdentityRequest request); - RefreshResponse refreshIdentity(RefreshTokenRequest input); + TokenRefreshResponse refreshIdentity(TokenRefreshRequest input); RawUidResponse mapHashedDiiIdentity(MapRequest request); diff --git a/src/main/java/com/uid2/operator/service/UIDOperatorService.java b/src/main/java/com/uid2/operator/service/UIDOperatorService.java index 9f0e36975..ee9685662 100644 --- a/src/main/java/com/uid2/operator/service/UIDOperatorService.java +++ b/src/main/java/com/uid2/operator/service/UIDOperatorService.java @@ -113,27 +113,27 @@ public IdentityResponse generateIdentity(IdentityRequest request) { request.establishedAt); if (request.shouldCheckOptOut() && getGlobalOptOutResult(firstLevelHashIdentity, false).isOptedOut()) { - return IdentityResponse.OptOutIdentityResponse; + return IdentityResponse.OptOutResponse; } else { return generateIdentity(request.sourcePublisher, firstLevelHashIdentity, request.privacyBits); } } @Override - public RefreshResponse refreshIdentity(RefreshTokenRequest input) { + public TokenRefreshResponse refreshIdentity(TokenRefreshRequest input) { // should not be possible as different scopes should be using different keys, but just in case if (input.firstLevelHashIdentity.identityScope != this.identityScope) { - return RefreshResponse.Invalid; + return TokenRefreshResponse.Invalid; } if (input.firstLevelHashIdentity.establishedAt.isBefore(RefreshCutoff)) { - return RefreshResponse.Deprecated; + return TokenRefreshResponse.Deprecated; } final Instant now = clock.instant(); if (input.expiresAt.isBefore(now)) { - return RefreshResponse.Expired; + return TokenRefreshResponse.Expired; } final boolean isCstg = input.privacyBits.isClientSideTokenGenerated(); @@ -149,14 +149,14 @@ public RefreshResponse refreshIdentity(RefreshTokenRequest input) { input.firstLevelHashIdentity, input.privacyBits); - return RefreshResponse.createRefreshedResponse(identityResponse, durationSinceLastRefresh, isCstg); + return TokenRefreshResponse.createRefreshedResponse(identityResponse, durationSinceLastRefresh, isCstg); } else { - return RefreshResponse.Optout; + return TokenRefreshResponse.Optout; } } catch (KeyManager.NoActiveKeyException e) { - return RefreshResponse.NoActiveKey; + return TokenRefreshResponse.NoActiveKey; } catch (Exception ex) { - return RefreshResponse.Invalid; + return TokenRefreshResponse.Invalid; } } @@ -262,17 +262,17 @@ private IdentityResponse generateIdentity(SourcePublisher sourcePublisher, return this.encoder.encodeIntoIdentityResponse( this.createAdvertisingTokenRequest(sourcePublisher, rawUidIdentity, nowUtc, privacyBits, firstLevelHashIdentity.establishedAt), - this.createRefreshTokenRequest(sourcePublisher, firstLevelHashIdentity, nowUtc, privacyBits), + this.createTokenRefreshRequest(sourcePublisher, firstLevelHashIdentity, nowUtc, privacyBits), nowUtc.plusMillis(refreshIdentityAfter.toMillis()), nowUtc ); } - private RefreshTokenRequest createRefreshTokenRequest(SourcePublisher sourcePublisher, + private TokenRefreshRequest createTokenRefreshRequest(SourcePublisher sourcePublisher, FirstLevelHashIdentity firstLevelHashIdentity, Instant now, PrivacyBits privacyBits) { - return new RefreshTokenRequest( + return new TokenRefreshRequest( this.refreshTokenVersion, now, now.plusMillis(refreshExpiresAfter.toMillis()), diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index 83c6e5ce8..ecfb9302e 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -813,7 +813,7 @@ private void handleTokenRefreshV1(RoutingContext rc) { } try { - final RefreshResponse r = this.refreshIdentity(rc, refreshToken); + final TokenRefreshResponse r = this.refreshIdentity(rc, refreshToken); siteId = rc.get(Const.RoutingContextData.SiteId); if (!r.isRefreshed()) { if (r.isOptOut() || r.isDeprecated()) { @@ -845,7 +845,7 @@ private void handleTokenRefreshV2(RoutingContext rc) { try { platformType = getPlatformType(rc); String tokenStr = (String) rc.data().get("request"); - final RefreshResponse r = this.refreshIdentity(rc, tokenStr); + final TokenRefreshResponse r = this.refreshIdentity(rc, tokenStr); siteId = rc.get(Const.RoutingContextData.SiteId); if (!r.isRefreshed()) { if (r.isOptOut() || r.isDeprecated()) { @@ -1070,7 +1070,7 @@ private void handleTokenRefresh(RoutingContext rc) { } try { - final RefreshResponse r = this.refreshIdentity(rc, tokenList.get(0)); + final TokenRefreshResponse r = this.refreshIdentity(rc, tokenList.get(0)); sendJsonResponse(rc, r.getIdentityResponse().toJsonV0()); @@ -1764,26 +1764,26 @@ private void recordRefreshTokenVersionCount(String siteId, TokenVersion tokenVer } - private RefreshResponse refreshIdentity(RoutingContext rc, String tokenStr) { - final RefreshTokenRequest refreshTokenRequest; + private TokenRefreshResponse refreshIdentity(RoutingContext rc, String tokenStr) { + final TokenRefreshRequest tokenRefreshRequest; try { if (AuthMiddleware.isAuthenticated(rc)) { rc.put(Const.RoutingContextData.SiteId, AuthMiddleware.getAuthClient(ClientKey.class, rc).getSiteId()); } - refreshTokenRequest = this.encoder.decodeRefreshToken(tokenStr); + tokenRefreshRequest = this.encoder.decodeRefreshToken(tokenStr); } catch (ClientInputValidationException cie) { LOGGER.warn("Failed to decode refresh token for site ID: " + rc.data().get(Const.RoutingContextData.SiteId), cie); - return RefreshResponse.Invalid; + return TokenRefreshResponse.Invalid; } - if (refreshTokenRequest == null) { - return RefreshResponse.Invalid; + if (tokenRefreshRequest == null) { + return TokenRefreshResponse.Invalid; } if (!AuthMiddleware.isAuthenticated(rc)) { - rc.put(Const.RoutingContextData.SiteId, refreshTokenRequest.sourcePublisher.siteId); + rc.put(Const.RoutingContextData.SiteId, tokenRefreshRequest.sourcePublisher.siteId); } recordRefreshTokenVersionCount(String.valueOf(rc.data().get(Const.RoutingContextData.SiteId)), this.getRefreshTokenVersion(tokenStr)); - return this.idService.refreshIdentity(refreshTokenRequest); + return this.idService.refreshIdentity(tokenRefreshRequest); } public static String getSiteName(ISiteStore siteStore, Integer siteId) { diff --git a/src/test/java/com/uid2/operator/TokenEncodingTest.java b/src/test/java/com/uid2/operator/TokenEncodingTest.java index 4affea1f0..0b0009ef7 100644 --- a/src/test/java/com/uid2/operator/TokenEncodingTest.java +++ b/src/test/java/com/uid2/operator/TokenEncodingTest.java @@ -55,7 +55,7 @@ public void testRefreshTokenEncoding(TokenVersion tokenVersion) { final byte[] firstLevelHash = TokenUtils.getFirstLevelHashFromIdentity("test@example.com", "some-salt"); - final RefreshTokenRequest refreshTokenRequest = new RefreshTokenRequest(tokenVersion, + final TokenRefreshRequest tokenRefreshRequest = new TokenRefreshRequest(tokenVersion, now, now.plusSeconds(360), new OperatorIdentity(101, OperatorType.Service, 102, 103), @@ -65,20 +65,20 @@ public void testRefreshTokenEncoding(TokenVersion tokenVersion) { ); if (tokenVersion == TokenVersion.V4) { - Assert.assertThrows(Exception.class, () -> encoder.encodeIntoRefreshToken(refreshTokenRequest, now)); + Assert.assertThrows(Exception.class, () -> encoder.encodeIntoRefreshToken(tokenRefreshRequest, now)); return; //V4 not supported for RefreshTokens } - final byte[] encodedBytes = encoder.encodeIntoRefreshToken(refreshTokenRequest, now); - final RefreshTokenRequest decoded = encoder.decodeRefreshToken(EncodingUtils.toBase64String(encodedBytes)); + final byte[] encodedBytes = encoder.encodeIntoRefreshToken(tokenRefreshRequest, now); + final TokenRefreshRequest decoded = encoder.decodeRefreshToken(EncodingUtils.toBase64String(encodedBytes)); assertEquals(tokenVersion, decoded.version); - assertEquals(refreshTokenRequest.createdAt, decoded.createdAt); + assertEquals(tokenRefreshRequest.createdAt, decoded.createdAt); int addSeconds = (tokenVersion == TokenVersion.V2) ? 60 : 0; //todo: why is there a 60 second buffer in encodeV2() but not in encodeV3()? - assertEquals(refreshTokenRequest.expiresAt.plusSeconds(addSeconds), decoded.expiresAt); - assertTrue(refreshTokenRequest.firstLevelHashIdentity.matches(decoded.firstLevelHashIdentity)); - assertEquals(refreshTokenRequest.privacyBits, decoded.privacyBits); - assertEquals(refreshTokenRequest.firstLevelHashIdentity.establishedAt, decoded.firstLevelHashIdentity.establishedAt); - assertEquals(refreshTokenRequest.sourcePublisher.siteId, decoded.sourcePublisher.siteId); + assertEquals(tokenRefreshRequest.expiresAt.plusSeconds(addSeconds), decoded.expiresAt); + assertTrue(tokenRefreshRequest.firstLevelHashIdentity.matches(decoded.firstLevelHashIdentity)); + assertEquals(tokenRefreshRequest.privacyBits, decoded.privacyBits); + assertEquals(tokenRefreshRequest.firstLevelHashIdentity.establishedAt, decoded.firstLevelHashIdentity.establishedAt); + assertEquals(tokenRefreshRequest.sourcePublisher.siteId, decoded.sourcePublisher.siteId); Buffer b = Buffer.buffer(encodedBytes); int keyId = b.getInt(tokenVersion == TokenVersion.V2 ? 25 : 2); diff --git a/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java b/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java index 741050dcd..8f32d9625 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java @@ -188,26 +188,26 @@ public void testGenerateAndRefresh(int siteId, TokenVersion tokenVersion) { assertEquals(identityRequest.establishedAt, advertisingTokenRequest.establishedAt); assertEquals(identityRequest.privacyBits, advertisingTokenRequest.privacyBits); - RefreshTokenRequest refreshTokenRequest = tokenEncoder.decodeRefreshToken(identityResponse.getRefreshToken()); - assertEquals(this.now, refreshTokenRequest.createdAt); - assertEquals(this.now.plusSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), refreshTokenRequest.expiresAt); - assertEquals(identityRequest.sourcePublisher.siteId, refreshTokenRequest.sourcePublisher.siteId); - assertIdentityScopeIdentityType(identityRequest.hashedDiiIdentity, refreshTokenRequest.firstLevelHashIdentity); - assertEquals(identityRequest.establishedAt, refreshTokenRequest.firstLevelHashIdentity.establishedAt); + TokenRefreshRequest tokenRefreshRequest = tokenEncoder.decodeRefreshToken(identityResponse.getRefreshToken()); + assertEquals(this.now, tokenRefreshRequest.createdAt); + assertEquals(this.now.plusSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), tokenRefreshRequest.expiresAt); + assertEquals(identityRequest.sourcePublisher.siteId, tokenRefreshRequest.sourcePublisher.siteId); + assertIdentityScopeIdentityType(identityRequest.hashedDiiIdentity, tokenRefreshRequest.firstLevelHashIdentity); + assertEquals(identityRequest.establishedAt, tokenRefreshRequest.firstLevelHashIdentity.establishedAt); final byte[] firstLevelHash = getFirstLevelHash(identityRequest.hashedDiiIdentity.hashedDii, saltProvider.getSnapshot(this.now).getFirstLevelSalt() ); - assertArrayEquals(firstLevelHash, refreshTokenRequest.firstLevelHashIdentity.firstLevelHash); + assertArrayEquals(firstLevelHash, tokenRefreshRequest.firstLevelHashIdentity.firstLevelHash); setNow(Instant.now().plusSeconds(200)); reset(shutdownHandler); - final RefreshResponse refreshResponse = uid2Service.refreshIdentity(refreshTokenRequest); + final TokenRefreshResponse refreshResponse = uid2Service.refreshIdentity(tokenRefreshRequest); verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); assertNotNull(refreshResponse); - assertEquals(RefreshResponse.Status.Refreshed, refreshResponse.getStatus()); + assertEquals(TokenRefreshResponse.Status.Refreshed, refreshResponse.getStatus()); assertNotNull(refreshResponse.getIdentityResponse()); UIDOperatorVerticleTest.validateAdvertisingToken(refreshResponse.getIdentityResponse().getAdvertisingToken(), tokenVersion, IdentityScope.UID2, IdentityType.Email); @@ -221,14 +221,14 @@ public void testGenerateAndRefresh(int siteId, TokenVersion tokenVersion) { advertisingTokenRequest2.rawUidIdentity.rawUid); assertEquals(identityRequest.privacyBits, advertisingTokenRequest2.privacyBits); - RefreshTokenRequest refreshTokenRequest2 = tokenEncoder.decodeRefreshToken(refreshResponse.getIdentityResponse().getRefreshToken()); - assertEquals(this.now, refreshTokenRequest2.createdAt); - assertEquals(this.now.plusSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), refreshTokenRequest2.expiresAt); - assertEquals(refreshTokenRequest.sourcePublisher.siteId, refreshTokenRequest2.sourcePublisher.siteId); - assertIdentityScopeIdentityType(refreshTokenRequest.firstLevelHashIdentity, refreshTokenRequest2.firstLevelHashIdentity); - assertEquals(refreshTokenRequest.firstLevelHashIdentity.establishedAt, refreshTokenRequest2.firstLevelHashIdentity.establishedAt); - assertArrayEquals(refreshTokenRequest.firstLevelHashIdentity.firstLevelHash, refreshTokenRequest2.firstLevelHashIdentity.firstLevelHash); - assertArrayEquals(firstLevelHash, refreshTokenRequest2.firstLevelHashIdentity.firstLevelHash); + TokenRefreshRequest tokenRefreshRequest2 = tokenEncoder.decodeRefreshToken(refreshResponse.getIdentityResponse().getRefreshToken()); + assertEquals(this.now, tokenRefreshRequest2.createdAt); + assertEquals(this.now.plusSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), tokenRefreshRequest2.expiresAt); + assertEquals(tokenRefreshRequest.sourcePublisher.siteId, tokenRefreshRequest2.sourcePublisher.siteId); + assertIdentityScopeIdentityType(tokenRefreshRequest.firstLevelHashIdentity, tokenRefreshRequest2.firstLevelHashIdentity); + assertEquals(tokenRefreshRequest.firstLevelHashIdentity.establishedAt, tokenRefreshRequest2.firstLevelHashIdentity.establishedAt); + assertArrayEquals(tokenRefreshRequest.firstLevelHashIdentity.firstLevelHash, tokenRefreshRequest2.firstLevelHashIdentity.firstLevelHash); + assertArrayEquals(firstLevelHash, tokenRefreshRequest2.firstLevelHashIdentity.firstLevelHash); } @Test @@ -246,8 +246,8 @@ public void testTestOptOutKey_DoNotRespectOptout() { assertNotNull(identityResponse); assertFalse(identityResponse.isOptedOut()); - final RefreshTokenRequest refreshTokenRequest = this.tokenEncoder.decodeRefreshToken(identityResponse.getRefreshToken()); - assertEquals(RefreshResponse.Optout, uid2Service.refreshIdentity(refreshTokenRequest)); + final TokenRefreshRequest tokenRefreshRequest = this.tokenEncoder.decodeRefreshToken(identityResponse.getRefreshToken()); + assertEquals(TokenRefreshResponse.Optout, uid2Service.refreshIdentity(tokenRefreshRequest)); } @Test @@ -280,9 +280,9 @@ public void testTestOptOutKeyIdentityScopeMismatch() { verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); assertNotNull(identityResponse); - final RefreshTokenRequest refreshTokenRequest = this.tokenEncoder.decodeRefreshToken(identityResponse.getRefreshToken()); + final TokenRefreshRequest tokenRefreshRequest = this.tokenEncoder.decodeRefreshToken(identityResponse.getRefreshToken()); reset(shutdownHandler); - assertEquals(RefreshResponse.Invalid, uid2Service.refreshIdentity(refreshTokenRequest)); + assertEquals(TokenRefreshResponse.Invalid, uid2Service.refreshIdentity(tokenRefreshRequest)); verify(shutdownHandler, never()).handleSaltRetrievalResponse(anyBoolean()); } @@ -446,7 +446,7 @@ void testSpecialIdentityOptOutTokenGenerate(TestIdentityInputType type, String i } verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); - assertEquals(identityResponse, IdentityResponse.OptOutIdentityResponse); + assertEquals(identityResponse, IdentityResponse.OptOutResponse); } @ParameterizedTest @@ -510,14 +510,14 @@ void testSpecialIdentityOptOutTokenRefresh(TestIdentityInputType type, String id verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); assertNotNull(identityResponse); - assertNotEquals(IdentityResponse.OptOutIdentityResponse, identityResponse); + assertNotEquals(IdentityResponse.OptOutResponse, identityResponse); // identity has no optout record, ensure refresh still returns optout when(this.optOutStore.getLatestEntry(any())).thenReturn(null); - final RefreshTokenRequest refreshTokenRequest = this.tokenEncoder.decodeRefreshToken(identityResponse.getRefreshToken()); + final TokenRefreshRequest tokenRefreshRequest = this.tokenEncoder.decodeRefreshToken(identityResponse.getRefreshToken()); reset(shutdownHandler); - assertEquals(RefreshResponse.Optout, (scope == IdentityScope.EUID? euidService: uid2Service).refreshIdentity(refreshTokenRequest)); + assertEquals(TokenRefreshResponse.Optout, (scope == IdentityScope.EUID? euidService: uid2Service).refreshIdentity(tokenRefreshRequest)); verify(shutdownHandler, never()).handleSaltRetrievalResponse(anyBoolean()); } @@ -552,14 +552,14 @@ void testSpecialIdentityRefreshOptOutGenerate(TestIdentityInputType type, String verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); assertNotNull(identityResponse); - assertNotEquals(IdentityResponse.OptOutIdentityResponse, identityResponse); + assertNotEquals(IdentityResponse.OptOutResponse, identityResponse); // identity has no optout record, ensure refresh still returns optout when(this.optOutStore.getLatestEntry(any())).thenReturn(null); - final RefreshTokenRequest refreshTokenRequest = this.tokenEncoder.decodeRefreshToken(identityResponse.getRefreshToken()); + final TokenRefreshRequest tokenRefreshRequest = this.tokenEncoder.decodeRefreshToken(identityResponse.getRefreshToken()); reset(shutdownHandler); - assertEquals(RefreshResponse.Optout, (scope == IdentityScope.EUID? euidService: uid2Service).refreshIdentity(refreshTokenRequest)); + assertEquals(TokenRefreshResponse.Optout, (scope == IdentityScope.EUID? euidService: uid2Service).refreshIdentity(tokenRefreshRequest)); verify(shutdownHandler, never()).handleSaltRetrievalResponse(anyBoolean()); } @@ -629,7 +629,7 @@ void testSpecialIdentityValidateGenerate(TestIdentityInputType type, String id, verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); assertNotNull(identityResponse); - assertNotEquals(IdentityResponse.OptOutIdentityResponse, identityResponse); + assertNotEquals(IdentityResponse.OptOutResponse, identityResponse); assertNotNull(advertisingTokenRequest.rawUidIdentity); } @@ -690,14 +690,14 @@ void testNormalIdentityOptIn(TestIdentityInputType type, String id, IdentityScop } verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); - assertNotEquals(identityResponse, IdentityResponse.OptOutIdentityResponse); + assertNotEquals(identityResponse, IdentityResponse.OptOutResponse); assertNotNull(identityResponse); - final RefreshTokenRequest refreshTokenRequest = this.tokenEncoder.decodeRefreshToken(identityResponse.getRefreshToken()); - RefreshResponse refreshResponse = (scope == IdentityScope.EUID? euidService: uid2Service).refreshIdentity(refreshTokenRequest); + final TokenRefreshRequest tokenRefreshRequest = this.tokenEncoder.decodeRefreshToken(identityResponse.getRefreshToken()); + TokenRefreshResponse refreshResponse = (scope == IdentityScope.EUID? euidService: uid2Service).refreshIdentity(tokenRefreshRequest); assertTrue(refreshResponse.isRefreshed()); assertNotNull(refreshResponse.getIdentityResponse()); - assertNotEquals(RefreshResponse.Optout, refreshResponse); + assertNotEquals(TokenRefreshResponse.Optout, refreshResponse); } @ParameterizedTest @@ -756,17 +756,17 @@ void testExpiredSaltsNotifiesShutdownHandler(TestIdentityInputType type, String verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(true); verify(shutdownHandler, never()).handleSaltRetrievalResponse(false); assertNotNull(identityResponse); - assertNotEquals(IdentityResponse.OptOutIdentityResponse, identityResponse); + assertNotEquals(IdentityResponse.OptOutResponse, identityResponse); assertNotNull(advertisingTokenRequest.rawUidIdentity); - final RefreshTokenRequest refreshTokenRequest = this.tokenEncoder.decodeRefreshToken(identityResponse.getRefreshToken()); + final TokenRefreshRequest tokenRefreshRequest = this.tokenEncoder.decodeRefreshToken(identityResponse.getRefreshToken()); reset(shutdownHandler); - RefreshResponse refreshResponse = (scope == IdentityScope.EUID? euidService: uid2Service).refreshIdentity(refreshTokenRequest); + TokenRefreshResponse refreshResponse = (scope == IdentityScope.EUID? euidService: uid2Service).refreshIdentity(tokenRefreshRequest); verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(true); verify(shutdownHandler, never()).handleSaltRetrievalResponse(false); assertTrue(refreshResponse.isRefreshed()); assertNotNull(refreshResponse.getIdentityResponse()); - assertNotEquals(RefreshResponse.Optout, refreshResponse); + assertNotEquals(TokenRefreshResponse.Optout, refreshResponse); final MapRequest mapRequest = new MapRequest( inputVal.toHashedDiiIdentity(scope), diff --git a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java index 6ceeca9be..cdc401ecd 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java @@ -848,11 +848,11 @@ public static void validateAdvertisingToken(String advertisingTokenString, Token } } - RefreshTokenRequest decodeRefreshToken(EncryptedTokenEncoder encoder, String refreshTokenString, IdentityType identityType) { - RefreshTokenRequest refreshTokenRequest = encoder.decodeRefreshToken(refreshTokenString); - assertEquals(getIdentityScope(), refreshTokenRequest.firstLevelHashIdentity.identityScope); - assertEquals(identityType, refreshTokenRequest.firstLevelHashIdentity.identityType); - return refreshTokenRequest; + TokenRefreshRequest decodeRefreshToken(EncryptedTokenEncoder encoder, String refreshTokenString, IdentityType identityType) { + TokenRefreshRequest tokenRefreshRequest = encoder.decodeRefreshToken(refreshTokenString); + assertEquals(getIdentityScope(), tokenRefreshRequest.firstLevelHashIdentity.identityScope); + assertEquals(identityType, tokenRefreshRequest.firstLevelHashIdentity.identityType); + return tokenRefreshRequest; } @ParameterizedTest @@ -1165,14 +1165,14 @@ void tokenGenerateOptOutToken(String policyParameterKey, String identity, Identi EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, body, identityType); - RefreshTokenRequest refreshTokenRequest = encoder.decodeRefreshToken(body.getString("decrypted_refresh_token")); + TokenRefreshRequest tokenRefreshRequest = encoder.decodeRefreshToken(body.getString("decrypted_refresh_token")); final byte[] rawUid = getRawUidFromIdentity(identityType, optOutTokenInput.getNormalized(), firstLevelSalt, rotatingSalt123.getSalt()); final byte[] firstLevelHash = TokenUtils.getFirstLevelHashFromIdentity(optOutTokenInput.getNormalized(), firstLevelSalt); assertArrayEquals(rawUid, advertisingTokenRequest.rawUidIdentity.rawUid); - assertArrayEquals(firstLevelHash, refreshTokenRequest.firstLevelHashIdentity.firstLevelHash); + assertArrayEquals(firstLevelHash, tokenRefreshRequest.firstLevelHashIdentity.firstLevelHash); String advertisingTokenString = body.getString("advertising_token"); final Instant now = Instant.now(); @@ -1225,9 +1225,9 @@ void tokenGenerateForEmail(String apiVersion, Vertx vertx, VertxTestContext test EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, body, IdentityType.Email); - RefreshTokenRequest refreshTokenRequest = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), IdentityType.Email); + TokenRefreshRequest tokenRefreshRequest = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), IdentityType.Email); - assertAdvertisingTokenRefreshTokenRequests(advertisingTokenRequest, refreshTokenRequest, clientSiteId, + assertAdvertisingTokenRefreshTokenRequests(advertisingTokenRequest, tokenRefreshRequest, clientSiteId, getRawUidFromIdentity(IdentityType.Email, emailAddress, firstLevelSalt, rotatingSalt123.getSalt()), PrivacyBits.DEFAULT, body, @@ -1239,15 +1239,15 @@ void tokenGenerateForEmail(String apiVersion, Vertx vertx, VertxTestContext test }); } - public void assertAdvertisingTokenRefreshTokenRequests(AdvertisingTokenRequest advertisingTokenRequest, RefreshTokenRequest refreshTokenRequest, + public void assertAdvertisingTokenRefreshTokenRequests(AdvertisingTokenRequest advertisingTokenRequest, TokenRefreshRequest tokenRefreshRequest, int expectedClientSiteId, byte[] expectedRawUidIdentity, PrivacyBits expectedPrivacyBits, JsonObject identityResponse, byte[] firstLevelHashIdentity) { assertEquals(expectedClientSiteId, advertisingTokenRequest.sourcePublisher.siteId); - assertEquals(expectedClientSiteId, refreshTokenRequest.sourcePublisher.siteId); + assertEquals(expectedClientSiteId, tokenRefreshRequest.sourcePublisher.siteId); assertArrayEquals(expectedRawUidIdentity, advertisingTokenRequest.rawUidIdentity.rawUid); - verifyPrivacyBits(expectedPrivacyBits, advertisingTokenRequest, refreshTokenRequest); - verifyFirstLevelHashIdentityAndEstablishedAt(firstLevelHashIdentity, refreshTokenRequest, identityResponse, advertisingTokenRequest.establishedAt); + verifyPrivacyBits(expectedPrivacyBits, advertisingTokenRequest, tokenRefreshRequest); + verifyFirstLevelHashIdentityAndEstablishedAt(firstLevelHashIdentity, tokenRefreshRequest, identityResponse, advertisingTokenRequest.establishedAt); assertEqualsClose(now.plusMillis(identityExpiresAfter.toMillis()), Instant.ofEpochMilli(identityResponse.getLong("identity_expires")), 10); assertEqualsClose(now.plusMillis(refreshExpiresAfter.toMillis()), Instant.ofEpochMilli(identityResponse.getLong("refresh_expires")), 10); @@ -1255,9 +1255,9 @@ public void assertAdvertisingTokenRefreshTokenRequests(AdvertisingTokenRequest a } public void verifyPrivacyBits(PrivacyBits expectedValue, AdvertisingTokenRequest advertisingTokenRequest, - RefreshTokenRequest refreshTokenRequest) { + TokenRefreshRequest tokenRefreshRequest) { assertEquals(advertisingTokenRequest.privacyBits, expectedValue); - assertEquals(advertisingTokenRequest.privacyBits, refreshTokenRequest.privacyBits); + assertEquals(advertisingTokenRequest.privacyBits, tokenRefreshRequest.privacyBits); } @ParameterizedTest @@ -1282,9 +1282,9 @@ void tokenGenerateForEmailHash(String apiVersion, Vertx vertx, VertxTestContext EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, body, IdentityType.Email); - RefreshTokenRequest refreshTokenRequest = decodeRefreshToken(encoder, apiVersion.equals("v2") ? body.getString("decrypted_refresh_token") : body.getString("refresh_token"), IdentityType.Email); + TokenRefreshRequest tokenRefreshRequest = decodeRefreshToken(encoder, apiVersion.equals("v2") ? body.getString("decrypted_refresh_token") : body.getString("refresh_token"), IdentityType.Email); - assertAdvertisingTokenRefreshTokenRequests(advertisingTokenRequest, refreshTokenRequest, clientSiteId, + assertAdvertisingTokenRefreshTokenRequests(advertisingTokenRequest, tokenRefreshRequest, clientSiteId, getRawUidFromIdentityHash(IdentityType.Email, emailHash, firstLevelSalt, rotatingSalt123.getSalt()), PrivacyBits.DEFAULT, body, @@ -1314,9 +1314,9 @@ void tokenGenerateThenRefresh(String apiVersion, Vertx vertx, VertxTestContext t AdvertisingTokenRequest firstAdvertisingTokenRequest = validateAndGetToken(encoder, bodyJson, IdentityType.Email); - RefreshTokenRequest firstRefreshTokenRequest = decodeRefreshToken(encoder, bodyJson.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), IdentityType.Email); + TokenRefreshRequest firstTokenRefreshRequest = decodeRefreshToken(encoder, bodyJson.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), IdentityType.Email); - assertEquals(firstAdvertisingTokenRequest.establishedAt, firstRefreshTokenRequest.firstLevelHashIdentity.establishedAt); + assertEquals(firstAdvertisingTokenRequest.establishedAt, firstTokenRefreshRequest.firstLevelHashIdentity.establishedAt); when(this.optOutStore.getLatestEntry(any())).thenReturn(null); @@ -1324,7 +1324,7 @@ void tokenGenerateThenRefresh(String apiVersion, Vertx vertx, VertxTestContext t byte[] expectedRawUidIdentity = getRawUidFromIdentity(IdentityType.Email, emailAddress, firstLevelSalt, rotatingSalt123.getSalt()); byte[] expectedFirstLevelHashIdentity = TokenUtils.getFirstLevelHashFromIdentity(emailAddress, firstLevelSalt); - assertAdvertisingTokenRefreshTokenRequests(firstAdvertisingTokenRequest, firstRefreshTokenRequest, clientSiteId, + assertAdvertisingTokenRefreshTokenRequests(firstAdvertisingTokenRequest, firstTokenRefreshRequest, clientSiteId, expectedRawUidIdentity, PrivacyBits.DEFAULT, bodyJson, @@ -1341,12 +1341,12 @@ void tokenGenerateThenRefresh(String apiVersion, Vertx vertx, VertxTestContext t AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, refreshBody, IdentityType.Email); String refreshTokenStringNew = refreshBody.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"); assertNotEquals(genRefreshToken, refreshTokenStringNew); - RefreshTokenRequest refreshTokenRequest = decodeRefreshToken(encoder, refreshTokenStringNew, IdentityType.Email); + TokenRefreshRequest tokenRefreshRequest = decodeRefreshToken(encoder, refreshTokenStringNew, IdentityType.Email); // assert if the ad/refresh tokens from original token/generate is same as the ad/refresh tokens from token/refresh assertAdvertisingTokenRefreshTokenRequests( advertisingTokenRequest, - firstRefreshTokenRequest, + firstTokenRefreshRequest, clientSiteId, expectedRawUidIdentity, PrivacyBits.DEFAULT, @@ -1354,7 +1354,7 @@ void tokenGenerateThenRefresh(String apiVersion, Vertx vertx, VertxTestContext t expectedFirstLevelHashIdentity); assertAdvertisingTokenRefreshTokenRequests( firstAdvertisingTokenRequest, - refreshTokenRequest, + tokenRefreshRequest, clientSiteId, expectedRawUidIdentity, PrivacyBits.DEFAULT, @@ -1362,7 +1362,7 @@ void tokenGenerateThenRefresh(String apiVersion, Vertx vertx, VertxTestContext t expectedFirstLevelHashIdentity); assertAdvertisingTokenRefreshTokenRequests( advertisingTokenRequest, - refreshTokenRequest, + tokenRefreshRequest, clientSiteId, expectedRawUidIdentity, PrivacyBits.DEFAULT, @@ -1420,9 +1420,9 @@ void tokenGenerateThenRefreshSaltsExpired(String apiVersion, Vertx vertx, VertxT String refreshTokenStringNew = refreshBody.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"); assertNotEquals(genRefreshToken, refreshTokenStringNew); - RefreshTokenRequest refreshTokenRequest = decodeRefreshToken(encoder, refreshTokenStringNew, IdentityType.Email); - assertEquals(clientSiteId, refreshTokenRequest.sourcePublisher.siteId); - assertArrayEquals(TokenUtils.getFirstLevelHashFromIdentity(emailAddress, firstLevelSalt), refreshTokenRequest.firstLevelHashIdentity.firstLevelHash); + TokenRefreshRequest tokenRefreshRequest = decodeRefreshToken(encoder, refreshTokenStringNew, IdentityType.Email); + assertEquals(clientSiteId, tokenRefreshRequest.sourcePublisher.siteId); + assertArrayEquals(TokenUtils.getFirstLevelHashFromIdentity(emailAddress, firstLevelSalt), tokenRefreshRequest.firstLevelHashIdentity.firstLevelHash); assertEqualsClose(now.plusMillis(identityExpiresAfter.toMillis()), Instant.ofEpochMilli(refreshBody.getLong("identity_expires")), 10); assertEqualsClose(now.plusMillis(refreshExpiresAfter.toMillis()), Instant.ofEpochMilli(refreshBody.getLong("refresh_expires")), 10); @@ -1593,9 +1593,9 @@ void tokenGenerateUsingCustomSiteKey(String apiVersion, Vertx vertx, VertxTestCo assertEquals(clientSiteId, advertisingTokenRequest.sourcePublisher.siteId); assertArrayEquals(getRawUidFromIdentity(IdentityType.Email, emailAddress, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenRequest.rawUidIdentity.rawUid); - RefreshTokenRequest refreshTokenRequest = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), IdentityType.Email); - assertEquals(clientSiteId, refreshTokenRequest.sourcePublisher.siteId); - assertArrayEquals(TokenUtils.getFirstLevelHashFromIdentity(emailAddress, firstLevelSalt), refreshTokenRequest.firstLevelHashIdentity.firstLevelHash); + TokenRefreshRequest tokenRefreshRequest = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), IdentityType.Email); + assertEquals(clientSiteId, tokenRefreshRequest.sourcePublisher.siteId); + assertArrayEquals(TokenUtils.getFirstLevelHashFromIdentity(emailAddress, firstLevelSalt), tokenRefreshRequest.firstLevelHashIdentity.firstLevelHash); testContext.completeNow(); }); @@ -1632,9 +1632,9 @@ void tokenGenerateSaltsExpired(String apiVersion, Vertx vertx, VertxTestContext assertEquals(clientSiteId, advertisingTokenRequest.sourcePublisher.siteId); assertArrayEquals(getRawUidFromIdentity(IdentityType.Email, emailAddress, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenRequest.rawUidIdentity.rawUid); - RefreshTokenRequest refreshTokenRequest = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), IdentityType.Email); - assertEquals(clientSiteId, refreshTokenRequest.sourcePublisher.siteId); - assertArrayEquals(TokenUtils.getFirstLevelHashFromIdentity(emailAddress, firstLevelSalt), refreshTokenRequest.firstLevelHashIdentity.firstLevelHash); + TokenRefreshRequest tokenRefreshRequest = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), IdentityType.Email); + assertEquals(clientSiteId, tokenRefreshRequest.sourcePublisher.siteId); + assertArrayEquals(TokenUtils.getFirstLevelHashFromIdentity(emailAddress, firstLevelSalt), tokenRefreshRequest.firstLevelHashIdentity.firstLevelHash); assertEqualsClose(now.plusMillis(identityExpiresAfter.toMillis()), Instant.ofEpochMilli(body.getLong("identity_expires")), 10); assertEqualsClose(now.plusMillis(refreshExpiresAfter.toMillis()), Instant.ofEpochMilli(body.getLong("refresh_expires")), 10); @@ -2538,9 +2538,9 @@ void tokenGenerateForPhone(String apiVersion, Vertx vertx, VertxTestContext test EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, body, IdentityType.Phone); - RefreshTokenRequest refreshTokenRequest = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), IdentityType.Phone); + TokenRefreshRequest tokenRefreshRequest = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), IdentityType.Phone); - assertAdvertisingTokenRefreshTokenRequests(advertisingTokenRequest, refreshTokenRequest, clientSiteId, + assertAdvertisingTokenRefreshTokenRequests(advertisingTokenRequest, tokenRefreshRequest, clientSiteId, getRawUidFromIdentity(IdentityType.Phone, phone, firstLevelSalt, rotatingSalt123.getSalt()), PrivacyBits.DEFAULT, body, @@ -2551,15 +2551,15 @@ void tokenGenerateForPhone(String apiVersion, Vertx vertx, VertxTestContext test } void verifyFirstLevelHashIdentityAndEstablishedAt(byte[] expectedFirstLevelHash, - RefreshTokenRequest refreshTokenRequest, + TokenRefreshRequest tokenRefreshRequest, JsonObject receivedJsonBody, Instant expectedEstablishedTime) { - assertArrayEquals(expectedFirstLevelHash, refreshTokenRequest.firstLevelHashIdentity.firstLevelHash); - assertEquals(expectedEstablishedTime, refreshTokenRequest.firstLevelHashIdentity.establishedAt); - assertTrue(refreshTokenRequest.firstLevelHashIdentity.establishedAt.toEpochMilli() < receivedJsonBody.getLong("identity_expires") ); - assertTrue(refreshTokenRequest.firstLevelHashIdentity.establishedAt.toEpochMilli() < receivedJsonBody.getLong("refresh_expires") ); - assertTrue(refreshTokenRequest.firstLevelHashIdentity.establishedAt.toEpochMilli() < receivedJsonBody.getLong("refresh_from") ); + assertArrayEquals(expectedFirstLevelHash, tokenRefreshRequest.firstLevelHashIdentity.firstLevelHash); + assertEquals(expectedEstablishedTime, tokenRefreshRequest.firstLevelHashIdentity.establishedAt); + assertTrue(tokenRefreshRequest.firstLevelHashIdentity.establishedAt.toEpochMilli() < receivedJsonBody.getLong("identity_expires") ); + assertTrue(tokenRefreshRequest.firstLevelHashIdentity.establishedAt.toEpochMilli() < receivedJsonBody.getLong("refresh_expires") ); + assertTrue(tokenRefreshRequest.firstLevelHashIdentity.establishedAt.toEpochMilli() < receivedJsonBody.getLong("refresh_from") ); } @ParameterizedTest @@ -2583,9 +2583,9 @@ void tokenGenerateForPhoneHash(String apiVersion, Vertx vertx, VertxTestContext EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, body, IdentityType.Phone); - RefreshTokenRequest refreshTokenRequest = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), IdentityType.Phone); + TokenRefreshRequest tokenRefreshRequest = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), IdentityType.Phone); - assertAdvertisingTokenRefreshTokenRequests(advertisingTokenRequest, refreshTokenRequest, clientSiteId, + assertAdvertisingTokenRefreshTokenRequests(advertisingTokenRequest, tokenRefreshRequest, clientSiteId, getRawUidFromIdentity(IdentityType.Phone, phone, firstLevelSalt, rotatingSalt123.getSalt()), PrivacyBits.DEFAULT, body, @@ -2614,14 +2614,14 @@ void tokenGenerateThenRefreshForPhone(String apiVersion, Vertx vertx, VertxTestC AdvertisingTokenRequest firstAdvertisingTokenRequest = validateAndGetToken(encoder, bodyJson, IdentityType.Phone); String genRefreshToken = bodyJson.getString("refresh_token"); - RefreshTokenRequest firstRefreshTokenRequest = decodeRefreshToken(encoder, bodyJson.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), IdentityType.Phone); + TokenRefreshRequest firstTokenRefreshRequest = decodeRefreshToken(encoder, bodyJson.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), IdentityType.Phone); when(this.optOutStore.getLatestEntry(any())).thenReturn(null); byte[] expectedRawUidIdentity = getRawUidFromIdentity(IdentityType.Phone, phone, firstLevelSalt, rotatingSalt123.getSalt()); byte[] expectedFirstLevelHashIdentity = TokenUtils.getFirstLevelHashFromIdentity(phone, firstLevelSalt); - assertAdvertisingTokenRefreshTokenRequests(firstAdvertisingTokenRequest, firstRefreshTokenRequest, clientSiteId, + assertAdvertisingTokenRefreshTokenRequests(firstAdvertisingTokenRequest, firstTokenRefreshRequest, clientSiteId, expectedRawUidIdentity, PrivacyBits.DEFAULT, bodyJson, @@ -2637,12 +2637,12 @@ void tokenGenerateThenRefreshForPhone(String apiVersion, Vertx vertx, VertxTestC AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, refreshBody, IdentityType.Phone); String refreshTokenStringNew = refreshBody.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"); assertNotEquals(genRefreshToken, refreshTokenStringNew); - RefreshTokenRequest refreshTokenRequest = decodeRefreshToken(encoder, refreshTokenStringNew, IdentityType.Phone); + TokenRefreshRequest tokenRefreshRequest = decodeRefreshToken(encoder, refreshTokenStringNew, IdentityType.Phone); // assert if the ad/refresh tokens from original token/generate is same as the ad/refresh tokens from token/refresh assertAdvertisingTokenRefreshTokenRequests( advertisingTokenRequest, - firstRefreshTokenRequest, + firstTokenRefreshRequest, clientSiteId, expectedRawUidIdentity, PrivacyBits.DEFAULT, @@ -2650,7 +2650,7 @@ void tokenGenerateThenRefreshForPhone(String apiVersion, Vertx vertx, VertxTestC expectedFirstLevelHashIdentity); assertAdvertisingTokenRefreshTokenRequests( firstAdvertisingTokenRequest, - refreshTokenRequest, + tokenRefreshRequest, clientSiteId, expectedRawUidIdentity, PrivacyBits.DEFAULT, @@ -2658,7 +2658,7 @@ void tokenGenerateThenRefreshForPhone(String apiVersion, Vertx vertx, VertxTestC expectedFirstLevelHashIdentity); assertAdvertisingTokenRefreshTokenRequests( advertisingTokenRequest, - refreshTokenRequest, + tokenRefreshRequest, clientSiteId, expectedRawUidIdentity, PrivacyBits.DEFAULT, @@ -4086,9 +4086,9 @@ void cstgUserOptsOutAfterTokenGenerate(String id, IdentityType identityType, Ver final JsonObject genBody = response.getJsonObject("body"); final AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, genBody, identityType); - final RefreshTokenRequest refreshTokenRequest = decodeRefreshToken(encoder, decodeV2RefreshToken(response), identityType); + final TokenRefreshRequest tokenRefreshRequest = decodeRefreshToken(encoder, decodeV2RefreshToken(response), identityType); - assertAreClientSideGeneratedTokens(advertisingTokenRequest, refreshTokenRequest, clientSideTokenGenerateSiteId, identityType, id); + assertAreClientSideGeneratedTokens(advertisingTokenRequest, tokenRefreshRequest, clientSideTokenGenerateSiteId, identityType, id); // When we refresh the token the user has opted out. when(optOutStore.getLatestEntry(any(FirstLevelHashIdentity.class))) @@ -4151,7 +4151,7 @@ void cstgSuccessForBothOptedAndNonOptedOutTest(boolean optOutExpected, String id AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, genBody, identityType); - RefreshTokenRequest refreshTokenRequest = decodeRefreshToken(encoder, genBody.getString("decrypted_refresh_token"), identityType); + TokenRefreshRequest tokenRefreshRequest = decodeRefreshToken(encoder, genBody.getString("decrypted_refresh_token"), identityType); @@ -4162,13 +4162,13 @@ void cstgSuccessForBothOptedAndNonOptedOutTest(boolean optOutExpected, String id expectedPrivacyBits.setLegacyBit(); expectedPrivacyBits.setClientSideTokenGenerate(); - assertAdvertisingTokenRefreshTokenRequests(advertisingTokenRequest, refreshTokenRequest, + assertAdvertisingTokenRefreshTokenRequests(advertisingTokenRequest, tokenRefreshRequest, clientSideTokenGenerateSiteId, expectedRawUidIdentity, expectedPrivacyBits, genBody, expectedFirstLevelHashIdentity); - assertAreClientSideGeneratedTokens(advertisingTokenRequest, refreshTokenRequest, clientSideTokenGenerateSiteId, identityType, id); + assertAreClientSideGeneratedTokens(advertisingTokenRequest, tokenRefreshRequest, clientSideTokenGenerateSiteId, identityType, id); assertEqualsClose(Instant.now().plusMillis(identityExpiresAfter.toMillis()), Instant.ofEpochMilli(genBody.getLong("identity_expires")), 10); assertEqualsClose(Instant.now().plusMillis(refreshExpiresAfter.toMillis()), Instant.ofEpochMilli(genBody.getLong("refresh_expires")), 10); assertEqualsClose(Instant.now().plusMillis(refreshIdentityAfter.toMillis()), Instant.ofEpochMilli(genBody.getLong("refresh_from")), 10); @@ -4193,7 +4193,7 @@ void cstgSuccessForBothOptedAndNonOptedOutTest(boolean optOutExpected, String id String refreshTokenStringNew = refreshBody.getString("decrypted_refresh_token"); assertNotEquals(genRefreshToken, refreshTokenStringNew); - RefreshTokenRequest refreshTokenAfterRefreshSource = decodeRefreshToken(encoder, refreshTokenStringNew, identityType); + TokenRefreshRequest refreshTokenAfterRefreshSource = decodeRefreshToken(encoder, refreshTokenStringNew, identityType); assertAdvertisingTokenRefreshTokenRequests(adTokenFromRefresh, refreshTokenAfterRefreshSource, clientSideTokenGenerateSiteId, @@ -4295,18 +4295,18 @@ void cstgInvalidInput(String identityType, String rawUID, Vertx vertx, VertxTest }); } - private void assertAreClientSideGeneratedTokens(AdvertisingTokenRequest advertisingTokenRequest, RefreshTokenRequest refreshTokenRequest, int siteId, IdentityType identityType, String identity) { + private void assertAreClientSideGeneratedTokens(AdvertisingTokenRequest advertisingTokenRequest, TokenRefreshRequest tokenRefreshRequest, int siteId, IdentityType identityType, String identity) { assertAreClientSideGeneratedTokens(advertisingTokenRequest, - refreshTokenRequest, + tokenRefreshRequest, siteId, identityType, identity, false); } - private void assertAreClientSideGeneratedTokens(AdvertisingTokenRequest advertisingTokenRequest, RefreshTokenRequest refreshTokenRequest, int siteId, IdentityType identityType, String identity, boolean expectedOptOut) { + private void assertAreClientSideGeneratedTokens(AdvertisingTokenRequest advertisingTokenRequest, TokenRefreshRequest tokenRefreshRequest, int siteId, IdentityType identityType, String identity, boolean expectedOptOut) { final PrivacyBits advertisingTokenPrivacyBits = advertisingTokenRequest.privacyBits; - final PrivacyBits refreshTokenPrivacyBits = refreshTokenRequest.privacyBits; + final PrivacyBits refreshTokenPrivacyBits = tokenRefreshRequest.privacyBits; final byte[] rawUid = getRawUidFromIdentity(identityType, identity, @@ -4323,10 +4323,10 @@ private void assertAreClientSideGeneratedTokens(AdvertisingTokenRequest advertis () -> assertEquals(expectedOptOut, refreshTokenPrivacyBits.isClientSideTokenOptedOut(), "Refresh token privacy bits CSTG optout flag is incorrect"), () -> assertEquals(siteId, advertisingTokenRequest.sourcePublisher.siteId, "Advertising token site ID is incorrect"), - () -> assertEquals(siteId, refreshTokenRequest.sourcePublisher.siteId, "Refresh token site ID is incorrect"), + () -> assertEquals(siteId, tokenRefreshRequest.sourcePublisher.siteId, "Refresh token site ID is incorrect"), () -> assertArrayEquals(rawUid, advertisingTokenRequest.rawUidIdentity.rawUid, "Advertising token ID is incorrect"), - () -> assertArrayEquals(firstLevelHash, refreshTokenRequest.firstLevelHashIdentity.firstLevelHash, "Refresh token ID is incorrect") + () -> assertArrayEquals(firstLevelHash, tokenRefreshRequest.firstLevelHashIdentity.firstLevelHash, "Refresh token ID is incorrect") ); } diff --git a/src/test/java/com/uid2/operator/benchmark/TokenEndecBenchmark.java b/src/test/java/com/uid2/operator/benchmark/TokenEndecBenchmark.java index 14a650047..5f43c9661 100644 --- a/src/test/java/com/uid2/operator/benchmark/TokenEndecBenchmark.java +++ b/src/test/java/com/uid2/operator/benchmark/TokenEndecBenchmark.java @@ -8,7 +8,6 @@ import org.openjdk.jmh.annotations.BenchmarkMode; import org.openjdk.jmh.annotations.Mode; -import java.time.Instant; import java.util.ArrayList; import java.util.List; @@ -59,7 +58,7 @@ public IdentityResponse TokenGenerationBenchmark() { @Benchmark @BenchmarkMode(Mode.Throughput) - public RefreshResponse TokenRefreshBenchmark() { + public TokenRefreshResponse TokenRefreshBenchmark() { return uidService.refreshIdentity( encoder.decodeRefreshToken( generatedTokens[(idx++) & 65535].getRefreshToken())); diff --git a/src/test/java/com/uid2/operator/utilTests/IdentityResponseTest.java b/src/test/java/com/uid2/operator/utilTests/IdentityResponseTest.java index c8ff6feed..ae44a0b94 100644 --- a/src/test/java/com/uid2/operator/utilTests/IdentityResponseTest.java +++ b/src/test/java/com/uid2/operator/utilTests/IdentityResponseTest.java @@ -15,8 +15,8 @@ public class IdentityResponseTest { @Test public void doIdentityResponseTest() throws NoSuchAlgorithmException { - assertEquals(IdentityResponse.OptOutIdentityResponse.getAdvertisingToken(), ""); - assertTrue(IdentityResponse.OptOutIdentityResponse.isOptedOut()); + assertEquals(IdentityResponse.OptOutResponse.getAdvertisingToken(), ""); + assertTrue(IdentityResponse.OptOutResponse.isOptedOut()); IdentityResponse nullAdTokenValue = new IdentityResponse(null, TokenVersion.V4, "refreshToken", null,null,null); assertTrue(nullAdTokenValue.isOptedOut()); From 242b08d037d6e949529e640ddb39ef72a4067121 Mon Sep 17 00:00:00 2001 From: Xu Yang <58192524+clarkxuyang@users.noreply.github.com> Date: Thu, 5 Dec 2024 09:42:38 -0800 Subject: [PATCH 147/431] xuy-UID2-4341-change-log-level Refine response logging (#1183) Refine response logging, client error, invalid http origin, and invalid app name responses are logged at info level --- .../uid2/operator/service/JsonParseUtils.java | 2 +- .../uid2/operator/service/ResponseUtil.java | 91 +++++++++---------- .../operator/vertx/UIDOperatorVerticle.java | 70 +++++++------- .../uid2/operator/vertx/V2PayloadHandler.java | 10 +- .../operator/service/ResponseUtilTest.java | 41 +++++---- 5 files changed, 107 insertions(+), 107 deletions(-) diff --git a/src/main/java/com/uid2/operator/service/JsonParseUtils.java b/src/main/java/com/uid2/operator/service/JsonParseUtils.java index 8860c6fd9..4255b799f 100644 --- a/src/main/java/com/uid2/operator/service/JsonParseUtils.java +++ b/src/main/java/com/uid2/operator/service/JsonParseUtils.java @@ -10,7 +10,7 @@ public static JsonArray parseArray(JsonObject object, String key, RoutingContext try { outArray = object.getJsonArray(key); } catch (ClassCastException e) { - ResponseUtil.ClientError(rc, String.format("%s must be an array", key)); + ResponseUtil.LogInfoAndSend400Response(rc, String.format("%s must be an array", key)); return null; } return outArray; diff --git a/src/main/java/com/uid2/operator/service/ResponseUtil.java b/src/main/java/com/uid2/operator/service/ResponseUtil.java index 5f59eab96..a1842c275 100644 --- a/src/main/java/com/uid2/operator/service/ResponseUtil.java +++ b/src/main/java/com/uid2/operator/service/ResponseUtil.java @@ -1,7 +1,6 @@ package com.uid2.operator.service; import com.uid2.operator.monitoring.TokenResponseStatsCollector; -import com.uid2.operator.vertx.UIDOperatorVerticle; import com.uid2.shared.model.TokenVersion; import com.uid2.shared.store.ISiteStore; import io.vertx.core.http.HttpHeaders; @@ -64,19 +63,28 @@ public static void SuccessV2(RoutingContext rc, Object body) { rc.data().put("response", json); } - public static void ClientError(RoutingContext rc, String message) { - Warning(ResponseStatus.ClientError, 400, rc, message); + public static void LogInfoAndSend400Response(RoutingContext rc, String message) { + LogInfoAndSendResponse(ResponseStatus.ClientError, 400, rc, message); } public static void SendClientErrorResponseAndRecordStats(String errorStatus, int statusCode, RoutingContext rc, String message, Integer siteId, TokenResponseStatsCollector.Endpoint endpoint, TokenResponseStatsCollector.ResponseStatus responseStatus, ISiteStore siteProvider, TokenResponseStatsCollector.PlatformType platformType) { - Warning(errorStatus, statusCode, rc, message); + if (ResponseStatus.ClientError.equals(errorStatus) || + ResponseStatus.InvalidAppName.equals(errorStatus) || + ResponseStatus.InvalidHttpOrigin.equals(errorStatus)) + { + LogInfoAndSendResponse(errorStatus, statusCode, rc, message); + } + else { + LogWarningAndSendResponse(errorStatus, statusCode, rc, message); + } + recordTokenResponseStats(siteId, endpoint, responseStatus, siteProvider, null, platformType); } public static void SendServerErrorResponseAndRecordStats(RoutingContext rc, String message, Integer siteId, TokenResponseStatsCollector.Endpoint endpoint, TokenResponseStatsCollector.ResponseStatus responseStatus, ISiteStore siteProvider, Exception exception, TokenResponseStatsCollector.PlatformType platformType) { - Error(ResponseStatus.UnknownError, 500, rc, message, exception); + LogErrorAndSendResponse(ResponseStatus.UnknownError, 500, rc, message, exception); rc.fail(500); recordTokenResponseStats(siteId, endpoint, responseStatus, siteProvider, null, platformType); } @@ -97,62 +105,40 @@ public static JsonObject Response(String status, String message) { return json; } - public static void Error(String errorStatus, int statusCode, RoutingContext rc, String message) { - logError(errorStatus, statusCode, message, new RoutingContextReader(rc), rc.request().remoteAddress().hostAddress()); + public static void LogErrorAndSendResponse(String errorStatus, int statusCode, RoutingContext rc, String message) { + String msg = ComposeMessage(errorStatus, statusCode, message, new RoutingContextReader(rc), rc.request().remoteAddress().hostAddress()); + LOGGER.error(msg); final JsonObject json = Response(errorStatus, message); rc.response().setStatusCode(statusCode).putHeader(HttpHeaders.CONTENT_TYPE, "application/json") .end(json.encode()); } - public static void Error(String errorStatus, int statusCode, RoutingContext rc, String message, Exception exception) { - logError(errorStatus, statusCode, message, new RoutingContextReader(rc), rc.request().remoteAddress().hostAddress(), exception); + public static void LogErrorAndSendResponse(String errorStatus, int statusCode, RoutingContext rc, String message, Exception exception) { + String msg = ComposeMessage(errorStatus, statusCode, message, new RoutingContextReader(rc), rc.request().remoteAddress().hostAddress()); + LOGGER.error(msg, exception); final JsonObject json = Response(errorStatus, message); rc.response().setStatusCode(statusCode).putHeader(HttpHeaders.CONTENT_TYPE, "application/json") .end(json.encode()); } - public static void Warning(String status, int statusCode, RoutingContext rc, String message) { - logWarning(status, statusCode, message, new RoutingContextReader(rc), rc.request().remoteAddress().hostAddress()); + public static void LogInfoAndSendResponse(String status, int statusCode, RoutingContext rc, String message) { + String msg = ComposeMessage(status, statusCode, message, new RoutingContextReader(rc), rc.request().remoteAddress().hostAddress()); + LOGGER.info(msg); final JsonObject json = Response(status, message); rc.response().setStatusCode(statusCode).putHeader(HttpHeaders.CONTENT_TYPE, "application/json") .end(json.encode()); } - private static void logError(String errorStatus, int statusCode, String message, RoutingContextReader contextReader, String clientAddress) { - JsonObject errorJsonObj = JsonObject.of( - "errorStatus", errorStatus, - "contact", contextReader.getContact(), - "siteId", contextReader.getSiteId(), - "statusCode", statusCode, - "clientAddress", clientAddress, - "message", message - ); - final String linkName = contextReader.getLinkName(); - if (!linkName.isBlank()) { - errorJsonObj.put(SecureLinkValidatorService.SERVICE_LINK_NAME, linkName); - } - final String serviceName = contextReader.getServiceName(); - if (!serviceName.isBlank()) { - errorJsonObj.put(SecureLinkValidatorService.SERVICE_NAME, serviceName); - } - LOGGER.error("Error response to http request. " + errorJsonObj.encode()); - } - - private static void logError(String errorStatus, int statusCode, String message, RoutingContextReader contextReader, String clientAddress, Exception exception) { - String errorMessage = "Error response to http request. " + JsonObject.of( - "errorStatus", errorStatus, - "contact", contextReader.getContact(), - "siteId", contextReader.getSiteId(), - "path", contextReader.getPath(), - "statusCode", statusCode, - "clientAddress", clientAddress, - "message", message - ).encode(); - LOGGER.error(errorMessage, exception); + public static void LogWarningAndSendResponse(String status, int statusCode, RoutingContext rc, String message) { + String msg = ComposeMessage(status, statusCode, message, new RoutingContextReader(rc), rc.request().remoteAddress().hostAddress()); + LOGGER.warn(msg); + final JsonObject json = Response(status, message); + rc.response().setStatusCode(statusCode).putHeader(HttpHeaders.CONTENT_TYPE, "application/json") + .end(json.encode()); } - private static void logWarning(String status, int statusCode, String message, RoutingContextReader contextReader, String clientAddress) { - JsonObject warnMessageJsonObject = JsonObject.of( + private static String ComposeMessage(String status, int statusCode, String message, RoutingContextReader contextReader, String clientAddress) { + JsonObject msgJsonObject = JsonObject.of( "errorStatus", status, "contact", contextReader.getContact(), "siteId", contextReader.getSiteId(), @@ -165,14 +151,22 @@ private static void logWarning(String status, int statusCode, String message, Ro final String origin = contextReader.getOrigin(); if (statusCode >= 400 && statusCode < 500) { if (referer != null) { - warnMessageJsonObject.put("referer", referer); + msgJsonObject.put("referer", referer); } if (origin != null) { - warnMessageJsonObject.put("origin", origin); + msgJsonObject.put("origin", origin); } } - String warnMessage = "Warning response to http request. " + warnMessageJsonObject.encode(); - LOGGER.warn(warnMessage); + + final String linkName = contextReader.getLinkName(); + if (!linkName.isBlank()) { + msgJsonObject.put(SecureLinkValidatorService.SERVICE_LINK_NAME, linkName); + } + final String serviceName = contextReader.getServiceName(); + if (!serviceName.isBlank()) { + msgJsonObject.put(SecureLinkValidatorService.SERVICE_NAME, serviceName); + } + return "Response to http request. " + msgJsonObject.encode(); } public static class ResponseStatus { @@ -183,6 +177,7 @@ public static class ResponseStatus { public static final String InvalidToken = "invalid_token"; public static final String ExpiredToken = "expired_token"; public static final String GenericError = "error"; + public static final String InvalidClient = "invalid_client"; public static final String UnknownError = "unknown"; public static final String InsufficientUserConsent = "insufficient_user_consent"; public static final String InvalidHttpOrigin = "invalid_http_origin"; diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index c3784ae38..19b14e927 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -569,7 +569,7 @@ private void handleKeysRequestCommon(RoutingContext rc, Handler onSuc final ClientKey clientKey = AuthMiddleware.getAuthClient(ClientKey.class, rc); final int clientSiteId = clientKey.getSiteId(); if (!clientKey.hasValidSiteId()) { - ResponseUtil.Warning("invalid_client", 401, rc, "Unexpected client site id " + Integer.toString(clientSiteId)); + ResponseUtil.LogWarningAndSendResponse(ResponseStatus.InvalidClient, 401, rc, "Unexpected client site id " + Integer.toString(clientSiteId)); return; } @@ -820,13 +820,13 @@ private void handleTokenRefreshV1(RoutingContext rc) { ResponseUtil.SuccessNoBody(ResponseStatus.OptOut, rc); } else if (!AuthMiddleware.isAuthenticated(rc)) { // unauthenticated clients get a generic error - ResponseUtil.Warning(ResponseStatus.GenericError, 400, rc, "Error refreshing token"); + ResponseUtil.LogWarningAndSendResponse(ResponseStatus.GenericError, 400, rc, "Error refreshing token"); } else if (r.isInvalidToken()) { - ResponseUtil.Warning(ResponseStatus.InvalidToken, 400, rc, "Invalid Token presented " + tokenList.get(0)); + ResponseUtil.LogWarningAndSendResponse(ResponseStatus.InvalidToken, 400, rc, "Invalid Token presented " + tokenList.get(0)); } else if (r.isExpired()) { - ResponseUtil.Warning(ResponseStatus.ExpiredToken, 400, rc, "Expired Token presented"); + ResponseUtil.LogWarningAndSendResponse(ResponseStatus.ExpiredToken, 400, rc, "Expired Token presented"); } else { - ResponseUtil.Error(ResponseStatus.UnknownError, 500, rc, "Unknown State"); + ResponseUtil.LogErrorAndSendResponse(ResponseStatus.UnknownError, 500, rc, "Unknown State"); } } else { ResponseUtil.Success(rc, toJsonV1(r.getTokens())); @@ -852,15 +852,15 @@ private void handleTokenRefreshV2(RoutingContext rc) { ResponseUtil.SuccessNoBodyV2(ResponseStatus.OptOut, rc); } else if (!AuthMiddleware.isAuthenticated(rc)) { // unauthenticated clients get a generic error - ResponseUtil.Warning(ResponseStatus.GenericError, 400, rc, "Error refreshing token"); + ResponseUtil.LogWarningAndSendResponse(ResponseStatus.GenericError, 400, rc, "Error refreshing token"); } else if (r.isInvalidToken()) { - ResponseUtil.Warning(ResponseStatus.InvalidToken, 400, rc, "Invalid Token presented"); + ResponseUtil.LogWarningAndSendResponse(ResponseStatus.InvalidToken, 400, rc, "Invalid Token presented"); } else if (r.isExpired()) { - ResponseUtil.Warning(ResponseStatus.ExpiredToken, 400, rc, "Expired Token presented"); + ResponseUtil.LogWarningAndSendResponse(ResponseStatus.ExpiredToken, 400, rc, "Expired Token presented"); } else if (r.noActiveKey()) { SendServerErrorResponseAndRecordStats(rc, "No active encryption key available", siteId, TokenResponseStatsCollector.Endpoint.RefreshV2, TokenResponseStatsCollector.ResponseStatus.NoActiveKey, siteProvider, new KeyManager.NoActiveKeyException("No active encryption key available"), platformType); } else { - ResponseUtil.Error(ResponseStatus.UnknownError, 500, rc, "Unknown State"); + ResponseUtil.LogErrorAndSendResponse(ResponseStatus.UnknownError, 500, rc, "Unknown State"); } } else { ResponseUtil.SuccessV2(rc, toJsonV1(r.getTokens())); @@ -894,7 +894,7 @@ private void handleTokenValidateV1(RoutingContext rc) { ResponseUtil.Success(rc, Boolean.FALSE); } } catch (ClientInputValidationException cie) { - ResponseUtil.Warning(ResponseStatus.InvalidToken, 400, rc, "Invalid Token presented"); + ResponseUtil.LogWarningAndSendResponse(ResponseStatus.InvalidToken, 400, rc, "Invalid Token presented"); } catch (Exception e) { LOGGER.error("Unknown error while validating token", e); rc.fail(500); @@ -1120,7 +1120,7 @@ private void handleLogoutAsync(RoutingContext rc) { } }); } else { - ResponseUtil.Warning(ResponseStatus.InvalidToken, 400, rc, "Invalid Token presented " + input); + ResponseUtil.LogWarningAndSendResponse(ResponseStatus.InvalidToken, 400, rc, "Invalid Token presented " + input); } } @@ -1143,7 +1143,7 @@ private Future handleLogoutAsyncV2(RoutingContext rc) { }); return promise.future(); } else { - ResponseUtil.Warning(ResponseStatus.InvalidToken, 400, rc, "Invalid Token presented " + input); + ResponseUtil.LogWarningAndSendResponse(ResponseStatus.InvalidToken, 400, rc, "Invalid Token presented " + input); return Future.failedFuture(""); } } @@ -1165,7 +1165,7 @@ private void handleOptOutGet(RoutingContext rc) { rc.fail(500); } } else { - ResponseUtil.Warning(ResponseStatus.InvalidToken, 400, rc, "Invalid Token presented " + input); + ResponseUtil.LogWarningAndSendResponse(ResponseStatus.InvalidToken, 400, rc, "Invalid Token presented " + input); } } @@ -1178,7 +1178,7 @@ private void handleBucketsV1(RoutingContext rc) { sinceTimestamp = ld.toInstant(ZoneOffset.UTC); LOGGER.info(String.format("identity bucket endpoint is called with since_timestamp %s and site id %s", ld, AuthMiddleware.getAuthClient(rc).getSiteId())); } catch (Exception e) { - ResponseUtil.ClientError(rc, "invalid date, must conform to ISO 8601"); + ResponseUtil.LogInfoAndSend400Response(rc, "invalid date, must conform to ISO 8601"); return; } final List modified = this.idService.getModifiedBuckets(sinceTimestamp); @@ -1195,7 +1195,7 @@ private void handleBucketsV1(RoutingContext rc) { ResponseUtil.Success(rc, resp); } } else { - ResponseUtil.ClientError(rc, "missing parameter since_timestamp"); + ResponseUtil.LogInfoAndSend400Response(rc, "missing parameter since_timestamp"); } } @@ -1210,7 +1210,7 @@ private void handleBucketsV2(RoutingContext rc) { sinceTimestamp = ld.toInstant(ZoneOffset.UTC); LOGGER.info(String.format("identity bucket endpoint is called with since_timestamp %s and site id %s", ld, AuthMiddleware.getAuthClient(rc).getSiteId())); } catch (Exception e) { - ResponseUtil.ClientError(rc, "invalid date, must conform to ISO 8601"); + ResponseUtil.LogInfoAndSend400Response(rc, "invalid date, must conform to ISO 8601"); return; } final List modified = this.idService.getModifiedBuckets(sinceTimestamp); @@ -1227,7 +1227,7 @@ private void handleBucketsV2(RoutingContext rc) { ResponseUtil.SuccessV2(rc, resp); } } else { - ResponseUtil.ClientError(rc, "missing parameter since_timestamp"); + ResponseUtil.LogInfoAndSend400Response(rc, "missing parameter since_timestamp"); } } @@ -1245,7 +1245,7 @@ private void handleIdentityMapV1(RoutingContext rc) { jsonObject.put("bucket_id", mappedIdentity.bucketId); ResponseUtil.Success(rc, jsonObject); } catch (Exception e) { - ResponseUtil.Error(ResponseStatus.UnknownError, 500, rc, "Unknown State", e); + ResponseUtil.LogErrorAndSendResponse(ResponseStatus.UnknownError, 500, rc, "Unknown State", e); } } @@ -1359,10 +1359,10 @@ private InputUtil.InputVal getTokenInputV1(RoutingContext rc) { private boolean isTokenInputValid(InputUtil.InputVal input, RoutingContext rc) { if (input == null) { String message = this.phoneSupport ? ERROR_INVALID_INPUT_WITH_PHONE_SUPPORT : ERROR_INVALID_INPUT_EMAIL_MISSING; - ResponseUtil.ClientError(rc, message); + ResponseUtil.LogInfoAndSend400Response(rc, message); return false; } else if (!input.isValid()) { - ResponseUtil.ClientError(rc, "Invalid Identifier"); + ResponseUtil.LogInfoAndSend400Response(rc, "Invalid Identifier"); return false; } return true; @@ -1374,11 +1374,11 @@ private InputUtil.InputVal[] getIdentityBulkInput(RoutingContext rc) { final JsonArray emailHashes = obj.getJsonArray("email_hash"); // FIXME TODO. Avoid Double Iteration. Turn to a decorator pattern if (emails == null && emailHashes == null) { - ResponseUtil.ClientError(rc, ERROR_INVALID_INPUT_EMAIL_MISSING); + ResponseUtil.LogInfoAndSend400Response(rc, ERROR_INVALID_INPUT_EMAIL_MISSING); return null; } else if (emails != null && !emails.isEmpty()) { if (emailHashes != null && !emailHashes.isEmpty()) { - ResponseUtil.ClientError(rc, ERROR_INVALID_INPUT_EMAIL_TWICE); + ResponseUtil.LogInfoAndSend400Response(rc, ERROR_INVALID_INPUT_EMAIL_TWICE); return null; } return createInputList(emails, false); @@ -1391,7 +1391,7 @@ private InputUtil.InputVal[] getIdentityBulkInput(RoutingContext rc) { private InputUtil.InputVal[] getIdentityBulkInputV1(RoutingContext rc) { final JsonObject obj = rc.body().asJsonObject(); if(obj.isEmpty()) { - ResponseUtil.ClientError(rc, ERROR_INVALID_INPUT_WITH_PHONE_SUPPORT); + ResponseUtil.LogInfoAndSend400Response(rc, ERROR_INVALID_INPUT_WITH_PHONE_SUPPORT); return null; } final JsonArray emails = JsonParseUtils.parseArray(obj, "email", rc); @@ -1423,7 +1423,7 @@ private InputUtil.InputVal[] getIdentityBulkInputV1(RoutingContext rc) { } if (validInputs == 0 || nonEmptyInputs > 1) { - ResponseUtil.ClientError(rc, ERROR_INVALID_INPUT_WITH_PHONE_SUPPORT); + ResponseUtil.LogInfoAndSend400Response(rc, ERROR_INVALID_INPUT_WITH_PHONE_SUPPORT); return null; } @@ -1495,7 +1495,7 @@ private void handleIdentityMapBatchV1(RoutingContext rc) { final JsonObject resp = handleIdentityMapCommon(rc, inputList); ResponseUtil.Success(rc, resp); } catch (Exception e) { - ResponseUtil.Error(ResponseStatus.UnknownError, 500, rc, "Unknown error while mapping batched identity", e); + ResponseUtil.LogErrorAndSendResponse(ResponseStatus.UnknownError, 500, rc, "Unknown error while mapping batched identity", e); } } @@ -1504,22 +1504,22 @@ private void handleIdentityMapV2(RoutingContext rc) { final InputUtil.InputVal[] inputList = getIdentityMapV2Input(rc); if (inputList == null) { if (this.phoneSupport) - ResponseUtil.ClientError(rc, ERROR_INVALID_INPUT_WITH_PHONE_SUPPORT); + ResponseUtil.LogInfoAndSend400Response(rc, ERROR_INVALID_INPUT_WITH_PHONE_SUPPORT); else - ResponseUtil.ClientError(rc, ERROR_INVALID_INPUT_EMAIL_MISSING); + ResponseUtil.LogInfoAndSend400Response(rc, ERROR_INVALID_INPUT_EMAIL_MISSING); return; } JsonObject requestJsonObject = (JsonObject) rc.data().get(REQUEST); if (!this.secureLinkValidatorService.validateRequest(rc, requestJsonObject, Role.MAPPER)) { - ResponseUtil.Error(ResponseStatus.Unauthorized, HttpStatus.SC_UNAUTHORIZED, rc, "Invalid link_id"); + ResponseUtil.LogErrorAndSendResponse(ResponseStatus.Unauthorized, HttpStatus.SC_UNAUTHORIZED, rc, "Invalid link_id"); return; } final JsonObject resp = handleIdentityMapCommon(rc, inputList); ResponseUtil.SuccessV2(rc, resp); } catch (Exception e) { - ResponseUtil.Error(ResponseStatus.UnknownError, 500, rc, "Unknown error while mapping identity v2", e); + ResponseUtil.LogErrorAndSendResponse(ResponseStatus.UnknownError, 500, rc, "Unknown error while mapping identity v2", e); } } @@ -1572,11 +1572,11 @@ private void handleIdentityMapBatch(RoutingContext rc) { final JsonArray emails = obj.getJsonArray("email"); final JsonArray emailHashes = obj.getJsonArray("email_hash"); if (emails == null && emailHashes == null) { - ResponseUtil.ClientError(rc, ERROR_INVALID_INPUT_EMAIL_MISSING); + ResponseUtil.LogInfoAndSend400Response(rc, ERROR_INVALID_INPUT_EMAIL_MISSING); return; } else if (emails != null && !emails.isEmpty()) { if (emailHashes != null && !emailHashes.isEmpty()) { - ResponseUtil.ClientError(rc, ERROR_INVALID_INPUT_EMAIL_TWICE); + ResponseUtil.LogInfoAndSend400Response(rc, ERROR_INVALID_INPUT_EMAIL_TWICE); return; } inputList = createInputList(emails, false); @@ -1678,16 +1678,16 @@ private void recordIdentityMapStatsForServiceLinks(RoutingContext rc, String api private List parseOptoutStatusRequestPayload(RoutingContext rc) { final JsonObject requestObj = (JsonObject) rc.data().get("request"); if (requestObj == null) { - ResponseUtil.Error(ResponseStatus.ClientError, HttpStatus.SC_BAD_REQUEST, rc, "Invalid request body"); + ResponseUtil.LogErrorAndSendResponse(ResponseStatus.ClientError, HttpStatus.SC_BAD_REQUEST, rc, "Invalid request body"); return null; } final JsonArray rawUidsJsonArray = requestObj.getJsonArray("advertising_ids"); if (rawUidsJsonArray == null) { - ResponseUtil.Error(ResponseStatus.ClientError, HttpStatus.SC_BAD_REQUEST, rc, "Required Parameter Missing: advertising_ids"); + ResponseUtil.LogErrorAndSendResponse(ResponseStatus.ClientError, HttpStatus.SC_BAD_REQUEST, rc, "Required Parameter Missing: advertising_ids"); return null; } if (rawUidsJsonArray.size() > optOutStatusMaxRequestSize) { - ResponseUtil.Error(ResponseStatus.ClientError, HttpStatus.SC_BAD_REQUEST, rc, "Request payload is too large"); + ResponseUtil.LogErrorAndSendResponse(ResponseStatus.ClientError, HttpStatus.SC_BAD_REQUEST, rc, "Request payload is too large"); return null; } List rawUID2sInputList = new ArrayList<>(rawUidsJsonArray.size()); @@ -1721,7 +1721,7 @@ private void handleOptoutStatus(RoutingContext rc) { ResponseUtil.SuccessV2(rc, bodyJsonObj); recordOptOutStatusEndpointStats(rc, rawUID2sInput.size(), optedOutJsonArray.size()); } catch (Exception e) { - ResponseUtil.Error(ResponseStatus.UnknownError, 500, rc, + ResponseUtil.LogErrorAndSendResponse(ResponseStatus.UnknownError, 500, rc, "Unknown error while getting optout status", e); } } diff --git a/src/main/java/com/uid2/operator/vertx/V2PayloadHandler.java b/src/main/java/com/uid2/operator/vertx/V2PayloadHandler.java index 07ab3ff58..10627ab10 100644 --- a/src/main/java/com/uid2/operator/vertx/V2PayloadHandler.java +++ b/src/main/java/com/uid2/operator/vertx/V2PayloadHandler.java @@ -51,7 +51,7 @@ public void handle(RoutingContext rc, Handler apiHandler) { V2RequestUtil.V2Request request = V2RequestUtil.parseRequest(rc.body().asString(), AuthMiddleware.getAuthClient(ClientKey.class, rc), new InstantClock()); if (!request.isValid()) { - ResponseUtil.ClientError(rc, request.errorMessage); + ResponseUtil.LogInfoAndSend400Response(rc, request.errorMessage); return; } rc.data().put("request", request.payload); @@ -69,7 +69,7 @@ public void handleAsync(RoutingContext rc, Function apiH V2RequestUtil.V2Request request = V2RequestUtil.parseRequest(rc.body().asString(), AuthMiddleware.getAuthClient(ClientKey.class, rc), new InstantClock()); if (!request.isValid()) { - ResponseUtil.ClientError(rc, request.errorMessage); + ResponseUtil.LogInfoAndSend400Response(rc, request.errorMessage); return; } rc.data().put("request", request.payload); @@ -110,7 +110,7 @@ public void handleTokenGenerate(RoutingContext rc, Handler apiHa } catch (Exception ex){ LOGGER.error("Failed to generate token", ex); - ResponseUtil.Error(ResponseUtil.ResponseStatus.GenericError, 500, rc, ""); + ResponseUtil.LogErrorAndSendResponse(ResponseUtil.ResponseStatus.GenericError, 500, rc, ""); } } @@ -163,7 +163,7 @@ public void handleTokenRefresh(RoutingContext rc, Handler apiHan } catch (Exception ex){ LOGGER.error("Failed to refresh token", ex); - ResponseUtil.Error(ResponseUtil.ResponseStatus.GenericError, 500, rc, ""); + ResponseUtil.LogErrorAndSendResponse(ResponseUtil.ResponseStatus.GenericError, 500, rc, ""); } } @@ -199,7 +199,7 @@ private void handleResponse(RoutingContext rc, V2RequestUtil.V2Request request) writeResponse(rc, request.nonce, respJson, request.encryptionKey); } catch (Exception ex) { LOGGER.error("Failed to generate response", ex); - ResponseUtil.Error(ResponseUtil.ResponseStatus.GenericError, 500, rc, ""); + ResponseUtil.LogErrorAndSendResponse(ResponseUtil.ResponseStatus.GenericError, 500, rc, ""); } } } diff --git a/src/test/java/com/uid2/operator/service/ResponseUtilTest.java b/src/test/java/com/uid2/operator/service/ResponseUtilTest.java index 103dd73a6..77f848cc3 100644 --- a/src/test/java/com/uid2/operator/service/ResponseUtilTest.java +++ b/src/test/java/com/uid2/operator/service/ResponseUtilTest.java @@ -42,12 +42,13 @@ void tearDown() { @Test void logsErrorWithNoExtraDetails() { - ResponseUtil.Error("Some error status", 500, rc, "Some error message"); + ResponseUtil.LogErrorAndSendResponse("Some error status", 500, rc, "Some error message"); - String expected = "Error response to http request. {" + + String expected = "Response to http request. {" + "\"errorStatus\":\"Some error status\"," + "\"contact\":null," + "\"siteId\":null," + + "\"path\":null," + "\"statusCode\":500," + "\"clientAddress\":null," + "\"message\":\"Some error message\"" + @@ -65,12 +66,13 @@ void logsErrorWithExtraDetailsFromAuthorizable() { when(mockAuthorizable.getSiteId()).thenReturn(10); when(rc.data().get("api-client")).thenReturn(mockAuthorizable); - ResponseUtil.Error("Some error status", 500, rc, "Some error message"); + ResponseUtil.LogErrorAndSendResponse("Some error status", 500, rc, "Some error message"); - String expected = "Error response to http request. {" + + String expected = "Response to http request. {" + "\"errorStatus\":\"Some error status\"," + "\"contact\":\"Test Contract\"," + "\"siteId\":10," + + "\"path\":null," + "\"statusCode\":500," + "\"clientAddress\":null," + "\"message\":\"Some error message\"" + @@ -83,12 +85,13 @@ void logsErrorWithExtraDetailsFromAuthorizable() { void logsErrorWithSiteIdFromContext() { when(rc.get(Const.RoutingContextData.SiteId)).thenReturn(20); - ResponseUtil.Error("Some error status", 500, rc, "Some error message"); + ResponseUtil.LogErrorAndSendResponse("Some error status", 500, rc, "Some error message"); - String expected = "Error response to http request. {" + + String expected = "Response to http request. {" + "\"errorStatus\":\"Some error status\"," + "\"contact\":null," + "\"siteId\":20," + + "\"path\":null," + "\"statusCode\":500," + "\"clientAddress\":null," + "\"message\":\"Some error message\"" + @@ -104,12 +107,13 @@ void logsErrorWithClientAddress() { when(rc.request().remoteAddress()).thenReturn(socket); - ResponseUtil.Error("Some error status", 500, rc, "Some error message"); + ResponseUtil.LogErrorAndSendResponse("Some error status", 500, rc, "Some error message"); - String expected = "Error response to http request. {" + + String expected = "Response to http request. {" + "\"errorStatus\":\"Some error status\"," + "\"contact\":null," + "\"siteId\":null," + + "\"path\":null," + "\"statusCode\":500," + "\"clientAddress\":\"192.168.10.10\"," + "\"message\":\"Some error message\"" + @@ -124,11 +128,12 @@ void logsErrorWithServiceAndServiceLinkNames() { when(rc1.get(SecureLinkValidatorService.SERVICE_LINK_NAME, "")).thenReturn("TestLink1"); when(rc1.get(SecureLinkValidatorService.SERVICE_NAME, "")).thenReturn("TestService1"); - ResponseUtil.Error("Some error status", 500, rc1, "Some error message"); - String expected = "Error response to http request. {" + + ResponseUtil.LogErrorAndSendResponse("Some error status", 500, rc1, "Some error message"); + String expected = "Response to http request. {" + "\"errorStatus\":\"Some error status\"," + "\"contact\":null," + "\"siteId\":null," + + "\"path\":null," + "\"statusCode\":500," + "\"clientAddress\":null," + "\"message\":\"Some error message\"," + @@ -144,9 +149,9 @@ void logsWarningWithOrigin() { when(request.getHeader("origin")).thenReturn("testOriginHeader"); when(rc.request()).thenReturn(request); - ResponseUtil.Warning("Some error status", 400, rc, "Some error message"); + ResponseUtil.LogInfoAndSendResponse("Some error status", 400, rc, "Some error message"); - String expected = "Warning response to http request. {" + + String expected = "Response to http request. {" + "\"errorStatus\":\"Some error status\"," + "\"contact\":null," + "\"siteId\":null," + @@ -165,9 +170,9 @@ void logsWarningWithOriginNull() { when(request.getHeader("origin")).thenReturn(null); when(rc.request()).thenReturn(request); - ResponseUtil.Warning("Some error status", 400, rc, "Some error message"); + ResponseUtil.LogWarningAndSendResponse("Some error status", 400, rc, "Some error message"); - String expected = "Warning response to http request. {" + + String expected = "Response to http request. {" + "\"errorStatus\":\"Some error status\"," + "\"contact\":null," + "\"siteId\":null," + @@ -185,9 +190,9 @@ void logsWarningWithReferer() { when(request.getHeader("referer")).thenReturn("testRefererHeader"); when(rc.request()).thenReturn(request); - ResponseUtil.Warning("Some error status", 400, rc, "Some error message"); + ResponseUtil.LogInfoAndSendResponse("Some error status", 400, rc, "Some error message"); - String expected = "Warning response to http request. {" + + String expected = "Response to http request. {" + "\"errorStatus\":\"Some error status\"," + "\"contact\":null," + "\"siteId\":null," + @@ -206,9 +211,9 @@ void logsWarningWithRefererNull() { when(request.getHeader("referer")).thenReturn(null); when(rc.request()).thenReturn(request); - ResponseUtil.Warning("Some error status", 400, rc, "Some error message"); + ResponseUtil.LogWarningAndSendResponse("Some error status", 400, rc, "Some error message"); - String expected = "Warning response to http request. {" + + String expected = "Response to http request. {" + "\"errorStatus\":\"Some error status\"," + "\"contact\":null," + "\"siteId\":null," + From 4a1af7de1ed658dc0ea6f76ba98db5cf400de8d7 Mon Sep 17 00:00:00 2001 From: Xu Yang <58192524+clarkxuyang@users.noreply.github.com> Date: Thu, 5 Dec 2024 10:02:34 -0800 Subject: [PATCH 148/431] Xuy UI d2 2578 update share lib to add site name to metrics (#1179) * Xuy UI d2 2578 Update share uid2 lib to 8.0.9, which added site name into metrics --- pom.xml | 2 +- src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index f5fb728c5..2450f1529 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ 2.1.0 2.1.0 2.1.0 - 8.0.6 + 8.0.9 ${project.version} 21 21 diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index 19b14e927..20472f9ec 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -221,7 +221,7 @@ private Router createRoutesSetup() throws IOException { final Router router = Router.router(vertx); router.allowForward(AllowForwardHeaders.X_FORWARD); - router.route().handler(new RequestCapturingHandler()); + router.route().handler(new RequestCapturingHandler(siteProvider)); router.route().handler(new ClientVersionCapturingHandler("static/js", "*.js", clientKeyProvider)); router.route().handler(CorsHandler.create() .addRelativeOrigin(".*.") From 06d49c8840e9e4dbed150a7cc1b2d0138f8dd06f Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Thu, 5 Dec 2024 21:13:19 +0000 Subject: [PATCH 149/431] [CI Pipeline] Released Minor version: 5.43.0 --- pom.xml | 2 +- version.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 2450f1529..d817ec3dd 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.42.17 + 5.43.0 UTF-8 diff --git a/version.json b/version.json index ad32c1ceb..61e9e46d1 100644 --- a/version.json +++ b/version.json @@ -1 +1 @@ -{ "$schema": "https://raw.githubusercontent.com/dotnet/Nerdbank.GitVersioning/master/src/NerdBank.GitVersioning/version.schema.json", "version": "5.42", "publicReleaseRefSpec": [ "^refs/heads/master$", "^refs/heads/v\\d+(?:\\.\\d+)?$" ], "cloudBuild": { "setVersionVariables": true, "buildNumber": { "enabled": true, "includeCommitId": { "when": "always" } } } } +{ "$schema": "https://raw.githubusercontent.com/dotnet/Nerdbank.GitVersioning/master/src/NerdBank.GitVersioning/version.schema.json", "version": "5.43", "publicReleaseRefSpec": [ "^refs/heads/master$", "^refs/heads/v\\d+(?:\\.\\d+)?$" ], "cloudBuild": { "setVersionVariables": true, "buildNumber": { "enabled": true, "includeCommitId": { "when": "always" } } } } From 5922a29bb6d580ba1c60c6384f6b750714efa14d Mon Sep 17 00:00:00 2001 From: Cody Constine Date: Fri, 6 Dec 2024 16:06:44 -0700 Subject: [PATCH 150/431] Updated the API readed to make simplier --- .../java/com/uid2/operator/reader/ApiStoreReader.java | 7 +++---- .../reader/RotatingCloudEncryptionKeyApiProvider.java | 2 +- src/test/java/com/uid2/operator/ApiStoreReaderTest.java | 6 +++--- .../RotatingCloudEncryptionKeyApiProviderTest.java | 8 ++++---- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/uid2/operator/reader/ApiStoreReader.java b/src/main/java/com/uid2/operator/reader/ApiStoreReader.java index d3d75c5d5..78937e16c 100644 --- a/src/main/java/com/uid2/operator/reader/ApiStoreReader.java +++ b/src/main/java/com/uid2/operator/reader/ApiStoreReader.java @@ -21,14 +21,13 @@ public ApiStoreReader(DownloadCloudStorage fileStreamProvider, StoreScope scope, super(fileStreamProvider, scope, parser, dataTypeName); } - @Override - public long loadContent(JsonObject contents, String dataType) throws Exception { + public long loadContent(JsonObject contents) throws Exception { if (contents == null) { - throw new IllegalArgumentException(String.format("No contents provided for loading data type %s, cannot load content", dataType)); + throw new IllegalArgumentException(String.format("No contents provided for loading data type %s, cannot load content", dataTypeName)); } try { - JsonArray dataArray = contents.getJsonArray(dataType); + JsonArray dataArray = contents.getJsonArray(dataTypeName); if (dataArray == null) { throw new IllegalArgumentException("No array found in the contents"); } diff --git a/src/main/java/com/uid2/operator/reader/RotatingCloudEncryptionKeyApiProvider.java b/src/main/java/com/uid2/operator/reader/RotatingCloudEncryptionKeyApiProvider.java index 594cfb7b6..4c91ea9dc 100644 --- a/src/main/java/com/uid2/operator/reader/RotatingCloudEncryptionKeyApiProvider.java +++ b/src/main/java/com/uid2/operator/reader/RotatingCloudEncryptionKeyApiProvider.java @@ -35,7 +35,7 @@ public CloudPath getMetadataPath() { @Override public long loadContent(JsonObject metadata) throws Exception { - return apiStoreReader.loadContent(metadata, "cloudEncryptionKeys"); + return apiStoreReader.loadContent(metadata); } @Override diff --git a/src/test/java/com/uid2/operator/ApiStoreReaderTest.java b/src/test/java/com/uid2/operator/ApiStoreReaderTest.java index 460d57b8d..7cd55fd1c 100644 --- a/src/test/java/com/uid2/operator/ApiStoreReaderTest.java +++ b/src/test/java/com/uid2/operator/ApiStoreReaderTest.java @@ -51,7 +51,7 @@ void getMetadataPathReturnsPathFromScope() { @Test void loadContentThrowsExceptionWhenContentsAreNull() { - assertThatThrownBy(() -> reader.loadContent(null, dataType)) + assertThatThrownBy(() -> reader.loadContent(null)) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("No contents provided for loading data type"); } @@ -59,7 +59,7 @@ void loadContentThrowsExceptionWhenContentsAreNull() { @Test void loadContentThrowsExceptionWhenArrayNotFound() { JsonObject contents = new JsonObject(); - assertThatThrownBy(() -> reader.loadContent(contents, dataType)) + assertThatThrownBy(() -> reader.loadContent(contents)) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("No array found in the contents"); } @@ -73,7 +73,7 @@ void loadContentSuccessfullyLoadsData() throws Exception { when(mockParser.deserialize(any(InputStream.class))) .thenReturn(new ParsingResult<>(expectedData, expectedData.size())); - long count = reader.loadContent(contents, dataType); + long count = reader.loadContent(contents); assertThat(count).isEqualTo(2); assertThat(reader.getSnapshot()).isEqualTo(expectedData); diff --git a/src/test/java/com/uid2/operator/RotatingCloudEncryptionKeyApiProviderTest.java b/src/test/java/com/uid2/operator/RotatingCloudEncryptionKeyApiProviderTest.java index e07489be5..04f1ad0aa 100644 --- a/src/test/java/com/uid2/operator/RotatingCloudEncryptionKeyApiProviderTest.java +++ b/src/test/java/com/uid2/operator/RotatingCloudEncryptionKeyApiProviderTest.java @@ -61,11 +61,11 @@ void testGetMetadataPath() { @Test void testLoadContentWithMetadata() throws Exception { JsonObject metadata = new JsonObject(); - when(mockApiStoreReader.loadContent(metadata, "cloudEncryptionKeys")).thenReturn(1L); + when(mockApiStoreReader.loadContent(metadata)).thenReturn(1L); long version = rotatingCloudEncryptionKeyApiProvider.loadContent(metadata); assertEquals(1L, version); - verify(mockApiStoreReader).loadContent(metadata, "cloudEncryptionKeys"); + verify(mockApiStoreReader).loadContent(metadata); } @Test @@ -94,10 +94,10 @@ void testGetAllWithNullSnapshot() { void testLoadContent() throws Exception { JsonObject metadata = new JsonObject().put("version", 1L); when(mockApiStoreReader.getMetadata()).thenReturn(metadata); - when(mockApiStoreReader.loadContent(metadata, "cloudEncryptionKeys")).thenReturn(1L); + when(mockApiStoreReader.loadContent(metadata)).thenReturn(1L); rotatingCloudEncryptionKeyApiProvider.loadContent(); verify(mockApiStoreReader).getMetadata(); - verify(mockApiStoreReader).loadContent(metadata, "cloudEncryptionKeys"); + verify(mockApiStoreReader).loadContent(metadata); } } \ No newline at end of file From ea0b247c1b6a2cd4c996d5a3a505acc01fa1f52a Mon Sep 17 00:00:00 2001 From: Sunny Wu Date: Mon, 9 Dec 2024 17:14:59 +1100 Subject: [PATCH 151/431] Renamed IdentityRequest/IdentityResponse to TokenGenerateRequest/Response --- ...Request.java => TokenGenerateRequest.java} | 6 +- ...sponse.java => TokenGenerateResponse.java} | 8 +- .../operator/model/TokenRefreshResponse.java | 28 +-- .../service/EncryptedTokenEncoder.java | 4 +- .../uid2/operator/service/ITokenEncoder.java | 4 +- .../operator/service/IUIDOperatorService.java | 2 +- .../operator/service/UIDOperatorService.java | 12 +- .../operator/vertx/UIDOperatorVerticle.java | 30 ++-- .../uid2/operator/UIDOperatorServiceTest.java | 166 +++++++++--------- .../benchmark/TokenEndecBenchmark.java | 14 +- ...st.java => TokenGenerateResponseTest.java} | 12 +- 11 files changed, 143 insertions(+), 143 deletions(-) rename src/main/java/com/uid2/operator/model/{IdentityRequest.java => TokenGenerateRequest.java} (92%) rename src/main/java/com/uid2/operator/model/{IdentityResponse.java => TokenGenerateResponse.java} (85%) rename src/test/java/com/uid2/operator/utilTests/{IdentityResponseTest.java => TokenGenerateResponseTest.java} (78%) diff --git a/src/main/java/com/uid2/operator/model/IdentityRequest.java b/src/main/java/com/uid2/operator/model/TokenGenerateRequest.java similarity index 92% rename from src/main/java/com/uid2/operator/model/IdentityRequest.java rename to src/main/java/com/uid2/operator/model/TokenGenerateRequest.java index c1cca1d75..f482e8d21 100644 --- a/src/main/java/com/uid2/operator/model/IdentityRequest.java +++ b/src/main/java/com/uid2/operator/model/TokenGenerateRequest.java @@ -5,7 +5,7 @@ import java.time.Instant; -public final class IdentityRequest { +public final class TokenGenerateRequest { public final SourcePublisher sourcePublisher; public final HashedDiiIdentity hashedDiiIdentity; public final OptoutCheckPolicy optoutCheckPolicy; @@ -13,7 +13,7 @@ public final class IdentityRequest { public final PrivacyBits privacyBits; public final Instant establishedAt; - public IdentityRequest( + public TokenGenerateRequest( SourcePublisher sourcePublisher, HashedDiiIdentity hashedDiiIdentity, OptoutCheckPolicy tokenGeneratePolicy, @@ -26,7 +26,7 @@ public IdentityRequest( this.establishedAt = establishedAt; } - public IdentityRequest( + public TokenGenerateRequest( SourcePublisher sourcePublisher, HashedDiiIdentity hashedDiiIdentity, OptoutCheckPolicy tokenGeneratePolicy) { diff --git a/src/main/java/com/uid2/operator/model/IdentityResponse.java b/src/main/java/com/uid2/operator/model/TokenGenerateResponse.java similarity index 85% rename from src/main/java/com/uid2/operator/model/IdentityResponse.java rename to src/main/java/com/uid2/operator/model/TokenGenerateResponse.java index 746294064..8c7d273f9 100644 --- a/src/main/java/com/uid2/operator/model/IdentityResponse.java +++ b/src/main/java/com/uid2/operator/model/TokenGenerateResponse.java @@ -8,8 +8,8 @@ // this defines all the fields for the response of the /token/generate and /client/generate endpoints before they are // jsonified // todo: can be converted to record later -public class IdentityResponse { - public static final IdentityResponse OptOutResponse = new IdentityResponse("", null, "", Instant.EPOCH, Instant.EPOCH, Instant.EPOCH); +public class TokenGenerateResponse { + public static final TokenGenerateResponse OptOutResponse = new TokenGenerateResponse("", null, "", Instant.EPOCH, Instant.EPOCH, Instant.EPOCH); //aka UID token private final String advertisingToken; @@ -20,8 +20,8 @@ public class IdentityResponse { private final Instant refreshExpires; private final Instant refreshFrom; - public IdentityResponse(String advertisingToken, TokenVersion advertisingTokenVersion, String refreshToken, - Instant identityExpires, Instant refreshExpires, Instant refreshFrom) { + public TokenGenerateResponse(String advertisingToken, TokenVersion advertisingTokenVersion, String refreshToken, + Instant identityExpires, Instant refreshExpires, Instant refreshFrom) { this.advertisingToken = advertisingToken; this.advertisingTokenVersion = advertisingTokenVersion; this.refreshToken = refreshToken; diff --git a/src/main/java/com/uid2/operator/model/TokenRefreshResponse.java b/src/main/java/com/uid2/operator/model/TokenRefreshResponse.java index 5a52fa571..40e5d73c9 100644 --- a/src/main/java/com/uid2/operator/model/TokenRefreshResponse.java +++ b/src/main/java/com/uid2/operator/model/TokenRefreshResponse.java @@ -5,37 +5,37 @@ public class TokenRefreshResponse { public static final TokenRefreshResponse Invalid = new TokenRefreshResponse(Status.Invalid, - IdentityResponse.OptOutResponse); - public static final TokenRefreshResponse Optout = new TokenRefreshResponse(Status.Optout, IdentityResponse.OptOutResponse); - public static final TokenRefreshResponse Expired = new TokenRefreshResponse(Status.Expired, IdentityResponse.OptOutResponse); - public static final TokenRefreshResponse Deprecated = new TokenRefreshResponse(Status.Deprecated, IdentityResponse.OptOutResponse); - public static final TokenRefreshResponse NoActiveKey = new TokenRefreshResponse(Status.NoActiveKey, IdentityResponse.OptOutResponse); + TokenGenerateResponse.OptOutResponse); + public static final TokenRefreshResponse Optout = new TokenRefreshResponse(Status.Optout, TokenGenerateResponse.OptOutResponse); + public static final TokenRefreshResponse Expired = new TokenRefreshResponse(Status.Expired, TokenGenerateResponse.OptOutResponse); + public static final TokenRefreshResponse Deprecated = new TokenRefreshResponse(Status.Deprecated, TokenGenerateResponse.OptOutResponse); + public static final TokenRefreshResponse NoActiveKey = new TokenRefreshResponse(Status.NoActiveKey, TokenGenerateResponse.OptOutResponse); private final Status status; - private final IdentityResponse identityResponse; + private final TokenGenerateResponse tokenGenerateResponse; private final Duration durationSinceLastRefresh; private final boolean isCstg; - private TokenRefreshResponse(Status status, IdentityResponse identityResponse, Duration durationSinceLastRefresh, boolean isCstg) { + private TokenRefreshResponse(Status status, TokenGenerateResponse tokenGenerateResponse, Duration durationSinceLastRefresh, boolean isCstg) { this.status = status; - this.identityResponse = identityResponse; + this.tokenGenerateResponse = tokenGenerateResponse; this.durationSinceLastRefresh = durationSinceLastRefresh; this.isCstg = isCstg; } - private TokenRefreshResponse(Status status, IdentityResponse identityResponse) { - this(status, identityResponse, null, false); + private TokenRefreshResponse(Status status, TokenGenerateResponse tokenGenerateResponse) { + this(status, tokenGenerateResponse, null, false); } - public static TokenRefreshResponse createRefreshedResponse(IdentityResponse identityResponse, Duration durationSinceLastRefresh, boolean isCstg) { - return new TokenRefreshResponse(Status.Refreshed, identityResponse, durationSinceLastRefresh, isCstg); + public static TokenRefreshResponse createRefreshedResponse(TokenGenerateResponse tokenGenerateResponse, Duration durationSinceLastRefresh, boolean isCstg) { + return new TokenRefreshResponse(Status.Refreshed, tokenGenerateResponse, durationSinceLastRefresh, isCstg); } public Status getStatus() { return status; } - public IdentityResponse getIdentityResponse() { - return identityResponse; + public TokenGenerateResponse getIdentityResponse() { + return tokenGenerateResponse; } public Duration getDurationSinceLastRefresh() { diff --git a/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java b/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java index 26d019795..06570391e 100644 --- a/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java +++ b/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java @@ -351,10 +351,10 @@ public static String bytesToBase64Token(byte[] advertisingTokenBytes, TokenVersi } @Override - public IdentityResponse encodeIntoIdentityResponse(AdvertisingTokenRequest advertisingTokenRequest, TokenRefreshRequest tokenRefreshRequest, Instant refreshFrom, Instant asOf) { + public TokenGenerateResponse encodeIntoIdentityResponse(AdvertisingTokenRequest advertisingTokenRequest, TokenRefreshRequest tokenRefreshRequest, Instant refreshFrom, Instant asOf) { final String advertisingToken = generateAdvertisingTokenString(advertisingTokenRequest, asOf); final String refreshToken = generateRefreshTokenString(tokenRefreshRequest, asOf); - return new IdentityResponse( + return new TokenGenerateResponse( advertisingToken, advertisingTokenRequest.version, refreshToken, diff --git a/src/main/java/com/uid2/operator/service/ITokenEncoder.java b/src/main/java/com/uid2/operator/service/ITokenEncoder.java index c8beddc85..73fe2e70c 100644 --- a/src/main/java/com/uid2/operator/service/ITokenEncoder.java +++ b/src/main/java/com/uid2/operator/service/ITokenEncoder.java @@ -1,13 +1,13 @@ package com.uid2.operator.service; import com.uid2.operator.model.AdvertisingTokenRequest; -import com.uid2.operator.model.IdentityResponse; +import com.uid2.operator.model.TokenGenerateResponse; import com.uid2.operator.model.TokenRefreshRequest; import java.time.Instant; public interface ITokenEncoder { - IdentityResponse encodeIntoIdentityResponse(AdvertisingTokenRequest advertisingTokenRequest, TokenRefreshRequest tokenRefreshRequest, Instant refreshFrom, Instant asOf); + TokenGenerateResponse encodeIntoIdentityResponse(AdvertisingTokenRequest advertisingTokenRequest, TokenRefreshRequest tokenRefreshRequest, Instant refreshFrom, Instant asOf); AdvertisingTokenRequest decodeAdvertisingToken(String base64String); diff --git a/src/main/java/com/uid2/operator/service/IUIDOperatorService.java b/src/main/java/com/uid2/operator/service/IUIDOperatorService.java index 879d784bd..6fec9288e 100644 --- a/src/main/java/com/uid2/operator/service/IUIDOperatorService.java +++ b/src/main/java/com/uid2/operator/service/IUIDOperatorService.java @@ -12,7 +12,7 @@ public interface IUIDOperatorService { - IdentityResponse generateIdentity(IdentityRequest request); + TokenGenerateResponse generateIdentity(TokenGenerateRequest request); TokenRefreshResponse refreshIdentity(TokenRefreshRequest input); diff --git a/src/main/java/com/uid2/operator/service/UIDOperatorService.java b/src/main/java/com/uid2/operator/service/UIDOperatorService.java index ee9685662..b4914c770 100644 --- a/src/main/java/com/uid2/operator/service/UIDOperatorService.java +++ b/src/main/java/com/uid2/operator/service/UIDOperatorService.java @@ -105,7 +105,7 @@ public UIDOperatorService(JsonObject config, IOptOutStore optOutStore, ISaltProv } @Override - public IdentityResponse generateIdentity(IdentityRequest request) { + public TokenGenerateResponse generateIdentity(TokenGenerateRequest request) { final Instant now = EncodingUtils.NowUTCMillis(this.clock); final byte[] firstLevelHash = getFirstLevelHash(request.hashedDiiIdentity.hashedDii, now); final FirstLevelHashIdentity firstLevelHashIdentity = new FirstLevelHashIdentity( @@ -113,7 +113,7 @@ public IdentityResponse generateIdentity(IdentityRequest request) { request.establishedAt); if (request.shouldCheckOptOut() && getGlobalOptOutResult(firstLevelHashIdentity, false).isOptedOut()) { - return IdentityResponse.OptOutResponse; + return TokenGenerateResponse.OptOutResponse; } else { return generateIdentity(request.sourcePublisher, firstLevelHashIdentity, request.privacyBits); } @@ -145,11 +145,11 @@ public TokenRefreshResponse refreshIdentity(TokenRefreshRequest input) { final Duration durationSinceLastRefresh = Duration.between(input.createdAt, now); if (!optedOut) { - IdentityResponse identityResponse = this.generateIdentity(input.sourcePublisher, + TokenGenerateResponse tokenGenerateResponse = this.generateIdentity(input.sourcePublisher, input.firstLevelHashIdentity, input.privacyBits); - return TokenRefreshResponse.createRefreshedResponse(identityResponse, durationSinceLastRefresh, isCstg); + return TokenRefreshResponse.createRefreshedResponse(tokenGenerateResponse, durationSinceLastRefresh, isCstg); } else { return TokenRefreshResponse.Optout; } @@ -250,8 +250,8 @@ private RawUidResponse generateRawUid(FirstLevelHashIdentity firstLevelHashIdent rotatingSalt.getHashedId()); } - private IdentityResponse generateIdentity(SourcePublisher sourcePublisher, - FirstLevelHashIdentity firstLevelHashIdentity, PrivacyBits privacyBits) { + private TokenGenerateResponse generateIdentity(SourcePublisher sourcePublisher, + FirstLevelHashIdentity firstLevelHashIdentity, PrivacyBits privacyBits) { final Instant nowUtc = EncodingUtils.NowUTCMillis(this.clock); final RawUidResponse rawUidResponse = generateRawUid(firstLevelHashIdentity, nowUtc); diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index ecfb9302e..8175cd25e 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -2,7 +2,7 @@ import com.uid2.operator.Const; import com.uid2.operator.model.*; -import com.uid2.operator.model.IdentityResponse; +import com.uid2.operator.model.TokenGenerateResponse; import com.uid2.operator.model.IdentityScope; import com.uid2.operator.model.userIdentity.HashedDiiIdentity; import com.uid2.operator.monitoring.IStatsCollectorQueue; @@ -464,10 +464,10 @@ else if(emailHash != null) { privacyBits.setLegacyBit(); privacyBits.setClientSideTokenGenerate(); - IdentityResponse identityResponse; + TokenGenerateResponse tokenGenerateResponse; try { - identityResponse = this.idService.generateIdentity( - new IdentityRequest( + tokenGenerateResponse = this.idService.generateIdentity( + new TokenGenerateRequest( new SourcePublisher(clientSideKeypair.getSiteId()), input.toHashedDiiIdentity(this.identityScope), OptoutCheckPolicy.RespectOptOut, privacyBits, Instant.now())); @@ -478,12 +478,12 @@ else if(emailHash != null) { JsonObject response; TokenResponseStatsCollector.ResponseStatus responseStatus = TokenResponseStatsCollector.ResponseStatus.Success; - if (identityResponse.isOptedOut()) { + if (tokenGenerateResponse.isOptedOut()) { response = ResponseUtil.SuccessNoBodyV2(ResponseStatus.OptOut); responseStatus = TokenResponseStatsCollector.ResponseStatus.OptOut; } else { //user not opted out and already generated valid identity token - response = ResponseUtil.SuccessV2(identityResponse.toJsonV1()); + response = ResponseUtil.SuccessV2(tokenGenerateResponse.toJsonV1()); } //if returning an optout token or a successful identity token created originally if (responseStatus == TokenResponseStatsCollector.ResponseStatus.Success) { @@ -491,7 +491,7 @@ else if(emailHash != null) { } final byte[] encryptedResponse = AesGcm.encrypt(response.toBuffer().getBytes(), sharedSecret); rc.response().setStatusCode(200).end(Buffer.buffer(Unpooled.wrappedBuffer(Base64.getEncoder().encode(encryptedResponse)))); - recordTokenResponseStats(clientSideKeypair.getSiteId(), TokenResponseStatsCollector.Endpoint.ClientSideTokenGenerateV2, responseStatus, siteProvider, identityResponse.getAdvertisingTokenVersion(), platformType); + recordTokenResponseStats(clientSideKeypair.getSiteId(), TokenResponseStatsCollector.Endpoint.ClientSideTokenGenerateV2, responseStatus, siteProvider, tokenGenerateResponse.getAdvertisingTokenVersion(), platformType); } private boolean hasValidOriginOrAppName(RoutingContext rc, CstgRequest request, ClientSideKeypair keypair, TokenResponseStatsCollector.PlatformType platformType) { @@ -939,8 +939,8 @@ private void handleTokenGenerateV1(RoutingContext rc) { final InputUtil.InputVal input = this.phoneSupport ? this.getTokenInputV1(rc) : this.getTokenInput(rc); platformType = getPlatformType(rc); if (isTokenInputValid(input, rc)) { - final IdentityResponse response = this.idService.generateIdentity( - new IdentityRequest( + final TokenGenerateResponse response = this.idService.generateIdentity( + new TokenGenerateRequest( new SourcePublisher(siteId), input.toHashedDiiIdentity(this.identityScope), OptoutCheckPolicy.defaultPolicy())); @@ -991,8 +991,8 @@ private void handleTokenGenerateV2(RoutingContext rc) { return; } - final IdentityResponse response = this.idService.generateIdentity( - new IdentityRequest( + final TokenGenerateResponse response = this.idService.generateIdentity( + new TokenGenerateRequest( new SourcePublisher(siteId), input.toHashedDiiIdentity(this.identityScope), OptoutCheckPolicy.respectOptOut())); @@ -1007,8 +1007,8 @@ private void handleTokenGenerateV2(RoutingContext rc) { pb.setLegacyBit(); pb.setClientSideTokenGenerateOptout(); - final IdentityResponse optOutTokens = this.idService.generateIdentity( - new IdentityRequest( + final TokenGenerateResponse optOutTokens = this.idService.generateIdentity( + new TokenGenerateRequest( new SourcePublisher(siteId), optOutTokenInput.toHashedDiiIdentity(this.identityScope), OptoutCheckPolicy.DoNotRespect, pb, Instant.now())); @@ -1047,8 +1047,8 @@ else if (!input.isValid()) { try { siteId = AuthMiddleware.getAuthClient(rc).getSiteId(); - final IdentityResponse response = this.idService.generateIdentity( - new IdentityRequest( + final TokenGenerateResponse response = this.idService.generateIdentity( + new TokenGenerateRequest( new SourcePublisher(siteId), input.toHashedDiiIdentity(this.identityScope), OptoutCheckPolicy.defaultPolicy())); diff --git a/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java b/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java index 8f32d9625..4ec802028 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java @@ -169,33 +169,33 @@ private void assertIdentityScopeIdentityType(UserIdentity expctedValues, @ParameterizedTest @CsvSource({"123, V2","127, V4","128, V4"}) //site id 127 and 128 is for testing "site_ids_using_v4_tokens" public void testGenerateAndRefresh(int siteId, TokenVersion tokenVersion) { - final IdentityRequest identityRequest = new IdentityRequest( + final TokenGenerateRequest tokenGenerateRequest = new TokenGenerateRequest( new SourcePublisher(siteId, 124, 125), createHashedDiiIdentity("test-email-hash", IdentityScope.UID2, IdentityType.Email), OptoutCheckPolicy.DoNotRespect, PrivacyBits.fromInt(0), this.now.minusSeconds(234) ); - final IdentityResponse identityResponse = uid2Service.generateIdentity(identityRequest); + final TokenGenerateResponse tokenGenerateResponse = uid2Service.generateIdentity(tokenGenerateRequest); verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); - assertNotNull(identityResponse); + assertNotNull(tokenGenerateResponse); - UIDOperatorVerticleTest.validateAdvertisingToken(identityResponse.getAdvertisingToken(), tokenVersion, IdentityScope.UID2, IdentityType.Email); - AdvertisingTokenRequest advertisingTokenRequest = tokenEncoder.decodeAdvertisingToken(identityResponse.getAdvertisingToken()); + UIDOperatorVerticleTest.validateAdvertisingToken(tokenGenerateResponse.getAdvertisingToken(), tokenVersion, IdentityScope.UID2, IdentityType.Email); + AdvertisingTokenRequest advertisingTokenRequest = tokenEncoder.decodeAdvertisingToken(tokenGenerateResponse.getAdvertisingToken()); assertEquals(this.now.plusSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS), advertisingTokenRequest.expiresAt); - assertEquals(identityRequest.sourcePublisher.siteId, advertisingTokenRequest.sourcePublisher.siteId); - assertIdentityScopeIdentityType(identityRequest.hashedDiiIdentity, advertisingTokenRequest.rawUidIdentity); - assertEquals(identityRequest.establishedAt, advertisingTokenRequest.establishedAt); - assertEquals(identityRequest.privacyBits, advertisingTokenRequest.privacyBits); + assertEquals(tokenGenerateRequest.sourcePublisher.siteId, advertisingTokenRequest.sourcePublisher.siteId); + assertIdentityScopeIdentityType(tokenGenerateRequest.hashedDiiIdentity, advertisingTokenRequest.rawUidIdentity); + assertEquals(tokenGenerateRequest.establishedAt, advertisingTokenRequest.establishedAt); + assertEquals(tokenGenerateRequest.privacyBits, advertisingTokenRequest.privacyBits); - TokenRefreshRequest tokenRefreshRequest = tokenEncoder.decodeRefreshToken(identityResponse.getRefreshToken()); + TokenRefreshRequest tokenRefreshRequest = tokenEncoder.decodeRefreshToken(tokenGenerateResponse.getRefreshToken()); assertEquals(this.now, tokenRefreshRequest.createdAt); assertEquals(this.now.plusSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), tokenRefreshRequest.expiresAt); - assertEquals(identityRequest.sourcePublisher.siteId, tokenRefreshRequest.sourcePublisher.siteId); - assertIdentityScopeIdentityType(identityRequest.hashedDiiIdentity, tokenRefreshRequest.firstLevelHashIdentity); - assertEquals(identityRequest.establishedAt, tokenRefreshRequest.firstLevelHashIdentity.establishedAt); + assertEquals(tokenGenerateRequest.sourcePublisher.siteId, tokenRefreshRequest.sourcePublisher.siteId); + assertIdentityScopeIdentityType(tokenGenerateRequest.hashedDiiIdentity, tokenRefreshRequest.firstLevelHashIdentity); + assertEquals(tokenGenerateRequest.establishedAt, tokenRefreshRequest.firstLevelHashIdentity.establishedAt); - final byte[] firstLevelHash = getFirstLevelHash(identityRequest.hashedDiiIdentity.hashedDii, + final byte[] firstLevelHash = getFirstLevelHash(tokenGenerateRequest.hashedDiiIdentity.hashedDii, saltProvider.getSnapshot(this.now).getFirstLevelSalt() ); assertArrayEquals(firstLevelHash, tokenRefreshRequest.firstLevelHashIdentity.firstLevelHash); @@ -219,7 +219,7 @@ public void testGenerateAndRefresh(int siteId, TokenVersion tokenVersion) { assertEquals(advertisingTokenRequest.establishedAt, advertisingTokenRequest2.establishedAt); assertArrayEquals(advertisingTokenRequest.rawUidIdentity.rawUid, advertisingTokenRequest2.rawUidIdentity.rawUid); - assertEquals(identityRequest.privacyBits, advertisingTokenRequest2.privacyBits); + assertEquals(tokenGenerateRequest.privacyBits, advertisingTokenRequest2.privacyBits); TokenRefreshRequest tokenRefreshRequest2 = tokenEncoder.decodeRefreshToken(refreshResponse.getIdentityResponse().getRefreshToken()); assertEquals(this.now, tokenRefreshRequest2.createdAt); @@ -235,18 +235,18 @@ public void testGenerateAndRefresh(int siteId, TokenVersion tokenVersion) { public void testTestOptOutKey_DoNotRespectOptout() { final InputUtil.InputVal inputVal = InputUtil.normalizeEmail(IdentityConst.OptOutIdentityForEmail); - final IdentityRequest identityRequest = new IdentityRequest( + final TokenGenerateRequest tokenGenerateRequest = new TokenGenerateRequest( new SourcePublisher(123, 124, 125), inputVal.toHashedDiiIdentity(IdentityScope.UID2), OptoutCheckPolicy.DoNotRespect, PrivacyBits.fromInt(0), this.now ); - final IdentityResponse identityResponse = uid2Service.generateIdentity(identityRequest); + final TokenGenerateResponse tokenGenerateResponse = uid2Service.generateIdentity(tokenGenerateRequest); verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); - assertNotNull(identityResponse); - assertFalse(identityResponse.isOptedOut()); + assertNotNull(tokenGenerateResponse); + assertFalse(tokenGenerateResponse.isOptedOut()); - final TokenRefreshRequest tokenRefreshRequest = this.tokenEncoder.decodeRefreshToken(identityResponse.getRefreshToken()); + final TokenRefreshRequest tokenRefreshRequest = this.tokenEncoder.decodeRefreshToken(tokenGenerateResponse.getRefreshToken()); assertEquals(TokenRefreshResponse.Optout, uid2Service.refreshIdentity(tokenRefreshRequest)); } @@ -254,13 +254,13 @@ public void testTestOptOutKey_DoNotRespectOptout() { public void testTestOptOutKey_RespectOptout() { final InputUtil.InputVal inputVal = InputUtil.normalizeEmail(IdentityConst.OptOutIdentityForEmail); - final IdentityRequest identityRequest = new IdentityRequest( + final TokenGenerateRequest tokenGenerateRequest = new TokenGenerateRequest( new SourcePublisher(123, 124, 125), inputVal.toHashedDiiIdentity(IdentityScope.UID2), OptoutCheckPolicy.RespectOptOut, PrivacyBits.fromInt(0), this.now ); - final IdentityResponse identityResponse = uid2Service.generateIdentity(identityRequest); - assertTrue(identityResponse.isOptedOut()); + final TokenGenerateResponse tokenGenerateResponse = uid2Service.generateIdentity(tokenGenerateRequest); + assertTrue(tokenGenerateResponse.isOptedOut()); verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); } @@ -270,17 +270,17 @@ public void testTestOptOutKeyIdentityScopeMismatch() { final String email = "optout@example.com"; final InputUtil.InputVal inputVal = InputUtil.normalizeEmail(email); - final IdentityRequest identityRequest = new IdentityRequest( + final TokenGenerateRequest tokenGenerateRequest = new TokenGenerateRequest( new SourcePublisher(123, 124, 125), inputVal.toHashedDiiIdentity(IdentityScope.EUID), OptoutCheckPolicy.DoNotRespect, PrivacyBits.fromInt(0), this.now ); - final IdentityResponse identityResponse = euidService.generateIdentity(identityRequest); + final TokenGenerateResponse tokenGenerateResponse = euidService.generateIdentity(tokenGenerateRequest); verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); - assertNotNull(identityResponse); + assertNotNull(tokenGenerateResponse); - final TokenRefreshRequest tokenRefreshRequest = this.tokenEncoder.decodeRefreshToken(identityResponse.getRefreshToken()); + final TokenRefreshRequest tokenRefreshRequest = this.tokenEncoder.decodeRefreshToken(tokenGenerateResponse.getRefreshToken()); reset(shutdownHandler); assertEquals(TokenRefreshResponse.Invalid, uid2Service.refreshIdentity(tokenRefreshRequest)); verify(shutdownHandler, never()).handleSaltRetrievalResponse(anyBoolean()); @@ -295,13 +295,13 @@ public void testGenerateTokenForOptOutUser(IdentityType type, String id, Identit final HashedDiiIdentity hashedDiiIdentity = createHashedDiiIdentity(TokenUtils.getIdentityHashString(id), scope, type); - final IdentityRequest identityRequestForceGenerate = new IdentityRequest( + final TokenGenerateRequest tokenGenerateRequestForceGenerate = new TokenGenerateRequest( new SourcePublisher(123, 124, 125), hashedDiiIdentity, OptoutCheckPolicy.DoNotRespect, PrivacyBits.fromInt(0), this.now.minusSeconds(234)); - final IdentityRequest identityRequestRespectOptOut = new IdentityRequest( + final TokenGenerateRequest tokenGenerateRequestRespectOptOut = new TokenGenerateRequest( new SourcePublisher(123, 124, 125), hashedDiiIdentity, OptoutCheckPolicy.RespectOptOut, PrivacyBits.fromInt(0), @@ -311,32 +311,32 @@ public void testGenerateTokenForOptOutUser(IdentityType type, String id, Identit when(optOutStore.getLatestEntry(any(FirstLevelHashIdentity.class))) .thenReturn(Instant.now().minus(1, ChronoUnit.HOURS)); - final IdentityResponse identityResponse; + final TokenGenerateResponse tokenGenerateResponse; final AdvertisingTokenRequest advertisingTokenRequest; - final IdentityResponse identityResponseAfterOptOut; + final TokenGenerateResponse tokenGenerateResponseAfterOptOut; if (scope == IdentityScope.UID2) { - identityResponse = uid2Service.generateIdentity(identityRequestForceGenerate); + tokenGenerateResponse = uid2Service.generateIdentity(tokenGenerateRequestForceGenerate); verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); - advertisingTokenRequest = validateAndGetToken(tokenEncoder, identityResponse.getAdvertisingToken(), IdentityScope.UID2, hashedDiiIdentity.identityType, identityRequestRespectOptOut.sourcePublisher.siteId); + advertisingTokenRequest = validateAndGetToken(tokenEncoder, tokenGenerateResponse.getAdvertisingToken(), IdentityScope.UID2, hashedDiiIdentity.identityType, tokenGenerateRequestRespectOptOut.sourcePublisher.siteId); reset(shutdownHandler); - identityResponseAfterOptOut = uid2Service.generateIdentity(identityRequestRespectOptOut); + tokenGenerateResponseAfterOptOut = uid2Service.generateIdentity(tokenGenerateRequestRespectOptOut); } else { - identityResponse = euidService.generateIdentity(identityRequestForceGenerate); + tokenGenerateResponse = euidService.generateIdentity(tokenGenerateRequestForceGenerate); verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); - advertisingTokenRequest = validateAndGetToken(tokenEncoder, identityResponse.getAdvertisingToken(), IdentityScope.EUID, hashedDiiIdentity.identityType, identityRequestRespectOptOut.sourcePublisher.siteId); + advertisingTokenRequest = validateAndGetToken(tokenEncoder, tokenGenerateResponse.getAdvertisingToken(), IdentityScope.EUID, hashedDiiIdentity.identityType, tokenGenerateRequestRespectOptOut.sourcePublisher.siteId); reset(shutdownHandler); - identityResponseAfterOptOut = euidService.generateIdentity(identityRequestRespectOptOut); + tokenGenerateResponseAfterOptOut = euidService.generateIdentity(tokenGenerateRequestRespectOptOut); } verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); - assertNotNull(identityResponse); + assertNotNull(tokenGenerateResponse); assertNotNull(advertisingTokenRequest.rawUidIdentity); - assertNotNull(identityResponseAfterOptOut); - assertTrue(identityResponseAfterOptOut.getAdvertisingToken() == null || identityResponseAfterOptOut.getAdvertisingToken().isEmpty()); - assertTrue(identityResponseAfterOptOut.isOptedOut()); + assertNotNull(tokenGenerateResponseAfterOptOut); + assertTrue(tokenGenerateResponseAfterOptOut.getAdvertisingToken() == null || tokenGenerateResponseAfterOptOut.getAdvertisingToken().isEmpty()); + assertTrue(tokenGenerateResponseAfterOptOut.isOptedOut()); } @ParameterizedTest @@ -428,7 +428,7 @@ private InputUtil.InputVal generateInputVal(TestIdentityInputType type, String i void testSpecialIdentityOptOutTokenGenerate(TestIdentityInputType type, String id, IdentityScope scope) { InputUtil.InputVal inputVal = generateInputVal(type, id); - final IdentityRequest identityRequest = new IdentityRequest( + final TokenGenerateRequest tokenGenerateRequest = new TokenGenerateRequest( new SourcePublisher(123, 124, 125), inputVal.toHashedDiiIdentity(scope), OptoutCheckPolicy.RespectOptOut, PrivacyBits.fromInt(0), this.now @@ -437,16 +437,16 @@ void testSpecialIdentityOptOutTokenGenerate(TestIdentityInputType type, String i // identity has no optout record, ensure generate still returns optout when(this.optOutStore.getLatestEntry(any())).thenReturn(null); - IdentityResponse identityResponse; + TokenGenerateResponse tokenGenerateResponse; if(scope == IdentityScope.EUID) { - identityResponse = euidService.generateIdentity(identityRequest); + tokenGenerateResponse = euidService.generateIdentity(tokenGenerateRequest); } else { - identityResponse = uid2Service.generateIdentity(identityRequest); + tokenGenerateResponse = uid2Service.generateIdentity(tokenGenerateRequest); } verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); - assertEquals(identityResponse, IdentityResponse.OptOutResponse); + assertEquals(tokenGenerateResponse, TokenGenerateResponse.OptOutResponse); } @ParameterizedTest @@ -494,28 +494,28 @@ void testSpecialIdentityOptOutIdentityMap(TestIdentityInputType type, String id, void testSpecialIdentityOptOutTokenRefresh(TestIdentityInputType type, String id, IdentityScope scope) { InputUtil.InputVal inputVal = generateInputVal(type, id); - final IdentityRequest identityRequest = new IdentityRequest( + final TokenGenerateRequest tokenGenerateRequest = new TokenGenerateRequest( new SourcePublisher(123, 124, 125), inputVal.toHashedDiiIdentity(scope), OptoutCheckPolicy.DoNotRespect, PrivacyBits.fromInt(0), this.now ); - IdentityResponse identityResponse; + TokenGenerateResponse tokenGenerateResponse; if(scope == IdentityScope.EUID) { - identityResponse = euidService.generateIdentity(identityRequest); + tokenGenerateResponse = euidService.generateIdentity(tokenGenerateRequest); } else { - identityResponse = uid2Service.generateIdentity(identityRequest); + tokenGenerateResponse = uid2Service.generateIdentity(tokenGenerateRequest); } verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); - assertNotNull(identityResponse); - assertNotEquals(IdentityResponse.OptOutResponse, identityResponse); + assertNotNull(tokenGenerateResponse); + assertNotEquals(TokenGenerateResponse.OptOutResponse, tokenGenerateResponse); // identity has no optout record, ensure refresh still returns optout when(this.optOutStore.getLatestEntry(any())).thenReturn(null); - final TokenRefreshRequest tokenRefreshRequest = this.tokenEncoder.decodeRefreshToken(identityResponse.getRefreshToken()); + final TokenRefreshRequest tokenRefreshRequest = this.tokenEncoder.decodeRefreshToken(tokenGenerateResponse.getRefreshToken()); reset(shutdownHandler); assertEquals(TokenRefreshResponse.Optout, (scope == IdentityScope.EUID? euidService: uid2Service).refreshIdentity(tokenRefreshRequest)); verify(shutdownHandler, never()).handleSaltRetrievalResponse(anyBoolean()); @@ -533,7 +533,7 @@ void testSpecialIdentityOptOutTokenRefresh(TestIdentityInputType type, String id void testSpecialIdentityRefreshOptOutGenerate(TestIdentityInputType type, String id, IdentityScope scope) { InputUtil.InputVal inputVal = generateInputVal(type, id); - final IdentityRequest identityRequest = new IdentityRequest( + final TokenGenerateRequest tokenGenerateRequest = new TokenGenerateRequest( new SourcePublisher(123, 124, 125), inputVal.toHashedDiiIdentity(scope), OptoutCheckPolicy.RespectOptOut, PrivacyBits.fromInt(0), this.now @@ -542,22 +542,22 @@ void testSpecialIdentityRefreshOptOutGenerate(TestIdentityInputType type, String // identity has optout record, ensure still generates when(this.optOutStore.getLatestEntry(any())).thenReturn(Instant.now()); - IdentityResponse identityResponse; + TokenGenerateResponse tokenGenerateResponse; if(scope == IdentityScope.EUID) { - identityResponse = euidService.generateIdentity(identityRequest); + tokenGenerateResponse = euidService.generateIdentity(tokenGenerateRequest); } else { - identityResponse = uid2Service.generateIdentity(identityRequest); + tokenGenerateResponse = uid2Service.generateIdentity(tokenGenerateRequest); } verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); - assertNotNull(identityResponse); - assertNotEquals(IdentityResponse.OptOutResponse, identityResponse); + assertNotNull(tokenGenerateResponse); + assertNotEquals(TokenGenerateResponse.OptOutResponse, tokenGenerateResponse); // identity has no optout record, ensure refresh still returns optout when(this.optOutStore.getLatestEntry(any())).thenReturn(null); - final TokenRefreshRequest tokenRefreshRequest = this.tokenEncoder.decodeRefreshToken(identityResponse.getRefreshToken()); + final TokenRefreshRequest tokenRefreshRequest = this.tokenEncoder.decodeRefreshToken(tokenGenerateResponse.getRefreshToken()); reset(shutdownHandler); assertEquals(TokenRefreshResponse.Optout, (scope == IdentityScope.EUID? euidService: uid2Service).refreshIdentity(tokenRefreshRequest)); verify(shutdownHandler, never()).handleSaltRetrievalResponse(anyBoolean()); @@ -608,7 +608,7 @@ void testSpecialIdentityRefreshOptOutIdentityMap(TestIdentityInputType type, Str void testSpecialIdentityValidateGenerate(TestIdentityInputType type, String id, IdentityScope scope) { InputUtil.InputVal inputVal = generateInputVal(type, id); - final IdentityRequest identityRequest = new IdentityRequest( + final TokenGenerateRequest tokenGenerateRequest = new TokenGenerateRequest( new SourcePublisher(123, 124, 125), inputVal.toHashedDiiIdentity(scope), OptoutCheckPolicy.RespectOptOut, PrivacyBits.fromInt(0), this.now @@ -617,19 +617,19 @@ void testSpecialIdentityValidateGenerate(TestIdentityInputType type, String id, // all identities have optout records, ensure validate identities still get generated when(this.optOutStore.getLatestEntry(any())).thenReturn(Instant.now()); - IdentityResponse identityResponse; + TokenGenerateResponse tokenGenerateResponse; AdvertisingTokenRequest advertisingTokenRequest; if (scope == IdentityScope.EUID) { - identityResponse = euidService.generateIdentity(identityRequest); + tokenGenerateResponse = euidService.generateIdentity(tokenGenerateRequest); } else { - identityResponse = uid2Service.generateIdentity(identityRequest); + tokenGenerateResponse = uid2Service.generateIdentity(tokenGenerateRequest); } - advertisingTokenRequest = validateAndGetToken(tokenEncoder, identityResponse.getAdvertisingToken(), scope, identityRequest.hashedDiiIdentity.identityType, identityRequest.sourcePublisher.siteId); + advertisingTokenRequest = validateAndGetToken(tokenEncoder, tokenGenerateResponse.getAdvertisingToken(), scope, tokenGenerateRequest.hashedDiiIdentity.identityType, tokenGenerateRequest.sourcePublisher.siteId); verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); - assertNotNull(identityResponse); - assertNotEquals(IdentityResponse.OptOutResponse, identityResponse); + assertNotNull(tokenGenerateResponse); + assertNotEquals(TokenGenerateResponse.OptOutResponse, tokenGenerateResponse); assertNotNull(advertisingTokenRequest.rawUidIdentity); } @@ -676,24 +676,24 @@ void testSpecialIdentityValidateIdentityMap(TestIdentityInputType type, String i "EmailHash,blah@unifiedid.com,EUID"}) void testNormalIdentityOptIn(TestIdentityInputType type, String id, IdentityScope scope) { InputUtil.InputVal inputVal = generateInputVal(type, id); - final IdentityRequest identityRequest = new IdentityRequest( + final TokenGenerateRequest tokenGenerateRequest = new TokenGenerateRequest( new SourcePublisher(123, 124, 125), inputVal.toHashedDiiIdentity(scope), OptoutCheckPolicy.DoNotRespect ); - IdentityResponse identityResponse; + TokenGenerateResponse tokenGenerateResponse; if(scope == IdentityScope.EUID) { - identityResponse = euidService.generateIdentity(identityRequest); + tokenGenerateResponse = euidService.generateIdentity(tokenGenerateRequest); } else { - identityResponse = uid2Service.generateIdentity(identityRequest); + tokenGenerateResponse = uid2Service.generateIdentity(tokenGenerateRequest); } verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); - assertNotEquals(identityResponse, IdentityResponse.OptOutResponse); - assertNotNull(identityResponse); + assertNotEquals(tokenGenerateResponse, TokenGenerateResponse.OptOutResponse); + assertNotNull(tokenGenerateResponse); - final TokenRefreshRequest tokenRefreshRequest = this.tokenEncoder.decodeRefreshToken(identityResponse.getRefreshToken()); + final TokenRefreshRequest tokenRefreshRequest = this.tokenEncoder.decodeRefreshToken(tokenGenerateResponse.getRefreshToken()); TokenRefreshResponse refreshResponse = (scope == IdentityScope.EUID? euidService: uid2Service).refreshIdentity(tokenRefreshRequest); assertTrue(refreshResponse.isRefreshed()); assertNotNull(refreshResponse.getIdentityResponse()); @@ -737,29 +737,29 @@ void testExpiredSaltsNotifiesShutdownHandler(TestIdentityInputType type, String InputUtil.InputVal inputVal = generateInputVal(type, id); - final IdentityRequest identityRequest = new IdentityRequest( + final TokenGenerateRequest tokenGenerateRequest = new TokenGenerateRequest( new SourcePublisher(123, 124, 125), inputVal.toHashedDiiIdentity(scope), OptoutCheckPolicy.RespectOptOut, PrivacyBits.fromInt(0), this.now); - IdentityResponse identityResponse; + TokenGenerateResponse tokenGenerateResponse; AdvertisingTokenRequest advertisingTokenRequest; reset(shutdownHandler); if(scope == IdentityScope.EUID) { - identityResponse = euidService.generateIdentity(identityRequest); - advertisingTokenRequest = validateAndGetToken(tokenEncoder, identityResponse.getAdvertisingToken(), IdentityScope.EUID, identityRequest.hashedDiiIdentity.identityType, identityRequest.sourcePublisher.siteId); + tokenGenerateResponse = euidService.generateIdentity(tokenGenerateRequest); + advertisingTokenRequest = validateAndGetToken(tokenEncoder, tokenGenerateResponse.getAdvertisingToken(), IdentityScope.EUID, tokenGenerateRequest.hashedDiiIdentity.identityType, tokenGenerateRequest.sourcePublisher.siteId); } else { - identityResponse = uid2Service.generateIdentity(identityRequest); - advertisingTokenRequest = validateAndGetToken(tokenEncoder, identityResponse.getAdvertisingToken(), IdentityScope.UID2, identityRequest.hashedDiiIdentity.identityType, identityRequest.sourcePublisher.siteId); + tokenGenerateResponse = uid2Service.generateIdentity(tokenGenerateRequest); + advertisingTokenRequest = validateAndGetToken(tokenEncoder, tokenGenerateResponse.getAdvertisingToken(), IdentityScope.UID2, tokenGenerateRequest.hashedDiiIdentity.identityType, tokenGenerateRequest.sourcePublisher.siteId); } verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(true); verify(shutdownHandler, never()).handleSaltRetrievalResponse(false); - assertNotNull(identityResponse); - assertNotEquals(IdentityResponse.OptOutResponse, identityResponse); + assertNotNull(tokenGenerateResponse); + assertNotEquals(TokenGenerateResponse.OptOutResponse, tokenGenerateResponse); assertNotNull(advertisingTokenRequest.rawUidIdentity); - final TokenRefreshRequest tokenRefreshRequest = this.tokenEncoder.decodeRefreshToken(identityResponse.getRefreshToken()); + final TokenRefreshRequest tokenRefreshRequest = this.tokenEncoder.decodeRefreshToken(tokenGenerateResponse.getRefreshToken()); reset(shutdownHandler); TokenRefreshResponse refreshResponse = (scope == IdentityScope.EUID? euidService: uid2Service).refreshIdentity(tokenRefreshRequest); verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(true); diff --git a/src/test/java/com/uid2/operator/benchmark/TokenEndecBenchmark.java b/src/test/java/com/uid2/operator/benchmark/TokenEndecBenchmark.java index 5f43c9661..b736c1a44 100644 --- a/src/test/java/com/uid2/operator/benchmark/TokenEndecBenchmark.java +++ b/src/test/java/com/uid2/operator/benchmark/TokenEndecBenchmark.java @@ -17,7 +17,7 @@ public class TokenEndecBenchmark { private static final HashedDiiIdentity[] hashedDiiIdentities; private static final SourcePublisher publisher; private static final EncryptedTokenEncoder encoder; - private static final IdentityResponse[] generatedTokens; + private static final TokenGenerateResponse[] generatedTokens; private static int idx = 0; static { @@ -35,22 +35,22 @@ public class TokenEndecBenchmark { } } - static IdentityResponse[] createAdvertisingTokens() { - List tokens = new ArrayList<>(); + static TokenGenerateResponse[] createAdvertisingTokens() { + List tokens = new ArrayList<>(); for (int i = 0; i < hashedDiiIdentities.length; i++) { tokens.add( - uidService.generateIdentity(new IdentityRequest( + uidService.generateIdentity(new TokenGenerateRequest( publisher, hashedDiiIdentities[i], OptoutCheckPolicy.DoNotRespect))); } - return tokens.toArray(new IdentityResponse[tokens.size()]); + return tokens.toArray(new TokenGenerateResponse[tokens.size()]); } @Benchmark @BenchmarkMode(Mode.Throughput) - public IdentityResponse TokenGenerationBenchmark() { - return uidService.generateIdentity(new IdentityRequest( + public TokenGenerateResponse TokenGenerationBenchmark() { + return uidService.generateIdentity(new TokenGenerateRequest( publisher, hashedDiiIdentities[(idx++) & 65535], OptoutCheckPolicy.DoNotRespect)); diff --git a/src/test/java/com/uid2/operator/utilTests/IdentityResponseTest.java b/src/test/java/com/uid2/operator/utilTests/TokenGenerateResponseTest.java similarity index 78% rename from src/test/java/com/uid2/operator/utilTests/IdentityResponseTest.java rename to src/test/java/com/uid2/operator/utilTests/TokenGenerateResponseTest.java index ae44a0b94..e659c964b 100644 --- a/src/test/java/com/uid2/operator/utilTests/IdentityResponseTest.java +++ b/src/test/java/com/uid2/operator/utilTests/TokenGenerateResponseTest.java @@ -1,6 +1,6 @@ package com.uid2.operator.utilTests; -import com.uid2.operator.model.IdentityResponse; +import com.uid2.operator.model.TokenGenerateResponse; import com.uid2.shared.model.TokenVersion; import io.vertx.core.json.JsonObject; import org.junit.Test; @@ -12,13 +12,13 @@ import static org.junit.Assert.*; -public class IdentityResponseTest { +public class TokenGenerateResponseTest { @Test public void doIdentityResponseTest() throws NoSuchAlgorithmException { - assertEquals(IdentityResponse.OptOutResponse.getAdvertisingToken(), ""); - assertTrue(IdentityResponse.OptOutResponse.isOptedOut()); + assertEquals(TokenGenerateResponse.OptOutResponse.getAdvertisingToken(), ""); + assertTrue(TokenGenerateResponse.OptOutResponse.isOptedOut()); - IdentityResponse nullAdTokenValue = new IdentityResponse(null, TokenVersion.V4, "refreshToken", null,null,null); + TokenGenerateResponse nullAdTokenValue = new TokenGenerateResponse(null, TokenVersion.V4, "refreshToken", null,null,null); assertTrue(nullAdTokenValue.isOptedOut()); Instant identityExpires = Instant.now(); @@ -27,7 +27,7 @@ public void doIdentityResponseTest() throws NoSuchAlgorithmException { - IdentityResponse response1 = new IdentityResponse("adToken", TokenVersion.V3, "refreshToken", identityExpires + TokenGenerateResponse response1 = new TokenGenerateResponse("adToken", TokenVersion.V3, "refreshToken", identityExpires , refreshExpires, refreshFrom); assertEquals(response1.getAdvertisingToken(), "adToken"); assertEquals(response1.getAdvertisingTokenVersion(), TokenVersion.V3); From 4476ee1ffda688c278594269d17970e30d7c9100 Mon Sep 17 00:00:00 2001 From: Sunny Wu Date: Mon, 9 Dec 2024 17:20:46 +1100 Subject: [PATCH 152/431] - Renamed RawUidResponse to IdentityMapResponseItem, MapRequest to IdentityMapRequestItem - Renamed IUIDOperatorService.mapHashedDiiIdentity to mapHashedDii --- ...quest.java => IdentityMapRequestItem.java} | 4 +- ...onse.java => IdentityMapResponseItem.java} | 6 +- .../operator/service/IUIDOperatorService.java | 4 +- .../operator/service/UIDOperatorService.java | 22 +++--- .../operator/vertx/UIDOperatorVerticle.java | 20 +++--- .../uid2/operator/UIDOperatorServiceTest.java | 72 +++++++++---------- .../benchmark/IdentityMapBenchmark.java | 6 +- ....java => IdentityMapResponseItemTest.java} | 12 ++-- 8 files changed, 73 insertions(+), 73 deletions(-) rename src/main/java/com/uid2/operator/model/{MapRequest.java => IdentityMapRequestItem.java} (89%) rename src/main/java/com/uid2/operator/model/{RawUidResponse.java => IdentityMapResponseItem.java} (69%) rename src/test/java/com/uid2/operator/utilTests/{RawUidResponseTest.java => IdentityMapResponseItemTest.java} (55%) diff --git a/src/main/java/com/uid2/operator/model/MapRequest.java b/src/main/java/com/uid2/operator/model/IdentityMapRequestItem.java similarity index 89% rename from src/main/java/com/uid2/operator/model/MapRequest.java rename to src/main/java/com/uid2/operator/model/IdentityMapRequestItem.java index 660503041..9d982725e 100644 --- a/src/main/java/com/uid2/operator/model/MapRequest.java +++ b/src/main/java/com/uid2/operator/model/IdentityMapRequestItem.java @@ -4,12 +4,12 @@ import java.time.Instant; -public final class MapRequest { +public final class IdentityMapRequestItem { public final HashedDiiIdentity hashedDiiIdentity; public final OptoutCheckPolicy optoutCheckPolicy; public final Instant asOf; - public MapRequest( + public IdentityMapRequestItem( HashedDiiIdentity hashedDiiIdentity, OptoutCheckPolicy optoutCheckPolicy, Instant asOf) { diff --git a/src/main/java/com/uid2/operator/model/RawUidResponse.java b/src/main/java/com/uid2/operator/model/IdentityMapResponseItem.java similarity index 69% rename from src/main/java/com/uid2/operator/model/RawUidResponse.java rename to src/main/java/com/uid2/operator/model/IdentityMapResponseItem.java index bbd98931f..909596a2f 100644 --- a/src/main/java/com/uid2/operator/model/RawUidResponse.java +++ b/src/main/java/com/uid2/operator/model/IdentityMapResponseItem.java @@ -1,13 +1,13 @@ package com.uid2.operator.model; // Contains the computed raw UID and its bucket ID from identity/map request -public class RawUidResponse { - public static final RawUidResponse OptoutIdentity = new RawUidResponse(new byte[33], ""); +public class IdentityMapResponseItem { + public static final IdentityMapResponseItem OptoutIdentity = new IdentityMapResponseItem(new byte[33], ""); // The raw UID is also known as Advertising Id (historically) public final byte[] rawUid; public final String bucketId; - public RawUidResponse(byte[] rawUid, String bucketId) { + public IdentityMapResponseItem(byte[] rawUid, String bucketId) { this.rawUid = rawUid; this.bucketId = bucketId; } diff --git a/src/main/java/com/uid2/operator/service/IUIDOperatorService.java b/src/main/java/com/uid2/operator/service/IUIDOperatorService.java index 6fec9288e..40636787c 100644 --- a/src/main/java/com/uid2/operator/service/IUIDOperatorService.java +++ b/src/main/java/com/uid2/operator/service/IUIDOperatorService.java @@ -16,10 +16,10 @@ public interface IUIDOperatorService { TokenRefreshResponse refreshIdentity(TokenRefreshRequest input); - RawUidResponse mapHashedDiiIdentity(MapRequest request); + IdentityMapResponseItem mapHashedDii(IdentityMapRequestItem request); @Deprecated - RawUidResponse map(HashedDiiIdentity hashedDiiIdentity, Instant asOf); + IdentityMapResponseItem map(HashedDiiIdentity hashedDiiIdentity, Instant asOf); List getModifiedBuckets(Instant sinceTimestamp); diff --git a/src/main/java/com/uid2/operator/service/UIDOperatorService.java b/src/main/java/com/uid2/operator/service/UIDOperatorService.java index b4914c770..2510629ed 100644 --- a/src/main/java/com/uid2/operator/service/UIDOperatorService.java +++ b/src/main/java/com/uid2/operator/service/UIDOperatorService.java @@ -161,18 +161,18 @@ public TokenRefreshResponse refreshIdentity(TokenRefreshRequest input) { } @Override - public RawUidResponse mapHashedDiiIdentity(MapRequest request) { + public IdentityMapResponseItem mapHashedDii(IdentityMapRequestItem request) { final FirstLevelHashIdentity firstLevelHashIdentity = getFirstLevelHashIdentity(request.hashedDiiIdentity, request.asOf); if (request.shouldCheckOptOut() && getGlobalOptOutResult(firstLevelHashIdentity, false).isOptedOut()) { - return RawUidResponse.OptoutIdentity; + return IdentityMapResponseItem.OptoutIdentity; } else { return generateRawUid(firstLevelHashIdentity, request.asOf); } } @Override - public RawUidResponse map(HashedDiiIdentity diiIdentity, Instant asOf) { + public IdentityMapResponseItem map(HashedDiiIdentity diiIdentity, Instant asOf) { final FirstLevelHashIdentity firstLevelHashIdentity = getFirstLevelHashIdentity(diiIdentity, asOf); return generateRawUid(firstLevelHashIdentity, asOf); } @@ -195,9 +195,9 @@ private ISaltProvider.ISaltSnapshot getSaltProviderSnapshot(Instant asOf) { @Override public void invalidateTokensAsync(HashedDiiIdentity diiIdentity, Instant asOf, Handler> handler) { final FirstLevelHashIdentity firstLevelHashIdentity = getFirstLevelHashIdentity(diiIdentity, asOf); - final RawUidResponse rawUidResponse = generateRawUid(firstLevelHashIdentity, asOf); + final IdentityMapResponseItem identityMapResponseItem = generateRawUid(firstLevelHashIdentity, asOf); - this.optOutStore.addEntry(firstLevelHashIdentity, rawUidResponse.rawUid, r -> { + this.optOutStore.addEntry(firstLevelHashIdentity, identityMapResponseItem.rawUid, r -> { if (r.succeeded()) { handler.handle(Future.succeededFuture(r.result())); } else { @@ -209,10 +209,10 @@ public void invalidateTokensAsync(HashedDiiIdentity diiIdentity, Instant asOf, H @Override public boolean advertisingTokenMatches(String advertisingToken, HashedDiiIdentity diiIdentity, Instant asOf) { final FirstLevelHashIdentity firstLevelHashIdentity = getFirstLevelHashIdentity(diiIdentity, asOf); - final RawUidResponse rawUidResponse = generateRawUid(firstLevelHashIdentity, asOf); + final IdentityMapResponseItem identityMapResponseItem = generateRawUid(firstLevelHashIdentity, asOf); final AdvertisingTokenRequest token = this.encoder.decodeAdvertisingToken(advertisingToken); - return Arrays.equals(rawUidResponse.rawUid, token.rawUidIdentity.rawUid); + return Arrays.equals(identityMapResponseItem.rawUid, token.rawUidIdentity.rawUid); } @Override @@ -239,10 +239,10 @@ private byte[] getFirstLevelHash(byte[] identityHash, Instant asOf) { return TokenUtils.getFirstLevelHash(identityHash, getSaltProviderSnapshot(asOf).getFirstLevelSalt()); } - private RawUidResponse generateRawUid(FirstLevelHashIdentity firstLevelHashIdentity, Instant asOf) { + private IdentityMapResponseItem generateRawUid(FirstLevelHashIdentity firstLevelHashIdentity, Instant asOf) { final SaltEntry rotatingSalt = getSaltProviderSnapshot(asOf).getRotatingSalt(firstLevelHashIdentity.firstLevelHash); - return new RawUidResponse( + return new IdentityMapResponseItem( this.identityV3Enabled ? TokenUtils.getRawUidV3(firstLevelHashIdentity.identityScope, firstLevelHashIdentity.identityType, firstLevelHashIdentity.firstLevelHash, rotatingSalt.getSalt()) @@ -254,10 +254,10 @@ private TokenGenerateResponse generateIdentity(SourcePublisher sourcePublisher, FirstLevelHashIdentity firstLevelHashIdentity, PrivacyBits privacyBits) { final Instant nowUtc = EncodingUtils.NowUTCMillis(this.clock); - final RawUidResponse rawUidResponse = generateRawUid(firstLevelHashIdentity, nowUtc); + final IdentityMapResponseItem identityMapResponseItem = generateRawUid(firstLevelHashIdentity, nowUtc); final RawUidIdentity rawUidIdentity = new RawUidIdentity(firstLevelHashIdentity.identityScope, firstLevelHashIdentity.identityType, - rawUidResponse.rawUid); + identityMapResponseItem.rawUid); return this.encoder.encodeIntoIdentityResponse( this.createAdvertisingTokenRequest(sourcePublisher, rawUidIdentity, nowUtc, privacyBits, diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index 8175cd25e..f8f1338d4 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -1237,11 +1237,11 @@ private void handleIdentityMapV1(RoutingContext rc) { } try { final Instant now = Instant.now(); - final RawUidResponse rawUidResponse = this.idService.map(input.toHashedDiiIdentity(this.identityScope), now); + final IdentityMapResponseItem identityMapResponseItem = this.idService.map(input.toHashedDiiIdentity(this.identityScope), now); final JsonObject jsonObject = new JsonObject(); jsonObject.put("identifier", input.getProvided()); - jsonObject.put("advertising_id", EncodingUtils.toBase64String(rawUidResponse.rawUid)); - jsonObject.put("bucket_id", rawUidResponse.bucketId); + jsonObject.put("advertising_id", EncodingUtils.toBase64String(identityMapResponseItem.rawUid)); + jsonObject.put("bucket_id", identityMapResponseItem.bucketId); ResponseUtil.Success(rc, jsonObject); } catch (Exception e) { ResponseUtil.Error(ResponseStatus.UnknownError, 500, rc, "Unknown State", e); @@ -1254,8 +1254,8 @@ private void handleIdentityMap(RoutingContext rc) { try { if (isTokenInputValid(input, rc)) { final Instant now = Instant.now(); - final RawUidResponse rawUidResponse = this.idService.map(input.toHashedDiiIdentity(this.identityScope), now); - rc.response().end(EncodingUtils.toBase64String(rawUidResponse.rawUid)); + final IdentityMapResponseItem identityMapResponseItem = this.idService.map(input.toHashedDiiIdentity(this.identityScope), now); + rc.response().end(EncodingUtils.toBase64String(identityMapResponseItem.rawUid)); } } catch (Exception ex) { LOGGER.error("Unexpected error while mapping identity", ex); @@ -1450,13 +1450,13 @@ private JsonObject handleIdentityMapCommon(RoutingContext rc, InputUtil.InputVal for (int i = 0; i < count; ++i) { final InputUtil.InputVal input = inputList[i]; if (input != null && input.isValid()) { - final RawUidResponse rawUidResponse = idService.mapHashedDiiIdentity( - new MapRequest( + final IdentityMapResponseItem identityMapResponseItem = idService.mapHashedDii( + new IdentityMapRequestItem( input.toHashedDiiIdentity(this.identityScope), OptoutCheckPolicy.respectOptOut(), now)); - if (rawUidResponse.isOptedOut()) { + if (identityMapResponseItem.isOptedOut()) { final JsonObject resp = new JsonObject(); resp.put("identifier", input.getProvided()); resp.put("reason", "optout"); @@ -1465,8 +1465,8 @@ private JsonObject handleIdentityMapCommon(RoutingContext rc, InputUtil.InputVal } else { final JsonObject resp = new JsonObject(); resp.put("identifier", input.getProvided()); - resp.put("advertising_id", EncodingUtils.toBase64String(rawUidResponse.rawUid)); - resp.put("bucket_id", rawUidResponse.bucketId); + resp.put("advertising_id", EncodingUtils.toBase64String(identityMapResponseItem.rawUid)); + resp.put("bucket_id", identityMapResponseItem.bucketId); mapped.add(resp); } } else { diff --git a/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java b/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java index 4ec802028..4e21b545d 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java @@ -348,12 +348,12 @@ public void testIdentityMapForOptOutUser(IdentityType type, String identity, Ide final HashedDiiIdentity hashedDiiIdentity = createHashedDiiIdentity(identity, scope, type); final Instant now = Instant.now(); - final MapRequest mapRequestForceMap = new MapRequest( + final IdentityMapRequestItem mapRequestForceIdentityMapItem = new IdentityMapRequestItem( hashedDiiIdentity, OptoutCheckPolicy.DoNotRespect, now); - final MapRequest mapRequestRespectOptOut = new MapRequest( + final IdentityMapRequestItem identityMapRequestItemRespectOptOut = new IdentityMapRequestItem( hashedDiiIdentity, OptoutCheckPolicy.RespectOptOut, now); @@ -362,27 +362,27 @@ public void testIdentityMapForOptOutUser(IdentityType type, String identity, Ide when(optOutStore.getLatestEntry(any(FirstLevelHashIdentity.class))) .thenReturn(Instant.now().minus(1, ChronoUnit.HOURS)); - final RawUidResponse rawUidResponse; - final RawUidResponse rawUidResponseShouldBeOptOut; + final IdentityMapResponseItem identityMapResponseItem; + final IdentityMapResponseItem identityMapResponseItemShouldBeOptOut; if (scope == IdentityScope.UID2) { verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); - rawUidResponse = uid2Service.mapHashedDiiIdentity(mapRequestForceMap); + identityMapResponseItem = uid2Service.mapHashedDii(mapRequestForceIdentityMapItem); reset(shutdownHandler); - rawUidResponseShouldBeOptOut = uid2Service.mapHashedDiiIdentity(mapRequestRespectOptOut); + identityMapResponseItemShouldBeOptOut = uid2Service.mapHashedDii(identityMapRequestItemRespectOptOut); } else { verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); - rawUidResponse = euidService.mapHashedDiiIdentity(mapRequestForceMap); + identityMapResponseItem = euidService.mapHashedDii(mapRequestForceIdentityMapItem); reset(shutdownHandler); - rawUidResponseShouldBeOptOut = euidService.mapHashedDiiIdentity(mapRequestRespectOptOut); + identityMapResponseItemShouldBeOptOut = euidService.mapHashedDii(identityMapRequestItemRespectOptOut); } verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); - assertNotNull(rawUidResponse); - assertFalse(rawUidResponse.isOptedOut()); - assertNotNull(rawUidResponseShouldBeOptOut); - assertTrue(rawUidResponseShouldBeOptOut.isOptedOut()); + assertNotNull(identityMapResponseItem); + assertFalse(identityMapResponseItem.isOptedOut()); + assertNotNull(identityMapResponseItemShouldBeOptOut); + assertTrue(identityMapResponseItemShouldBeOptOut.isOptedOut()); } private enum TestIdentityInputType { @@ -461,7 +461,7 @@ void testSpecialIdentityOptOutTokenGenerate(TestIdentityInputType type, String i void testSpecialIdentityOptOutIdentityMap(TestIdentityInputType type, String id, IdentityScope scope) { InputUtil.InputVal inputVal = generateInputVal(type, id); - final MapRequest mapRequestRespectOptOut = new MapRequest( + final IdentityMapRequestItem identityMapRequestItemRespectOptOut = new IdentityMapRequestItem( inputVal.toHashedDiiIdentity(scope), OptoutCheckPolicy.RespectOptOut, now); @@ -469,17 +469,17 @@ void testSpecialIdentityOptOutIdentityMap(TestIdentityInputType type, String id, // identity has no optout record, ensure map still returns optout when(this.optOutStore.getLatestEntry(any())).thenReturn(null); - final RawUidResponse rawUidResponse; + final IdentityMapResponseItem identityMapResponseItem; if(scope == IdentityScope.EUID) { - rawUidResponse = euidService.mapHashedDiiIdentity(mapRequestRespectOptOut); + identityMapResponseItem = euidService.mapHashedDii(identityMapRequestItemRespectOptOut); } else { - rawUidResponse = uid2Service.mapHashedDiiIdentity(mapRequestRespectOptOut); + identityMapResponseItem = uid2Service.mapHashedDii(identityMapRequestItemRespectOptOut); } verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); - assertNotNull(rawUidResponse); - assertTrue(rawUidResponse.isOptedOut()); + assertNotNull(identityMapResponseItem); + assertTrue(identityMapResponseItem.isOptedOut()); } @ParameterizedTest @@ -575,7 +575,7 @@ void testSpecialIdentityRefreshOptOutGenerate(TestIdentityInputType type, String void testSpecialIdentityRefreshOptOutIdentityMap(TestIdentityInputType type, String id, IdentityScope scope) { InputUtil.InputVal inputVal = generateInputVal(type, id); - final MapRequest mapRequestRespectOptOut = new MapRequest( + final IdentityMapRequestItem identityMapRequestItemRespectOptOut = new IdentityMapRequestItem( inputVal.toHashedDiiIdentity(scope), OptoutCheckPolicy.RespectOptOut, now); @@ -583,17 +583,17 @@ void testSpecialIdentityRefreshOptOutIdentityMap(TestIdentityInputType type, Str // all identities have optout records, ensure refresh-optout identities still map when(this.optOutStore.getLatestEntry(any())).thenReturn(Instant.now()); - final RawUidResponse rawUidResponse; + final IdentityMapResponseItem identityMapResponseItem; if(scope == IdentityScope.EUID) { - rawUidResponse = euidService.mapHashedDiiIdentity(mapRequestRespectOptOut); + identityMapResponseItem = euidService.mapHashedDii(identityMapRequestItemRespectOptOut); } else { - rawUidResponse = uid2Service.mapHashedDiiIdentity(mapRequestRespectOptOut); + identityMapResponseItem = uid2Service.mapHashedDii(identityMapRequestItemRespectOptOut); } verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); - assertNotNull(rawUidResponse); - assertFalse(rawUidResponse.isOptedOut()); + assertNotNull(identityMapResponseItem); + assertFalse(identityMapResponseItem.isOptedOut()); } @ParameterizedTest @@ -646,7 +646,7 @@ void testSpecialIdentityValidateGenerate(TestIdentityInputType type, String id, void testSpecialIdentityValidateIdentityMap(TestIdentityInputType type, String id, IdentityScope scope) { InputUtil.InputVal inputVal = generateInputVal(type, id); - final MapRequest mapRequestRespectOptOut = new MapRequest( + final IdentityMapRequestItem identityMapRequestItemRespectOptOut = new IdentityMapRequestItem( inputVal.toHashedDiiIdentity(scope), OptoutCheckPolicy.RespectOptOut, now); @@ -654,17 +654,17 @@ void testSpecialIdentityValidateIdentityMap(TestIdentityInputType type, String i // all identities have optout records, ensure validate identities still get mapped when(this.optOutStore.getLatestEntry(any())).thenReturn(Instant.now()); - final RawUidResponse rawUidResponse; + final IdentityMapResponseItem identityMapResponseItem; if(scope == IdentityScope.EUID) { - rawUidResponse = euidService.mapHashedDiiIdentity(mapRequestRespectOptOut); + identityMapResponseItem = euidService.mapHashedDii(identityMapRequestItemRespectOptOut); } else { - rawUidResponse = uid2Service.mapHashedDiiIdentity(mapRequestRespectOptOut); + identityMapResponseItem = uid2Service.mapHashedDii(identityMapRequestItemRespectOptOut); } verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); - assertNotNull(rawUidResponse); - assertFalse(rawUidResponse.isOptedOut()); + assertNotNull(identityMapResponseItem); + assertFalse(identityMapResponseItem.isOptedOut()); } @ParameterizedTest @@ -768,22 +768,22 @@ void testExpiredSaltsNotifiesShutdownHandler(TestIdentityInputType type, String assertNotNull(refreshResponse.getIdentityResponse()); assertNotEquals(TokenRefreshResponse.Optout, refreshResponse); - final MapRequest mapRequest = new MapRequest( + final IdentityMapRequestItem identityMapRequestItem = new IdentityMapRequestItem( inputVal.toHashedDiiIdentity(scope), OptoutCheckPolicy.RespectOptOut, now); - final RawUidResponse rawUidResponse; + final IdentityMapResponseItem identityMapResponseItem; reset(shutdownHandler); if(scope == IdentityScope.EUID) { - rawUidResponse = euidService.mapHashedDiiIdentity(mapRequest); + identityMapResponseItem = euidService.mapHashedDii(identityMapRequestItem); } else { - rawUidResponse = uid2Service.mapHashedDiiIdentity(mapRequest); + identityMapResponseItem = uid2Service.mapHashedDii(identityMapRequestItem); } verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(true); verify(shutdownHandler, never()).handleSaltRetrievalResponse(false); - assertNotNull(rawUidResponse); - assertFalse(rawUidResponse.isOptedOut()); + assertNotNull(identityMapResponseItem); + assertFalse(identityMapResponseItem.isOptedOut()); } } diff --git a/src/test/java/com/uid2/operator/benchmark/IdentityMapBenchmark.java b/src/test/java/com/uid2/operator/benchmark/IdentityMapBenchmark.java index 59b033542..63652ab09 100644 --- a/src/test/java/com/uid2/operator/benchmark/IdentityMapBenchmark.java +++ b/src/test/java/com/uid2/operator/benchmark/IdentityMapBenchmark.java @@ -25,13 +25,13 @@ public class IdentityMapBenchmark { @Benchmark @BenchmarkMode(Mode.Throughput) - public RawUidResponse IdentityMapRawThroughput() { + public IdentityMapResponseItem IdentityMapRawThroughput() { return uidService.map(hashedDiiIdentities[(idx++) & 65535], Instant.now()); } @Benchmark @BenchmarkMode(Mode.Throughput) - public RawUidResponse IdentityMapWithOptOutThroughput() { - return uidService.mapHashedDiiIdentity(new MapRequest(hashedDiiIdentities[(idx++) & 65535], OptoutCheckPolicy.RespectOptOut, Instant.now())); + public IdentityMapResponseItem IdentityMapWithOptOutThroughput() { + return uidService.mapHashedDii(new IdentityMapRequestItem(hashedDiiIdentities[(idx++) & 65535], OptoutCheckPolicy.RespectOptOut, Instant.now())); } } diff --git a/src/test/java/com/uid2/operator/utilTests/RawUidResponseTest.java b/src/test/java/com/uid2/operator/utilTests/IdentityMapResponseItemTest.java similarity index 55% rename from src/test/java/com/uid2/operator/utilTests/RawUidResponseTest.java rename to src/test/java/com/uid2/operator/utilTests/IdentityMapResponseItemTest.java index 4e759b853..b45b2de6d 100644 --- a/src/test/java/com/uid2/operator/utilTests/RawUidResponseTest.java +++ b/src/test/java/com/uid2/operator/utilTests/IdentityMapResponseItemTest.java @@ -1,6 +1,6 @@ package com.uid2.operator.utilTests; -import com.uid2.operator.model.RawUidResponse; +import com.uid2.operator.model.IdentityMapResponseItem; import org.junit.Test; import java.security.NoSuchAlgorithmException; @@ -9,13 +9,13 @@ import static org.junit.Assert.*; -public class RawUidResponseTest { +public class IdentityMapResponseItemTest { @Test public void doRawUidResponseTest() throws NoSuchAlgorithmException { - assertEquals(RawUidResponse.OptoutIdentity.bucketId, ""); - assertTrue(RawUidResponse.OptoutIdentity.isOptedOut()); + assertEquals(IdentityMapResponseItem.OptoutIdentity.bucketId, ""); + assertTrue(IdentityMapResponseItem.OptoutIdentity.isOptedOut()); - RawUidResponse optoutResponse = new RawUidResponse(new byte[33], null); + IdentityMapResponseItem optoutResponse = new IdentityMapResponseItem(new byte[33], null); assertTrue(optoutResponse.isOptedOut()); byte[] rawUid = new byte[33]; @@ -23,7 +23,7 @@ public void doRawUidResponseTest() throws NoSuchAlgorithmException { rawUid[i] = (byte) i; } - RawUidResponse generatedUid = new RawUidResponse(rawUid, "12345"); + IdentityMapResponseItem generatedUid = new IdentityMapResponseItem(rawUid, "12345"); assertFalse(generatedUid.isOptedOut()); assertTrue(Arrays.equals(rawUid, generatedUid.rawUid)); } From 6bb1fb9fda166dcd37ca488e06e57018cfd4a3ca Mon Sep 17 00:00:00 2001 From: Sunny Wu Date: Mon, 9 Dec 2024 18:07:00 +1100 Subject: [PATCH 153/431] 1. Renamed FirstLevelHashIdentity/HashedDiiIdentity/RawUidIdentity to FirstLevelHash/HashedDii/RawUid 2. Removed the abstract class UserIdentity 3. fixed a unit test as removing UserIdentity broke it --- .../model/AdvertisingTokenRequest.java | 8 +- .../model/IdentityMapRequestItem.java | 8 +- .../operator/model/TokenGenerateRequest.java | 12 +- .../operator/model/TokenRefreshRequest.java | 8 +- .../model/identities/FirstLevelHash.java | 22 ++++ .../operator/model/identities/HashedDii.java | 9 ++ .../operator/model/identities/RawUid.java | 16 +++ .../userIdentity/FirstLevelHashIdentity.java | 30 ----- .../model/userIdentity/HashedDiiIdentity.java | 17 --- .../model/userIdentity/RawUidIdentity.java | 23 ---- .../model/userIdentity/UserIdentity.java | 16 --- .../service/EncryptedTokenEncoder.java | 42 +++---- .../operator/service/IUIDOperatorService.java | 10 +- .../com/uid2/operator/service/InputUtil.java | 8 +- .../operator/service/UIDOperatorService.java | 118 +++++++++--------- .../operator/store/CloudSyncOptOutStore.java | 10 +- .../com/uid2/operator/store/IOptOutStore.java | 8 +- .../operator/vertx/UIDOperatorVerticle.java | 6 +- .../com/uid2/operator/TokenEncodingTest.java | 14 +-- .../uid2/operator/UIDOperatorServiceTest.java | 99 +++++++++------ .../operator/UIDOperatorVerticleTest.java | 64 +++++----- .../operator/benchmark/BenchmarkCommon.java | 16 +-- .../benchmark/IdentityMapBenchmark.java | 4 +- .../benchmark/TokenEndecBenchmark.java | 4 +- 24 files changed, 276 insertions(+), 296 deletions(-) create mode 100644 src/main/java/com/uid2/operator/model/identities/FirstLevelHash.java create mode 100644 src/main/java/com/uid2/operator/model/identities/HashedDii.java create mode 100644 src/main/java/com/uid2/operator/model/identities/RawUid.java delete mode 100644 src/main/java/com/uid2/operator/model/userIdentity/FirstLevelHashIdentity.java delete mode 100644 src/main/java/com/uid2/operator/model/userIdentity/HashedDiiIdentity.java delete mode 100644 src/main/java/com/uid2/operator/model/userIdentity/RawUidIdentity.java delete mode 100644 src/main/java/com/uid2/operator/model/userIdentity/UserIdentity.java diff --git a/src/main/java/com/uid2/operator/model/AdvertisingTokenRequest.java b/src/main/java/com/uid2/operator/model/AdvertisingTokenRequest.java index f9a41caf6..d63fa66a8 100644 --- a/src/main/java/com/uid2/operator/model/AdvertisingTokenRequest.java +++ b/src/main/java/com/uid2/operator/model/AdvertisingTokenRequest.java @@ -2,7 +2,7 @@ import java.time.Instant; -import com.uid2.operator.model.userIdentity.RawUidIdentity; +import com.uid2.operator.model.identities.RawUid; import com.uid2.operator.util.PrivacyBits; import com.uid2.shared.model.TokenVersion; @@ -10,17 +10,17 @@ public class AdvertisingTokenRequest extends VersionedTokenRequest { public final OperatorIdentity operatorIdentity; public final SourcePublisher sourcePublisher; - public final RawUidIdentity rawUidIdentity; + public final RawUid rawUid; public final PrivacyBits privacyBits; public final Instant establishedAt; public AdvertisingTokenRequest(TokenVersion version, Instant createdAt, Instant expiresAt, OperatorIdentity operatorIdentity, - SourcePublisher sourcePublisher, RawUidIdentity rawUidIdentity, PrivacyBits privacyBits, + SourcePublisher sourcePublisher, RawUid rawUid, PrivacyBits privacyBits, Instant establishedAt) { super(version, createdAt, expiresAt); this.operatorIdentity = operatorIdentity; this.sourcePublisher = sourcePublisher; - this.rawUidIdentity = rawUidIdentity; + this.rawUid = rawUid; this.privacyBits = privacyBits; this.establishedAt = establishedAt; } diff --git a/src/main/java/com/uid2/operator/model/IdentityMapRequestItem.java b/src/main/java/com/uid2/operator/model/IdentityMapRequestItem.java index 9d982725e..079af8e76 100644 --- a/src/main/java/com/uid2/operator/model/IdentityMapRequestItem.java +++ b/src/main/java/com/uid2/operator/model/IdentityMapRequestItem.java @@ -1,19 +1,19 @@ package com.uid2.operator.model; -import com.uid2.operator.model.userIdentity.HashedDiiIdentity; +import com.uid2.operator.model.identities.HashedDii; import java.time.Instant; public final class IdentityMapRequestItem { - public final HashedDiiIdentity hashedDiiIdentity; + public final HashedDii hashedDii; public final OptoutCheckPolicy optoutCheckPolicy; public final Instant asOf; public IdentityMapRequestItem( - HashedDiiIdentity hashedDiiIdentity, + HashedDii hashedDii, OptoutCheckPolicy optoutCheckPolicy, Instant asOf) { - this.hashedDiiIdentity = hashedDiiIdentity; + this.hashedDii = hashedDii; this.optoutCheckPolicy = optoutCheckPolicy; this.asOf = asOf; } diff --git a/src/main/java/com/uid2/operator/model/TokenGenerateRequest.java b/src/main/java/com/uid2/operator/model/TokenGenerateRequest.java index f482e8d21..39f3b56fc 100644 --- a/src/main/java/com/uid2/operator/model/TokenGenerateRequest.java +++ b/src/main/java/com/uid2/operator/model/TokenGenerateRequest.java @@ -1,13 +1,13 @@ package com.uid2.operator.model; -import com.uid2.operator.model.userIdentity.HashedDiiIdentity; +import com.uid2.operator.model.identities.HashedDii; import com.uid2.operator.util.PrivacyBits; import java.time.Instant; public final class TokenGenerateRequest { public final SourcePublisher sourcePublisher; - public final HashedDiiIdentity hashedDiiIdentity; + public final HashedDii hashedDii; public final OptoutCheckPolicy optoutCheckPolicy; public final PrivacyBits privacyBits; @@ -15,12 +15,12 @@ public final class TokenGenerateRequest { public TokenGenerateRequest( SourcePublisher sourcePublisher, - HashedDiiIdentity hashedDiiIdentity, + HashedDii hashedDii, OptoutCheckPolicy tokenGeneratePolicy, PrivacyBits privacyBits, Instant establishedAt) { this.sourcePublisher = sourcePublisher; - this.hashedDiiIdentity = hashedDiiIdentity; + this.hashedDii = hashedDii; this.optoutCheckPolicy = tokenGeneratePolicy; this.privacyBits = privacyBits; this.establishedAt = establishedAt; @@ -28,9 +28,9 @@ public TokenGenerateRequest( public TokenGenerateRequest( SourcePublisher sourcePublisher, - HashedDiiIdentity hashedDiiIdentity, + HashedDii hashedDii, OptoutCheckPolicy tokenGeneratePolicy) { - this(sourcePublisher, hashedDiiIdentity, tokenGeneratePolicy, PrivacyBits.DEFAULT, Instant.now()); + this(sourcePublisher, hashedDii, tokenGeneratePolicy, PrivacyBits.DEFAULT, Instant.now()); } diff --git a/src/main/java/com/uid2/operator/model/TokenRefreshRequest.java b/src/main/java/com/uid2/operator/model/TokenRefreshRequest.java index f5cc8b6eb..e2e51971a 100644 --- a/src/main/java/com/uid2/operator/model/TokenRefreshRequest.java +++ b/src/main/java/com/uid2/operator/model/TokenRefreshRequest.java @@ -2,7 +2,7 @@ import java.time.Instant; -import com.uid2.operator.model.userIdentity.FirstLevelHashIdentity; +import com.uid2.operator.model.identities.FirstLevelHash; import com.uid2.operator.util.PrivacyBits; import com.uid2.shared.model.TokenVersion; @@ -10,17 +10,17 @@ public class TokenRefreshRequest extends VersionedTokenRequest { public final OperatorIdentity operatorIdentity; public final SourcePublisher sourcePublisher; - public final FirstLevelHashIdentity firstLevelHashIdentity; + public final FirstLevelHash firstLevelHash; // by default, inherited from the previous refresh token's privacy bits public final PrivacyBits privacyBits; public TokenRefreshRequest(TokenVersion version, Instant createdAt, Instant expiresAt, OperatorIdentity operatorIdentity, - SourcePublisher sourcePublisher, FirstLevelHashIdentity firstLevelHashIdentity, PrivacyBits privacyBits) { + SourcePublisher sourcePublisher, FirstLevelHash firstLevelHash, PrivacyBits privacyBits) { super(version, createdAt, expiresAt); this.operatorIdentity = operatorIdentity; this.sourcePublisher = sourcePublisher; - this.firstLevelHashIdentity = firstLevelHashIdentity; + this.firstLevelHash = firstLevelHash; this.privacyBits = privacyBits; } } diff --git a/src/main/java/com/uid2/operator/model/identities/FirstLevelHash.java b/src/main/java/com/uid2/operator/model/identities/FirstLevelHash.java new file mode 100644 index 000000000..d46cc0ad5 --- /dev/null +++ b/src/main/java/com/uid2/operator/model/identities/FirstLevelHash.java @@ -0,0 +1,22 @@ +package com.uid2.operator.model.identities; + +import com.uid2.operator.model.IdentityScope; +import com.uid2.operator.model.IdentityType; + +import java.time.Instant; +import java.util.Arrays; + +/** + * Contains a first level salted hash computed from Hashed DII (email/phone number) + * @param establishedAt for brand new token generation, it should be the time it is generated if the first level hash is from token/refresh call, it will be when the raw UID was originally created in the earliest token generation + */ +public record FirstLevelHash(IdentityScope identityScope, IdentityType identityType, byte[] firstLevelHash, + Instant establishedAt) { + + // explicitly not checking establishedAt - this is only for making sure the first level hash matches a new input + public boolean matches(FirstLevelHash that) { + return this.identityScope.equals(that.identityScope) && + this.identityType.equals(that.identityType) && + Arrays.equals(this.firstLevelHash, that.firstLevelHash); + } +} diff --git a/src/main/java/com/uid2/operator/model/identities/HashedDii.java b/src/main/java/com/uid2/operator/model/identities/HashedDii.java new file mode 100644 index 000000000..da040c066 --- /dev/null +++ b/src/main/java/com/uid2/operator/model/identities/HashedDii.java @@ -0,0 +1,9 @@ +package com.uid2.operator.model.identities; + +import com.uid2.operator.model.IdentityScope; +import com.uid2.operator.model.IdentityType; + +// Contains a hash DII, +// This hash can either be computed from a raw email/phone number DII input or provided by the UID Participant directly +public record HashedDii(IdentityScope identityScope, IdentityType identityType, byte[] hashedDii) { +} diff --git a/src/main/java/com/uid2/operator/model/identities/RawUid.java b/src/main/java/com/uid2/operator/model/identities/RawUid.java new file mode 100644 index 000000000..775c7c46a --- /dev/null +++ b/src/main/java/com/uid2/operator/model/identities/RawUid.java @@ -0,0 +1,16 @@ +package com.uid2.operator.model.identities; + +import com.uid2.operator.model.IdentityScope; +import com.uid2.operator.model.IdentityType; + +import java.util.Arrays; + +// A raw UID is stored inside +public record RawUid(IdentityScope identityScope, IdentityType identityType, byte[] rawUid) { + + public boolean matches(RawUid that) { + return this.identityScope.equals(that.identityScope) && + this.identityType.equals(that.identityType) && + Arrays.equals(this.rawUid, that.rawUid); + } +} diff --git a/src/main/java/com/uid2/operator/model/userIdentity/FirstLevelHashIdentity.java b/src/main/java/com/uid2/operator/model/userIdentity/FirstLevelHashIdentity.java deleted file mode 100644 index 4df2b1c6c..000000000 --- a/src/main/java/com/uid2/operator/model/userIdentity/FirstLevelHashIdentity.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.uid2.operator.model.userIdentity; - -import com.uid2.operator.model.IdentityScope; -import com.uid2.operator.model.IdentityType; - -import java.time.Instant; -import java.util.Arrays; - -// Contains a first level salted hash computed from Hashed DII (email/phone number) -public class FirstLevelHashIdentity extends UserIdentity { - public final byte[] firstLevelHash; - - // for brand new token generation, it should be the time it is generated - // if the first level hash is from token/refresh call, it will be when the raw UID was originally created in the earliest token generation - public final Instant establishedAt; - - public FirstLevelHashIdentity(IdentityScope identityScope, IdentityType identityType, byte[] firstLevelHash, - Instant establishedAt) { - super(identityScope, identityType); - this.firstLevelHash = firstLevelHash; - this.establishedAt = establishedAt; - } - - // explicitly not checking establishedAt - this is only for making sure the first level hash matches a new input - public boolean matches(FirstLevelHashIdentity that) { - return this.identityScope.equals(that.identityScope) && - this.identityType.equals(that.identityType) && - Arrays.equals(this.firstLevelHash, that.firstLevelHash); - } -} diff --git a/src/main/java/com/uid2/operator/model/userIdentity/HashedDiiIdentity.java b/src/main/java/com/uid2/operator/model/userIdentity/HashedDiiIdentity.java deleted file mode 100644 index a9a6efe0a..000000000 --- a/src/main/java/com/uid2/operator/model/userIdentity/HashedDiiIdentity.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.uid2.operator.model.userIdentity; - -import com.uid2.operator.model.IdentityScope; -import com.uid2.operator.model.IdentityType; - -import java.time.Instant; - -// Contains a hash DII, -// This hash can either be computed from a raw email/phone number DII input or provided by the UID Participant directly -public class HashedDiiIdentity extends UserIdentity { - public final byte[] hashedDii; - - public HashedDiiIdentity(IdentityScope identityScope, IdentityType identityType, byte[] hashedDii) { - super(identityScope, identityType); - this.hashedDii = hashedDii; - } -} diff --git a/src/main/java/com/uid2/operator/model/userIdentity/RawUidIdentity.java b/src/main/java/com/uid2/operator/model/userIdentity/RawUidIdentity.java deleted file mode 100644 index b86740a05..000000000 --- a/src/main/java/com/uid2/operator/model/userIdentity/RawUidIdentity.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.uid2.operator.model.userIdentity; - -import com.uid2.operator.model.IdentityScope; -import com.uid2.operator.model.IdentityType; - -import java.time.Instant; -import java.util.Arrays; - -// A raw UID is stored inside -public class RawUidIdentity extends UserIdentity { - public final byte[] rawUid; - - public RawUidIdentity(IdentityScope identityScope, IdentityType identityType, byte[] rawUid) { - super(identityScope, identityType); - this.rawUid = rawUid; - } - - public boolean matches(RawUidIdentity that) { - return this.identityScope.equals(that.identityScope) && - this.identityType.equals(that.identityType) && - Arrays.equals(this.rawUid, that.rawUid); - } -} diff --git a/src/main/java/com/uid2/operator/model/userIdentity/UserIdentity.java b/src/main/java/com/uid2/operator/model/userIdentity/UserIdentity.java deleted file mode 100644 index 8b6c042c5..000000000 --- a/src/main/java/com/uid2/operator/model/userIdentity/UserIdentity.java +++ /dev/null @@ -1,16 +0,0 @@ -package com.uid2.operator.model.userIdentity; - -import com.uid2.operator.model.IdentityScope; -import com.uid2.operator.model.IdentityType; - -//base class for all other HashedDii/FirstLevelHash/RawUIDIdentity class and define the basic common fields -public abstract class UserIdentity { - - public final IdentityScope identityScope; - public final IdentityType identityType; - - public UserIdentity(IdentityScope identityScope, IdentityType identityType) { - this.identityScope = identityScope; - this.identityType = identityType; - } -} diff --git a/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java b/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java index 06570391e..87f86b244 100644 --- a/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java +++ b/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java @@ -1,8 +1,8 @@ package com.uid2.operator.service; import com.uid2.operator.model.*; -import com.uid2.operator.model.userIdentity.FirstLevelHashIdentity; -import com.uid2.operator.model.userIdentity.RawUidIdentity; +import com.uid2.operator.model.identities.FirstLevelHash; +import com.uid2.operator.model.identities.RawUid; import com.uid2.operator.util.PrivacyBits; import com.uid2.operator.vertx.ClientInputValidationException; import com.uid2.shared.Const.Data; @@ -42,7 +42,7 @@ private byte[] encodeIntoAdvertisingTokenV2(AdvertisingTokenRequest t, KeysetKey Buffer b2 = Buffer.buffer(); b2.appendLong(t.expiresAt.toEpochMilli()); - encodeSiteIdentityV2(b2, t.sourcePublisher, t.rawUidIdentity, siteKey, t.privacyBits, t.establishedAt); + encodeSiteIdentityV2(b2, t.sourcePublisher, t.rawUid, siteKey, t.privacyBits, t.establishedAt); final byte[] encryptedId = AesCbc.encrypt(b2.getBytes(), masterKey).getPayload(); @@ -59,7 +59,7 @@ private byte[] encodeIntoAdvertisingTokenV3(AdvertisingTokenRequest t, KeysetKey // this is the refreshedAt field in the spec - but effectively it is the time this advertising token is generated // this is a redundant field as it is stored in master payload again, can consider dropping this field in future token version sitePayload.appendLong(t.createdAt.toEpochMilli()); - sitePayload.appendBytes(t.rawUidIdentity.rawUid); // 32 or 33 bytes + sitePayload.appendBytes(t.rawUid.rawUid()); // 32 or 33 bytes final Buffer masterPayload = Buffer.buffer(130); masterPayload.appendLong(t.expiresAt.toEpochMilli()); @@ -69,7 +69,7 @@ private byte[] encodeIntoAdvertisingTokenV3(AdvertisingTokenRequest t, KeysetKey masterPayload.appendBytes(AesGcm.encrypt(sitePayload.getBytes(), siteKey).getPayload()); final Buffer b = Buffer.buffer(164); - b.appendByte(encodeIdentityTypeV3(t.rawUidIdentity.identityScope, t.rawUidIdentity.identityType)); + b.appendByte(encodeIdentityTypeV3(t.rawUid.identityScope(), t.rawUid.identityType())); b.appendByte((byte) t.version.rawVersion); b.appendInt(masterKey.getId()); b.appendBytes(AesGcm.encrypt(masterPayload.getBytes(), masterKey).getPayload()); @@ -129,7 +129,7 @@ private TokenRefreshRequest decodeRefreshTokenV2(Buffer b) { TokenVersion.V2, createdAt, validTill, new OperatorIdentity(0, OperatorType.Service, 0, 0), new SourcePublisher(siteId), - new FirstLevelHashIdentity(IdentityScope.UID2, IdentityType.Email, identity, + new FirstLevelHash(IdentityScope.UID2, IdentityType.Email, identity, Instant.ofEpochMilli(establishedMillis)), privacyBits); } @@ -164,7 +164,7 @@ private TokenRefreshRequest decodeRefreshTokenV3(Buffer b, byte[] bytes) { return new TokenRefreshRequest( TokenVersion.V3, createdAt, expiresAt, operatorIdentity, sourcePublisher, - new FirstLevelHashIdentity(identityScope, identityType, firstLevelHash, establishedAt), + new FirstLevelHash(identityScope, identityType, firstLevelHash, establishedAt), privacyBits); } @@ -233,7 +233,7 @@ public AdvertisingTokenRequest decodeAdvertisingTokenV2(Buffer b) { Instant.ofEpochMilli(expiresMillis), new OperatorIdentity(0, OperatorType.Service, 0, masterKeyId), new SourcePublisher(siteId, siteKeyId, 0), - new RawUidIdentity(IdentityScope.UID2, IdentityType.Email, rawUid), + new RawUid(IdentityScope.UID2, IdentityType.Email, rawUid), privacyBits, Instant.ofEpochMilli(establishedMillis) ); @@ -276,7 +276,7 @@ public AdvertisingTokenRequest decodeAdvertisingTokenV3orV4(Buffer b, byte[] byt return new AdvertisingTokenRequest( tokenVersion, createdAt, expiresAt, operatorIdentity, sourcePublisher, - new RawUidIdentity(identityScope, identityType, rawUid), + new RawUid(identityScope, identityType, rawUid), privacyBits, establishedAt ); } @@ -312,7 +312,7 @@ public byte[] encodeIntoRefreshTokenV2(TokenRefreshRequest t, KeysetKey serviceK // give an extra minute for clients which are trying to refresh tokens close to or at the refresh expiry timestamp b.appendLong(t.expiresAt.plusSeconds(60).toEpochMilli()); b.appendInt(serviceKey.getId()); - final byte[] encryptedIdentity = encryptIdentityV2(t.sourcePublisher, t.firstLevelHashIdentity, serviceKey, + final byte[] encryptedIdentity = encryptIdentityV2(t.sourcePublisher, t.firstLevelHash, serviceKey, t.privacyBits); b.appendBytes(encryptedIdentity); return b.getBytes(); @@ -325,12 +325,12 @@ public byte[] encodeIntoRefreshTokenV3(TokenRefreshRequest t, KeysetKey serviceK encodeOperatorIdentityV3(refreshPayload, t.operatorIdentity); encodePublisherRequesterV3(refreshPayload, t.sourcePublisher); refreshPayload.appendInt(t.privacyBits.getAsInt()); - refreshPayload.appendLong(t.firstLevelHashIdentity.establishedAt.toEpochMilli()); - refreshPayload.appendByte(encodeIdentityTypeV3(t.firstLevelHashIdentity.identityScope, t.firstLevelHashIdentity.identityType)); - refreshPayload.appendBytes(t.firstLevelHashIdentity.firstLevelHash); + refreshPayload.appendLong(t.firstLevelHash.establishedAt().toEpochMilli()); + refreshPayload.appendByte(encodeIdentityTypeV3(t.firstLevelHash.identityScope(), t.firstLevelHash.identityType())); + refreshPayload.appendBytes(t.firstLevelHash.firstLevelHash()); final Buffer b = Buffer.buffer(124); - b.appendByte(encodeIdentityTypeV3(t.firstLevelHashIdentity.identityScope, t.firstLevelHashIdentity.identityType)); + b.appendByte(encodeIdentityTypeV3(t.firstLevelHash.identityScope(), t.firstLevelHash.identityType())); b.appendByte((byte) t.version.rawVersion); b.appendInt(serviceKey.getId()); b.appendBytes(AesGcm.encrypt(refreshPayload.getBytes(), serviceKey).getPayload()); @@ -338,10 +338,10 @@ public byte[] encodeIntoRefreshTokenV3(TokenRefreshRequest t, KeysetKey serviceK return b.getBytes(); } - private void encodeSiteIdentityV2(Buffer b, SourcePublisher sourcePublisher, RawUidIdentity rawUidIdentity, + private void encodeSiteIdentityV2(Buffer b, SourcePublisher sourcePublisher, RawUid rawUid, KeysetKey siteEncryptionKey, PrivacyBits privacyBits, Instant establishedAt) { b.appendInt(siteEncryptionKey.getId()); - final byte[] encryptedIdentity = encryptIdentityV2(sourcePublisher, rawUidIdentity, siteEncryptionKey, privacyBits, establishedAt); + final byte[] encryptedIdentity = encryptIdentityV2(sourcePublisher, rawUid, siteEncryptionKey, privacyBits, establishedAt); b.appendBytes(encryptedIdentity); } @@ -373,15 +373,15 @@ private String generateAdvertisingTokenString(AdvertisingTokenRequest advertisin return bytesToBase64Token(advertisingTokenBytes, advertisingTokenRequest.version); } - private byte[] encryptIdentityV2(SourcePublisher sourcePublisher, FirstLevelHashIdentity firstLevelHashIdentity, + private byte[] encryptIdentityV2(SourcePublisher sourcePublisher, FirstLevelHash firstLevelHash, KeysetKey key, PrivacyBits privacyBits) { - return encryptIdentityV2(sourcePublisher, firstLevelHashIdentity.firstLevelHash, privacyBits, - firstLevelHashIdentity.establishedAt, key); + return encryptIdentityV2(sourcePublisher, firstLevelHash.firstLevelHash(), privacyBits, + firstLevelHash.establishedAt(), key); } - private byte[] encryptIdentityV2(SourcePublisher sourcePublisher, RawUidIdentity rawUidIdentity, + private byte[] encryptIdentityV2(SourcePublisher sourcePublisher, RawUid rawUid, KeysetKey key, PrivacyBits privacyBits, Instant establishedAt) { - return encryptIdentityV2(sourcePublisher, rawUidIdentity.rawUid, privacyBits, + return encryptIdentityV2(sourcePublisher, rawUid.rawUid(), privacyBits, establishedAt, key); } diff --git a/src/main/java/com/uid2/operator/service/IUIDOperatorService.java b/src/main/java/com/uid2/operator/service/IUIDOperatorService.java index 40636787c..c53e66057 100644 --- a/src/main/java/com/uid2/operator/service/IUIDOperatorService.java +++ b/src/main/java/com/uid2/operator/service/IUIDOperatorService.java @@ -1,7 +1,7 @@ package com.uid2.operator.service; import com.uid2.operator.model.*; -import com.uid2.operator.model.userIdentity.HashedDiiIdentity; +import com.uid2.operator.model.identities.HashedDii; import com.uid2.shared.model.SaltEntry; import io.vertx.core.AsyncResult; import io.vertx.core.Handler; @@ -19,15 +19,15 @@ public interface IUIDOperatorService { IdentityMapResponseItem mapHashedDii(IdentityMapRequestItem request); @Deprecated - IdentityMapResponseItem map(HashedDiiIdentity hashedDiiIdentity, Instant asOf); + IdentityMapResponseItem map(HashedDii hashedDii, Instant asOf); List getModifiedBuckets(Instant sinceTimestamp); - void invalidateTokensAsync(HashedDiiIdentity hashedDiiIdentity, Instant asOf, Handler> handler); + void invalidateTokensAsync(HashedDii hashedDii, Instant asOf, Handler> handler); - boolean advertisingTokenMatches(String advertisingToken, HashedDiiIdentity hashedDiiIdentity, Instant asOf); + boolean advertisingTokenMatches(String advertisingToken, HashedDii hashedDii, Instant asOf); - Instant getLatestOptoutEntry(HashedDiiIdentity hashedDiiIdentity, Instant asOf); + Instant getLatestOptoutEntry(HashedDii hashedDii, Instant asOf); Duration getIdentityExpiryDuration(); } diff --git a/src/main/java/com/uid2/operator/service/InputUtil.java b/src/main/java/com/uid2/operator/service/InputUtil.java index 8e44e5717..9610d20f3 100644 --- a/src/main/java/com/uid2/operator/service/InputUtil.java +++ b/src/main/java/com/uid2/operator/service/InputUtil.java @@ -2,9 +2,7 @@ import com.uid2.operator.model.IdentityScope; import com.uid2.operator.model.IdentityType; -import com.uid2.operator.model.userIdentity.HashedDiiIdentity; - -import java.time.Instant; +import com.uid2.operator.model.identities.HashedDii; public class InputUtil { @@ -261,8 +259,8 @@ public boolean isValid() { return valid; } - public HashedDiiIdentity toHashedDiiIdentity(IdentityScope identityScope) { - return new HashedDiiIdentity( + public HashedDii toHashedDiiIdentity(IdentityScope identityScope) { + return new HashedDii( identityScope, this.identityType, getIdentityInput()); diff --git a/src/main/java/com/uid2/operator/service/UIDOperatorService.java b/src/main/java/com/uid2/operator/service/UIDOperatorService.java index 2510629ed..703735e49 100644 --- a/src/main/java/com/uid2/operator/service/UIDOperatorService.java +++ b/src/main/java/com/uid2/operator/service/UIDOperatorService.java @@ -1,9 +1,9 @@ package com.uid2.operator.service; import com.uid2.operator.model.*; -import com.uid2.operator.model.userIdentity.FirstLevelHashIdentity; -import com.uid2.operator.model.userIdentity.HashedDiiIdentity; -import com.uid2.operator.model.userIdentity.RawUidIdentity; +import com.uid2.operator.model.identities.FirstLevelHash; +import com.uid2.operator.model.identities.HashedDii; +import com.uid2.operator.model.identities.RawUid; import com.uid2.operator.util.PrivacyBits; import com.uid2.shared.model.SaltEntry; import com.uid2.operator.store.IOptOutStore; @@ -39,12 +39,12 @@ public class UIDOperatorService implements IUIDOperatorService { private final EncryptedTokenEncoder encoder; private final Clock clock; private final IdentityScope identityScope; - private final FirstLevelHashIdentity testOptOutIdentityForEmail; - private final FirstLevelHashIdentity testOptOutIdentityForPhone; - private final FirstLevelHashIdentity testValidateIdentityForEmail; - private final FirstLevelHashIdentity testValidateIdentityForPhone; - private final FirstLevelHashIdentity testRefreshOptOutIdentityForEmail; - private final FirstLevelHashIdentity testRefreshOptOutIdentityForPhone; + private final FirstLevelHash testOptOutIdentityForEmail; + private final FirstLevelHash testOptOutIdentityForPhone; + private final FirstLevelHash testValidateIdentityForEmail; + private final FirstLevelHash testValidateIdentityForPhone; + private final FirstLevelHash testRefreshOptOutIdentityForEmail; + private final FirstLevelHash testRefreshOptOutIdentityForPhone; private final Duration identityExpiresAfter; private final Duration refreshExpiresAfter; private final Duration refreshIdentityAfter; @@ -107,9 +107,9 @@ public UIDOperatorService(JsonObject config, IOptOutStore optOutStore, ISaltProv @Override public TokenGenerateResponse generateIdentity(TokenGenerateRequest request) { final Instant now = EncodingUtils.NowUTCMillis(this.clock); - final byte[] firstLevelHash = getFirstLevelHash(request.hashedDiiIdentity.hashedDii, now); - final FirstLevelHashIdentity firstLevelHashIdentity = new FirstLevelHashIdentity( - request.hashedDiiIdentity.identityScope, request.hashedDiiIdentity.identityType, firstLevelHash, + final byte[] firstLevelHash = getFirstLevelHash(request.hashedDii.hashedDii(), now); + final FirstLevelHash firstLevelHashIdentity = new FirstLevelHash( + request.hashedDii.identityScope(), request.hashedDii.identityType(), firstLevelHash, request.establishedAt); if (request.shouldCheckOptOut() && getGlobalOptOutResult(firstLevelHashIdentity, false).isOptedOut()) { @@ -122,11 +122,11 @@ public TokenGenerateResponse generateIdentity(TokenGenerateRequest request) { @Override public TokenRefreshResponse refreshIdentity(TokenRefreshRequest input) { // should not be possible as different scopes should be using different keys, but just in case - if (input.firstLevelHashIdentity.identityScope != this.identityScope) { + if (input.firstLevelHash.identityScope() != this.identityScope) { return TokenRefreshResponse.Invalid; } - if (input.firstLevelHashIdentity.establishedAt.isBefore(RefreshCutoff)) { + if (input.firstLevelHash.establishedAt().isBefore(RefreshCutoff)) { return TokenRefreshResponse.Deprecated; } @@ -139,14 +139,14 @@ public TokenRefreshResponse refreshIdentity(TokenRefreshRequest input) { final boolean isCstg = input.privacyBits.isClientSideTokenGenerated(); try { - final GlobalOptoutResult logoutEntry = getGlobalOptOutResult(input.firstLevelHashIdentity, true); + final GlobalOptoutResult logoutEntry = getGlobalOptOutResult(input.firstLevelHash, true); final boolean optedOut = logoutEntry.isOptedOut(); final Duration durationSinceLastRefresh = Duration.between(input.createdAt, now); if (!optedOut) { TokenGenerateResponse tokenGenerateResponse = this.generateIdentity(input.sourcePublisher, - input.firstLevelHashIdentity, + input.firstLevelHash, input.privacyBits); return TokenRefreshResponse.createRefreshedResponse(tokenGenerateResponse, durationSinceLastRefresh, isCstg); @@ -162,19 +162,19 @@ public TokenRefreshResponse refreshIdentity(TokenRefreshRequest input) { @Override public IdentityMapResponseItem mapHashedDii(IdentityMapRequestItem request) { - final FirstLevelHashIdentity firstLevelHashIdentity = getFirstLevelHashIdentity(request.hashedDiiIdentity, + final FirstLevelHash firstLevelHash = getFirstLevelHashIdentity(request.hashedDii, request.asOf); - if (request.shouldCheckOptOut() && getGlobalOptOutResult(firstLevelHashIdentity, false).isOptedOut()) { + if (request.shouldCheckOptOut() && getGlobalOptOutResult(firstLevelHash, false).isOptedOut()) { return IdentityMapResponseItem.OptoutIdentity; } else { - return generateRawUid(firstLevelHashIdentity, request.asOf); + return generateRawUid(firstLevelHash, request.asOf); } } @Override - public IdentityMapResponseItem map(HashedDiiIdentity diiIdentity, Instant asOf) { - final FirstLevelHashIdentity firstLevelHashIdentity = getFirstLevelHashIdentity(diiIdentity, asOf); - return generateRawUid(firstLevelHashIdentity, asOf); + public IdentityMapResponseItem map(HashedDii diiIdentity, Instant asOf) { + final FirstLevelHash firstLevelHash = getFirstLevelHashIdentity(diiIdentity, asOf); + return generateRawUid(firstLevelHash, asOf); } @Override @@ -193,11 +193,11 @@ private ISaltProvider.ISaltSnapshot getSaltProviderSnapshot(Instant asOf) { } @Override - public void invalidateTokensAsync(HashedDiiIdentity diiIdentity, Instant asOf, Handler> handler) { - final FirstLevelHashIdentity firstLevelHashIdentity = getFirstLevelHashIdentity(diiIdentity, asOf); - final IdentityMapResponseItem identityMapResponseItem = generateRawUid(firstLevelHashIdentity, asOf); + public void invalidateTokensAsync(HashedDii diiIdentity, Instant asOf, Handler> handler) { + final FirstLevelHash firstLevelHash = getFirstLevelHashIdentity(diiIdentity, asOf); + final IdentityMapResponseItem identityMapResponseItem = generateRawUid(firstLevelHash, asOf); - this.optOutStore.addEntry(firstLevelHashIdentity, identityMapResponseItem.rawUid, r -> { + this.optOutStore.addEntry(firstLevelHash, identityMapResponseItem.rawUid, r -> { if (r.succeeded()) { handler.handle(Future.succeededFuture(r.result())); } else { @@ -207,18 +207,18 @@ public void invalidateTokensAsync(HashedDiiIdentity diiIdentity, Instant asOf, H } @Override - public boolean advertisingTokenMatches(String advertisingToken, HashedDiiIdentity diiIdentity, Instant asOf) { - final FirstLevelHashIdentity firstLevelHashIdentity = getFirstLevelHashIdentity(diiIdentity, asOf); - final IdentityMapResponseItem identityMapResponseItem = generateRawUid(firstLevelHashIdentity, asOf); + public boolean advertisingTokenMatches(String advertisingToken, HashedDii diiIdentity, Instant asOf) { + final FirstLevelHash firstLevelHash = getFirstLevelHashIdentity(diiIdentity, asOf); + final IdentityMapResponseItem identityMapResponseItem = generateRawUid(firstLevelHash, asOf); final AdvertisingTokenRequest token = this.encoder.decodeAdvertisingToken(advertisingToken); - return Arrays.equals(identityMapResponseItem.rawUid, token.rawUidIdentity.rawUid); + return Arrays.equals(identityMapResponseItem.rawUid, token.rawUid.rawUid()); } @Override - public Instant getLatestOptoutEntry(HashedDiiIdentity hashedDiiIdentity, Instant asOf) { - final FirstLevelHashIdentity firstLevelHashIdentity = getFirstLevelHashIdentity(hashedDiiIdentity, asOf); - return this.optOutStore.getLatestEntry(firstLevelHashIdentity); + public Instant getLatestOptoutEntry(HashedDii hashedDii, Instant asOf) { + final FirstLevelHash firstLevelHash = getFirstLevelHashIdentity(hashedDii, asOf); + return this.optOutStore.getLatestEntry(firstLevelHash); } @Override @@ -226,50 +226,50 @@ public Duration getIdentityExpiryDuration() { return this.identityExpiresAfter; } - private FirstLevelHashIdentity getFirstLevelHashIdentity(HashedDiiIdentity hashedDiiIdentity, Instant asOf) { - return getFirstLevelHashIdentity(hashedDiiIdentity.identityScope, hashedDiiIdentity.identityType, hashedDiiIdentity.hashedDii, asOf); + private FirstLevelHash getFirstLevelHashIdentity(HashedDii hashedDii, Instant asOf) { + return getFirstLevelHashIdentity(hashedDii.identityScope(), hashedDii.identityType(), hashedDii.hashedDii(), asOf); } - private FirstLevelHashIdentity getFirstLevelHashIdentity(IdentityScope identityScope, IdentityType identityType, byte[] identityHash, Instant asOf) { + private FirstLevelHash getFirstLevelHashIdentity(IdentityScope identityScope, IdentityType identityType, byte[] identityHash, Instant asOf) { final byte[] firstLevelHash = getFirstLevelHash(identityHash, asOf); - return new FirstLevelHashIdentity(identityScope, identityType, firstLevelHash, null); + return new FirstLevelHash(identityScope, identityType, firstLevelHash, null); } private byte[] getFirstLevelHash(byte[] identityHash, Instant asOf) { return TokenUtils.getFirstLevelHash(identityHash, getSaltProviderSnapshot(asOf).getFirstLevelSalt()); } - private IdentityMapResponseItem generateRawUid(FirstLevelHashIdentity firstLevelHashIdentity, Instant asOf) { - final SaltEntry rotatingSalt = getSaltProviderSnapshot(asOf).getRotatingSalt(firstLevelHashIdentity.firstLevelHash); + private IdentityMapResponseItem generateRawUid(FirstLevelHash firstLevelHash, Instant asOf) { + final SaltEntry rotatingSalt = getSaltProviderSnapshot(asOf).getRotatingSalt(firstLevelHash.firstLevelHash()); return new IdentityMapResponseItem( this.identityV3Enabled - ? TokenUtils.getRawUidV3(firstLevelHashIdentity.identityScope, - firstLevelHashIdentity.identityType, firstLevelHashIdentity.firstLevelHash, rotatingSalt.getSalt()) - : TokenUtils.getRawUidV2(firstLevelHashIdentity.firstLevelHash, rotatingSalt.getSalt()), + ? TokenUtils.getRawUidV3(firstLevelHash.identityScope(), + firstLevelHash.identityType(), firstLevelHash.firstLevelHash(), rotatingSalt.getSalt()) + : TokenUtils.getRawUidV2(firstLevelHash.firstLevelHash(), rotatingSalt.getSalt()), rotatingSalt.getHashedId()); } private TokenGenerateResponse generateIdentity(SourcePublisher sourcePublisher, - FirstLevelHashIdentity firstLevelHashIdentity, PrivacyBits privacyBits) { + FirstLevelHash firstLevelHash, PrivacyBits privacyBits) { final Instant nowUtc = EncodingUtils.NowUTCMillis(this.clock); - final IdentityMapResponseItem identityMapResponseItem = generateRawUid(firstLevelHashIdentity, nowUtc); - final RawUidIdentity rawUidIdentity = new RawUidIdentity(firstLevelHashIdentity.identityScope, - firstLevelHashIdentity.identityType, + final IdentityMapResponseItem identityMapResponseItem = generateRawUid(firstLevelHash, nowUtc); + final RawUid rawUid = new RawUid(firstLevelHash.identityScope(), + firstLevelHash.identityType(), identityMapResponseItem.rawUid); return this.encoder.encodeIntoIdentityResponse( - this.createAdvertisingTokenRequest(sourcePublisher, rawUidIdentity, nowUtc, privacyBits, - firstLevelHashIdentity.establishedAt), - this.createTokenRefreshRequest(sourcePublisher, firstLevelHashIdentity, nowUtc, privacyBits), + this.createAdvertisingTokenRequest(sourcePublisher, rawUid, nowUtc, privacyBits, + firstLevelHash.establishedAt()), + this.createTokenRefreshRequest(sourcePublisher, firstLevelHash, nowUtc, privacyBits), nowUtc.plusMillis(refreshIdentityAfter.toMillis()), nowUtc ); } private TokenRefreshRequest createTokenRefreshRequest(SourcePublisher sourcePublisher, - FirstLevelHashIdentity firstLevelHashIdentity, + FirstLevelHash firstLevelHash, Instant now, PrivacyBits privacyBits) { return new TokenRefreshRequest( @@ -278,18 +278,18 @@ private TokenRefreshRequest createTokenRefreshRequest(SourcePublisher sourcePubl now.plusMillis(refreshExpiresAfter.toMillis()), this.operatorIdentity, sourcePublisher, - firstLevelHashIdentity, + firstLevelHash, privacyBits); } - private AdvertisingTokenRequest createAdvertisingTokenRequest(SourcePublisher sourcePublisher, RawUidIdentity rawUidIdentity, + private AdvertisingTokenRequest createAdvertisingTokenRequest(SourcePublisher sourcePublisher, RawUid rawUidIdentity, Instant now, PrivacyBits privacyBits, Instant establishedAt) { TokenVersion tokenVersion; if (siteIdsUsingV4Tokens.contains(sourcePublisher.siteId)) { tokenVersion = TokenVersion.V4; } else { int pseudoRandomNumber = 1; - final var rawUid = rawUidIdentity.rawUid; + final var rawUid = rawUidIdentity.rawUid(); if (rawUid.length > 2) { int hash = ((rawUid[0] & 0xFF) << 12) | ((rawUid[1] & 0xFF) << 4) | ((rawUid[2] & 0xFF) & 0xF); //using same logic as ModBasedSaltEntryIndexer.getIndex() in uid2-shared @@ -322,16 +322,16 @@ public Instant getTime() { } } - private GlobalOptoutResult getGlobalOptOutResult(FirstLevelHashIdentity firstLevelHashIdentity, boolean forRefresh) { - if (forRefresh && (firstLevelHashIdentity.matches(testRefreshOptOutIdentityForEmail) || firstLevelHashIdentity.matches(testRefreshOptOutIdentityForPhone))) { + private GlobalOptoutResult getGlobalOptOutResult(FirstLevelHash firstLevelHash, boolean forRefresh) { + if (forRefresh && (firstLevelHash.matches(testRefreshOptOutIdentityForEmail) || firstLevelHash.matches(testRefreshOptOutIdentityForPhone))) { return new GlobalOptoutResult(Instant.now()); - } else if (firstLevelHashIdentity.matches(testValidateIdentityForEmail) || firstLevelHashIdentity.matches(testValidateIdentityForPhone) - || firstLevelHashIdentity.matches(testRefreshOptOutIdentityForEmail) || firstLevelHashIdentity.matches(testRefreshOptOutIdentityForPhone)) { + } else if (firstLevelHash.matches(testValidateIdentityForEmail) || firstLevelHash.matches(testValidateIdentityForPhone) + || firstLevelHash.matches(testRefreshOptOutIdentityForEmail) || firstLevelHash.matches(testRefreshOptOutIdentityForPhone)) { return new GlobalOptoutResult(null); - } else if (firstLevelHashIdentity.matches(testOptOutIdentityForEmail) || firstLevelHashIdentity.matches(testOptOutIdentityForPhone)) { + } else if (firstLevelHash.matches(testOptOutIdentityForEmail) || firstLevelHash.matches(testOptOutIdentityForPhone)) { return new GlobalOptoutResult(Instant.now()); } - Instant result = this.optOutStore.getLatestEntry(firstLevelHashIdentity); + Instant result = this.optOutStore.getLatestEntry(firstLevelHash); return new GlobalOptoutResult(result); } } diff --git a/src/main/java/com/uid2/operator/store/CloudSyncOptOutStore.java b/src/main/java/com/uid2/operator/store/CloudSyncOptOutStore.java index de52a6c75..cf2655984 100644 --- a/src/main/java/com/uid2/operator/store/CloudSyncOptOutStore.java +++ b/src/main/java/com/uid2/operator/store/CloudSyncOptOutStore.java @@ -4,7 +4,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.uid2.operator.Const; -import com.uid2.operator.model.userIdentity.FirstLevelHashIdentity; +import com.uid2.operator.model.identities.FirstLevelHash; import com.uid2.operator.service.EncodingUtils; import com.uid2.shared.Utils; import com.uid2.shared.cloud.CloudStorageException; @@ -74,8 +74,8 @@ public CloudSyncOptOutStore(Vertx vertx, ICloudStorage fsLocal, JsonObject jsonC } @Override - public Instant getLatestEntry(FirstLevelHashIdentity firstLevelHashIdentity) { - long epochSecond = this.snapshot.get().getOptOutTimestamp(firstLevelHashIdentity.firstLevelHash); + public Instant getLatestEntry(FirstLevelHash firstLevelHash) { + long epochSecond = this.snapshot.get().getOptOutTimestamp(firstLevelHash.firstLevelHash()); Instant instant = epochSecond > 0 ? Instant.ofEpochSecond(epochSecond) : null; return instant; } @@ -86,14 +86,14 @@ public long getOptOutTimestampByAdId(String adId) { } @Override - public void addEntry(FirstLevelHashIdentity firstLevelHashIdentity, byte[] advertisingId, Handler> handler) { + public void addEntry(FirstLevelHash firstLevelHash, byte[] advertisingId, Handler> handler) { if (remoteApiHost == null) { handler.handle(Future.failedFuture("remote api not set")); return; } this.webClient.get(remoteApiPort, remoteApiHost, remoteApiPath) - .addQueryParam("identity_hash", EncodingUtils.toBase64String(firstLevelHashIdentity.firstLevelHash)) + .addQueryParam("identity_hash", EncodingUtils.toBase64String(firstLevelHash.firstLevelHash())) .addQueryParam("advertising_id", EncodingUtils.toBase64String(advertisingId)) // advertising id aka raw UID .putHeader("Authorization", remoteApiBearerToken) .as(BodyCodec.string()) diff --git a/src/main/java/com/uid2/operator/store/IOptOutStore.java b/src/main/java/com/uid2/operator/store/IOptOutStore.java index 995939c70..09ba19e28 100644 --- a/src/main/java/com/uid2/operator/store/IOptOutStore.java +++ b/src/main/java/com/uid2/operator/store/IOptOutStore.java @@ -1,6 +1,6 @@ package com.uid2.operator.store; -import com.uid2.operator.model.userIdentity.FirstLevelHashIdentity; +import com.uid2.operator.model.identities.FirstLevelHash; import io.vertx.core.AsyncResult; import io.vertx.core.Handler; @@ -11,12 +11,12 @@ public interface IOptOutStore { /** * Get latest opt-out record * - * @param firstLevelHashIdentity The first level hash of a DII Hash + * @param firstLevelHash The first level hash of a DII Hash * @return The timestamp of latest opt-out record. NULL if no record. */ - Instant getLatestEntry(FirstLevelHashIdentity firstLevelHashIdentity); + Instant getLatestEntry(FirstLevelHash firstLevelHash); long getOptOutTimestampByAdId(String adId); - void addEntry(FirstLevelHashIdentity firstLevelHashIdentity, byte[] advertisingId, Handler> handler); + void addEntry(FirstLevelHash firstLevelHash, byte[] advertisingId, Handler> handler); } diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index f8f1338d4..a61af9790 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -4,7 +4,7 @@ import com.uid2.operator.model.*; import com.uid2.operator.model.TokenGenerateResponse; import com.uid2.operator.model.IdentityScope; -import com.uid2.operator.model.userIdentity.HashedDiiIdentity; +import com.uid2.operator.model.identities.HashedDii; import com.uid2.operator.monitoring.IStatsCollectorQueue; import com.uid2.operator.monitoring.StatsCollectorHandler; import com.uid2.operator.monitoring.TokenResponseStatsCollector; @@ -1152,8 +1152,8 @@ private void handleOptOutGet(RoutingContext rc) { if (input.isValid()) { try { final Instant now = Instant.now(); - final HashedDiiIdentity hashedDiiIdentity = input.toHashedDiiIdentity(this.identityScope); - final Instant result = this.idService.getLatestOptoutEntry(hashedDiiIdentity, now); + final HashedDii hashedDii = input.toHashedDiiIdentity(this.identityScope); + final Instant result = this.idService.getLatestOptoutEntry(hashedDii, now); long timestamp = result == null ? -1 : result.getEpochSecond(); rc.response().setStatusCode(200) .setChunked(true) diff --git a/src/test/java/com/uid2/operator/TokenEncodingTest.java b/src/test/java/com/uid2/operator/TokenEncodingTest.java index 0b0009ef7..039ebd17f 100644 --- a/src/test/java/com/uid2/operator/TokenEncodingTest.java +++ b/src/test/java/com/uid2/operator/TokenEncodingTest.java @@ -1,8 +1,8 @@ package com.uid2.operator; import com.uid2.operator.model.*; -import com.uid2.operator.model.userIdentity.FirstLevelHashIdentity; -import com.uid2.operator.model.userIdentity.RawUidIdentity; +import com.uid2.operator.model.identities.FirstLevelHash; +import com.uid2.operator.model.identities.RawUid; import com.uid2.operator.service.EncodingUtils; import com.uid2.operator.service.EncryptedTokenEncoder; import com.uid2.operator.service.TokenUtils; @@ -60,7 +60,7 @@ public void testRefreshTokenEncoding(TokenVersion tokenVersion) { now.plusSeconds(360), new OperatorIdentity(101, OperatorType.Service, 102, 103), new SourcePublisher(111, 112, 113), - new FirstLevelHashIdentity(IdentityScope.UID2, IdentityType.Email, firstLevelHash, now), + new FirstLevelHash(IdentityScope.UID2, IdentityType.Email, firstLevelHash, now), PrivacyBits.fromInt(121) ); @@ -75,9 +75,9 @@ public void testRefreshTokenEncoding(TokenVersion tokenVersion) { assertEquals(tokenRefreshRequest.createdAt, decoded.createdAt); int addSeconds = (tokenVersion == TokenVersion.V2) ? 60 : 0; //todo: why is there a 60 second buffer in encodeV2() but not in encodeV3()? assertEquals(tokenRefreshRequest.expiresAt.plusSeconds(addSeconds), decoded.expiresAt); - assertTrue(tokenRefreshRequest.firstLevelHashIdentity.matches(decoded.firstLevelHashIdentity)); + assertTrue(tokenRefreshRequest.firstLevelHash.matches(decoded.firstLevelHash)); assertEquals(tokenRefreshRequest.privacyBits, decoded.privacyBits); - assertEquals(tokenRefreshRequest.firstLevelHashIdentity.establishedAt, decoded.firstLevelHashIdentity.establishedAt); + assertEquals(tokenRefreshRequest.firstLevelHash.establishedAt(), decoded.firstLevelHash.establishedAt()); assertEquals(tokenRefreshRequest.sourcePublisher.siteId, decoded.sourcePublisher.siteId); Buffer b = Buffer.buffer(encodedBytes); @@ -103,7 +103,7 @@ public void testAdvertisingTokenEncodings(TokenVersion tokenVersion) { now.plusSeconds(60), new OperatorIdentity(101, OperatorType.Service, 102, 103), new SourcePublisher(111, 112, 113), - new RawUidIdentity(IdentityScope.UID2, IdentityType.Email, rawUid), + new RawUid(IdentityScope.UID2, IdentityType.Email, rawUid), PrivacyBits.fromInt(121), now ); @@ -114,7 +114,7 @@ public void testAdvertisingTokenEncodings(TokenVersion tokenVersion) { assertEquals(tokenVersion, decoded.version); assertEquals(adTokenRequest.createdAt, decoded.createdAt); assertEquals(adTokenRequest.expiresAt, decoded.expiresAt); - assertTrue(adTokenRequest.rawUidIdentity.matches(decoded.rawUidIdentity)); + assertTrue(adTokenRequest.rawUid.matches(decoded.rawUid)); assertEquals(adTokenRequest.privacyBits, decoded.privacyBits); assertEquals(adTokenRequest.establishedAt, decoded.establishedAt); assertEquals(adTokenRequest.sourcePublisher.siteId, decoded.sourcePublisher.siteId); diff --git a/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java b/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java index 4e21b545d..bd4874c6e 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java @@ -1,9 +1,9 @@ package com.uid2.operator; import com.uid2.operator.model.*; -import com.uid2.operator.model.userIdentity.FirstLevelHashIdentity; -import com.uid2.operator.model.userIdentity.HashedDiiIdentity; -import com.uid2.operator.model.userIdentity.UserIdentity; +import com.uid2.operator.model.identities.FirstLevelHash; +import com.uid2.operator.model.identities.HashedDii; +import com.uid2.operator.model.identities.RawUid; import com.uid2.operator.service.*; import com.uid2.operator.service.EncodingUtils; import com.uid2.operator.service.EncryptedTokenEncoder; @@ -146,8 +146,8 @@ private void setNow(Instant now) { when(clock.instant()).thenAnswer(i -> this.now); } - private HashedDiiIdentity createHashedDiiIdentity(String rawIdentityHash, IdentityScope scope, IdentityType type) { - return new HashedDiiIdentity( + private HashedDii createHashedDiiIdentity(String rawIdentityHash, IdentityScope scope, IdentityType type) { + return new HashedDii( scope, type, rawIdentityHash.getBytes(StandardCharsets.UTF_8) @@ -160,18 +160,38 @@ private AdvertisingTokenRequest validateAndGetToken(EncryptedTokenEncoder tokenE return tokenEncoder.decodeAdvertisingToken(advertisingTokenString); } - private void assertIdentityScopeIdentityType(UserIdentity expctedValues, - UserIdentity actualValues) { - assertEquals(expctedValues.identityScope, actualValues.identityScope); - assertEquals(expctedValues.identityType, actualValues.identityType); + private void assertIdentityScopeIdentityType(IdentityScope expectedScope, IdentityType expectedIdentityType, + HashedDii hashedDii) { + assertEquals(expectedScope, hashedDii.identityScope()); + assertEquals(expectedIdentityType, hashedDii.identityType()); } + private void assertIdentityScopeIdentityType(IdentityScope expectedScope, IdentityType expectedIdentityType, + RawUid rawUid) { + assertEquals(expectedScope, rawUid.identityScope()); + assertEquals(expectedIdentityType, rawUid.identityType()); + } + + private void assertIdentityScopeIdentityType(IdentityScope expectedScope, IdentityType expectedIdentityType, + FirstLevelHash firstLevelHash) { + assertEquals(expectedScope, firstLevelHash.identityScope()); + assertEquals(expectedIdentityType, firstLevelHash.identityType()); + } + + + + + @ParameterizedTest @CsvSource({"123, V2","127, V4","128, V4"}) //site id 127 and 128 is for testing "site_ids_using_v4_tokens" public void testGenerateAndRefresh(int siteId, TokenVersion tokenVersion) { + IdentityScope expectedIdentityScope = IdentityScope.UID2; + IdentityType expectedIdentityType = IdentityType.Email; + + final TokenGenerateRequest tokenGenerateRequest = new TokenGenerateRequest( new SourcePublisher(siteId, 124, 125), - createHashedDiiIdentity("test-email-hash", IdentityScope.UID2, IdentityType.Email), + createHashedDiiIdentity("test-email-hash", expectedIdentityScope, expectedIdentityType), OptoutCheckPolicy.DoNotRespect, PrivacyBits.fromInt(0), this.now.minusSeconds(234) ); @@ -184,7 +204,8 @@ public void testGenerateAndRefresh(int siteId, TokenVersion tokenVersion) { AdvertisingTokenRequest advertisingTokenRequest = tokenEncoder.decodeAdvertisingToken(tokenGenerateResponse.getAdvertisingToken()); assertEquals(this.now.plusSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS), advertisingTokenRequest.expiresAt); assertEquals(tokenGenerateRequest.sourcePublisher.siteId, advertisingTokenRequest.sourcePublisher.siteId); - assertIdentityScopeIdentityType(tokenGenerateRequest.hashedDiiIdentity, advertisingTokenRequest.rawUidIdentity); + assertIdentityScopeIdentityType(expectedIdentityScope, expectedIdentityType, + advertisingTokenRequest.rawUid); assertEquals(tokenGenerateRequest.establishedAt, advertisingTokenRequest.establishedAt); assertEquals(tokenGenerateRequest.privacyBits, advertisingTokenRequest.privacyBits); @@ -192,12 +213,12 @@ public void testGenerateAndRefresh(int siteId, TokenVersion tokenVersion) { assertEquals(this.now, tokenRefreshRequest.createdAt); assertEquals(this.now.plusSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), tokenRefreshRequest.expiresAt); assertEquals(tokenGenerateRequest.sourcePublisher.siteId, tokenRefreshRequest.sourcePublisher.siteId); - assertIdentityScopeIdentityType(tokenGenerateRequest.hashedDiiIdentity, tokenRefreshRequest.firstLevelHashIdentity); - assertEquals(tokenGenerateRequest.establishedAt, tokenRefreshRequest.firstLevelHashIdentity.establishedAt); + assertIdentityScopeIdentityType(expectedIdentityScope, expectedIdentityType, tokenRefreshRequest.firstLevelHash); + assertEquals(tokenGenerateRequest.establishedAt, tokenRefreshRequest.firstLevelHash.establishedAt()); - final byte[] firstLevelHash = getFirstLevelHash(tokenGenerateRequest.hashedDiiIdentity.hashedDii, + final byte[] firstLevelHash = getFirstLevelHash(tokenGenerateRequest.hashedDii.hashedDii(), saltProvider.getSnapshot(this.now).getFirstLevelSalt() ); - assertArrayEquals(firstLevelHash, tokenRefreshRequest.firstLevelHashIdentity.firstLevelHash); + assertArrayEquals(firstLevelHash, tokenRefreshRequest.firstLevelHash.firstLevelHash()); setNow(Instant.now().plusSeconds(200)); @@ -214,21 +235,21 @@ public void testGenerateAndRefresh(int siteId, TokenVersion tokenVersion) { AdvertisingTokenRequest advertisingTokenRequest2 = tokenEncoder.decodeAdvertisingToken(refreshResponse.getIdentityResponse().getAdvertisingToken()); assertEquals(this.now.plusSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS), advertisingTokenRequest2.expiresAt); assertEquals(advertisingTokenRequest.sourcePublisher.siteId, advertisingTokenRequest2.sourcePublisher.siteId); - assertIdentityScopeIdentityType(advertisingTokenRequest.rawUidIdentity, - advertisingTokenRequest2.rawUidIdentity); + assertIdentityScopeIdentityType(expectedIdentityScope, expectedIdentityType, + advertisingTokenRequest2.rawUid); assertEquals(advertisingTokenRequest.establishedAt, advertisingTokenRequest2.establishedAt); - assertArrayEquals(advertisingTokenRequest.rawUidIdentity.rawUid, - advertisingTokenRequest2.rawUidIdentity.rawUid); + assertArrayEquals(advertisingTokenRequest.rawUid.rawUid(), + advertisingTokenRequest2.rawUid.rawUid()); assertEquals(tokenGenerateRequest.privacyBits, advertisingTokenRequest2.privacyBits); TokenRefreshRequest tokenRefreshRequest2 = tokenEncoder.decodeRefreshToken(refreshResponse.getIdentityResponse().getRefreshToken()); assertEquals(this.now, tokenRefreshRequest2.createdAt); assertEquals(this.now.plusSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), tokenRefreshRequest2.expiresAt); assertEquals(tokenRefreshRequest.sourcePublisher.siteId, tokenRefreshRequest2.sourcePublisher.siteId); - assertIdentityScopeIdentityType(tokenRefreshRequest.firstLevelHashIdentity, tokenRefreshRequest2.firstLevelHashIdentity); - assertEquals(tokenRefreshRequest.firstLevelHashIdentity.establishedAt, tokenRefreshRequest2.firstLevelHashIdentity.establishedAt); - assertArrayEquals(tokenRefreshRequest.firstLevelHashIdentity.firstLevelHash, tokenRefreshRequest2.firstLevelHashIdentity.firstLevelHash); - assertArrayEquals(firstLevelHash, tokenRefreshRequest2.firstLevelHashIdentity.firstLevelHash); + assertIdentityScopeIdentityType(expectedIdentityScope, expectedIdentityType, tokenRefreshRequest2.firstLevelHash); + assertEquals(tokenRefreshRequest.firstLevelHash.establishedAt(), tokenRefreshRequest2.firstLevelHash.establishedAt()); + assertArrayEquals(tokenRefreshRequest.firstLevelHash.firstLevelHash(), tokenRefreshRequest2.firstLevelHash.firstLevelHash()); + assertArrayEquals(firstLevelHash, tokenRefreshRequest2.firstLevelHash.firstLevelHash()); } @Test @@ -292,23 +313,23 @@ public void testTestOptOutKeyIdentityScopeMismatch() { "Phone,+01010101010,UID2", "Phone,+01010101010,EUID"}) public void testGenerateTokenForOptOutUser(IdentityType type, String id, IdentityScope scope) { - final HashedDiiIdentity hashedDiiIdentity = createHashedDiiIdentity(TokenUtils.getIdentityHashString(id), + final HashedDii hashedDii = createHashedDiiIdentity(TokenUtils.getIdentityHashString(id), scope, type); final TokenGenerateRequest tokenGenerateRequestForceGenerate = new TokenGenerateRequest( new SourcePublisher(123, 124, 125), - hashedDiiIdentity, + hashedDii, OptoutCheckPolicy.DoNotRespect, PrivacyBits.fromInt(0), this.now.minusSeconds(234)); final TokenGenerateRequest tokenGenerateRequestRespectOptOut = new TokenGenerateRequest( new SourcePublisher(123, 124, 125), - hashedDiiIdentity, + hashedDii, OptoutCheckPolicy.RespectOptOut, PrivacyBits.fromInt(0), this.now.minusSeconds(234)); // the clock value shouldn't matter here - when(optOutStore.getLatestEntry(any(FirstLevelHashIdentity.class))) + when(optOutStore.getLatestEntry(any(FirstLevelHash.class))) .thenReturn(Instant.now().minus(1, ChronoUnit.HOURS)); final TokenGenerateResponse tokenGenerateResponse; @@ -318,7 +339,7 @@ public void testGenerateTokenForOptOutUser(IdentityType type, String id, Identit tokenGenerateResponse = uid2Service.generateIdentity(tokenGenerateRequestForceGenerate); verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); - advertisingTokenRequest = validateAndGetToken(tokenEncoder, tokenGenerateResponse.getAdvertisingToken(), IdentityScope.UID2, hashedDiiIdentity.identityType, tokenGenerateRequestRespectOptOut.sourcePublisher.siteId); + advertisingTokenRequest = validateAndGetToken(tokenEncoder, tokenGenerateResponse.getAdvertisingToken(), IdentityScope.UID2, hashedDii.identityType(), tokenGenerateRequestRespectOptOut.sourcePublisher.siteId); reset(shutdownHandler); tokenGenerateResponseAfterOptOut = uid2Service.generateIdentity(tokenGenerateRequestRespectOptOut); @@ -326,14 +347,14 @@ public void testGenerateTokenForOptOutUser(IdentityType type, String id, Identit tokenGenerateResponse = euidService.generateIdentity(tokenGenerateRequestForceGenerate); verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); - advertisingTokenRequest = validateAndGetToken(tokenEncoder, tokenGenerateResponse.getAdvertisingToken(), IdentityScope.EUID, hashedDiiIdentity.identityType, tokenGenerateRequestRespectOptOut.sourcePublisher.siteId); + advertisingTokenRequest = validateAndGetToken(tokenEncoder, tokenGenerateResponse.getAdvertisingToken(), IdentityScope.EUID, hashedDii.identityType(), tokenGenerateRequestRespectOptOut.sourcePublisher.siteId); reset(shutdownHandler); tokenGenerateResponseAfterOptOut = euidService.generateIdentity(tokenGenerateRequestRespectOptOut); } verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); assertNotNull(tokenGenerateResponse); - assertNotNull(advertisingTokenRequest.rawUidIdentity); + assertNotNull(advertisingTokenRequest.rawUid); assertNotNull(tokenGenerateResponseAfterOptOut); assertTrue(tokenGenerateResponseAfterOptOut.getAdvertisingToken() == null || tokenGenerateResponseAfterOptOut.getAdvertisingToken().isEmpty()); assertTrue(tokenGenerateResponseAfterOptOut.isOptedOut()); @@ -345,21 +366,21 @@ public void testGenerateTokenForOptOutUser(IdentityType type, String id, Identit "Phone,+01010101010,UID2", "Phone,+01010101010,EUID"}) public void testIdentityMapForOptOutUser(IdentityType type, String identity, IdentityScope scope) { - final HashedDiiIdentity hashedDiiIdentity = createHashedDiiIdentity(identity, scope, type); + final HashedDii hashedDii = createHashedDiiIdentity(identity, scope, type); final Instant now = Instant.now(); final IdentityMapRequestItem mapRequestForceIdentityMapItem = new IdentityMapRequestItem( - hashedDiiIdentity, + hashedDii, OptoutCheckPolicy.DoNotRespect, now); final IdentityMapRequestItem identityMapRequestItemRespectOptOut = new IdentityMapRequestItem( - hashedDiiIdentity, + hashedDii, OptoutCheckPolicy.RespectOptOut, now); // the clock value shouldn't matter here - when(optOutStore.getLatestEntry(any(FirstLevelHashIdentity.class))) + when(optOutStore.getLatestEntry(any(FirstLevelHash.class))) .thenReturn(Instant.now().minus(1, ChronoUnit.HOURS)); final IdentityMapResponseItem identityMapResponseItem; @@ -625,12 +646,12 @@ void testSpecialIdentityValidateGenerate(TestIdentityInputType type, String id, else { tokenGenerateResponse = uid2Service.generateIdentity(tokenGenerateRequest); } - advertisingTokenRequest = validateAndGetToken(tokenEncoder, tokenGenerateResponse.getAdvertisingToken(), scope, tokenGenerateRequest.hashedDiiIdentity.identityType, tokenGenerateRequest.sourcePublisher.siteId); + advertisingTokenRequest = validateAndGetToken(tokenEncoder, tokenGenerateResponse.getAdvertisingToken(), scope, tokenGenerateRequest.hashedDii.identityType(), tokenGenerateRequest.sourcePublisher.siteId); verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); assertNotNull(tokenGenerateResponse); assertNotEquals(TokenGenerateResponse.OptOutResponse, tokenGenerateResponse); - assertNotNull(advertisingTokenRequest.rawUidIdentity); + assertNotNull(advertisingTokenRequest.rawUid); } @@ -747,17 +768,17 @@ void testExpiredSaltsNotifiesShutdownHandler(TestIdentityInputType type, String reset(shutdownHandler); if(scope == IdentityScope.EUID) { tokenGenerateResponse = euidService.generateIdentity(tokenGenerateRequest); - advertisingTokenRequest = validateAndGetToken(tokenEncoder, tokenGenerateResponse.getAdvertisingToken(), IdentityScope.EUID, tokenGenerateRequest.hashedDiiIdentity.identityType, tokenGenerateRequest.sourcePublisher.siteId); + advertisingTokenRequest = validateAndGetToken(tokenEncoder, tokenGenerateResponse.getAdvertisingToken(), IdentityScope.EUID, tokenGenerateRequest.hashedDii.identityType(), tokenGenerateRequest.sourcePublisher.siteId); } else { tokenGenerateResponse = uid2Service.generateIdentity(tokenGenerateRequest); - advertisingTokenRequest = validateAndGetToken(tokenEncoder, tokenGenerateResponse.getAdvertisingToken(), IdentityScope.UID2, tokenGenerateRequest.hashedDiiIdentity.identityType, tokenGenerateRequest.sourcePublisher.siteId); + advertisingTokenRequest = validateAndGetToken(tokenEncoder, tokenGenerateResponse.getAdvertisingToken(), IdentityScope.UID2, tokenGenerateRequest.hashedDii.identityType(), tokenGenerateRequest.sourcePublisher.siteId); } verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(true); verify(shutdownHandler, never()).handleSaltRetrievalResponse(false); assertNotNull(tokenGenerateResponse); assertNotEquals(TokenGenerateResponse.OptOutResponse, tokenGenerateResponse); - assertNotNull(advertisingTokenRequest.rawUidIdentity); + assertNotNull(advertisingTokenRequest.rawUid); final TokenRefreshRequest tokenRefreshRequest = this.tokenEncoder.decodeRefreshToken(tokenGenerateResponse.getRefreshToken()); reset(shutdownHandler); diff --git a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java index cdc401ecd..6ee61fe3c 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java @@ -5,7 +5,7 @@ import ch.qos.logback.core.read.ListAppender; import com.uid2.operator.model.*; import com.uid2.operator.model.IdentityScope; -import com.uid2.operator.model.userIdentity.FirstLevelHashIdentity; +import com.uid2.operator.model.identities.FirstLevelHash; import com.uid2.operator.monitoring.IStatsCollectorQueue; import com.uid2.operator.monitoring.TokenResponseStatsCollector; import com.uid2.operator.service.*; @@ -818,7 +818,7 @@ private AdvertisingTokenRequest validateAndGetToken(EncryptedTokenEncoder encode validateAdvertisingToken(advertisingTokenString, getTokenVersion(), getIdentityScope(), identityType); AdvertisingTokenRequest advertisingTokenRequest = encoder.decodeAdvertisingToken(advertisingTokenString); if (getTokenVersion() == TokenVersion.V4) { - assertEquals(identityType, advertisingTokenRequest.rawUidIdentity.identityType); + assertEquals(identityType, advertisingTokenRequest.rawUid.identityType()); } return advertisingTokenRequest; } @@ -850,8 +850,8 @@ public static void validateAdvertisingToken(String advertisingTokenString, Token TokenRefreshRequest decodeRefreshToken(EncryptedTokenEncoder encoder, String refreshTokenString, IdentityType identityType) { TokenRefreshRequest tokenRefreshRequest = encoder.decodeRefreshToken(refreshTokenString); - assertEquals(getIdentityScope(), tokenRefreshRequest.firstLevelHashIdentity.identityScope); - assertEquals(identityType, tokenRefreshRequest.firstLevelHashIdentity.identityType); + assertEquals(getIdentityScope(), tokenRefreshRequest.firstLevelHash.identityScope()); + assertEquals(identityType, tokenRefreshRequest.firstLevelHash.identityType()); return tokenRefreshRequest; } @@ -864,7 +864,7 @@ void identityMapNewClientNoPolicySpecified(String apiVersion, Vertx vertx, Vertx setupKeys(); // the clock value shouldn't matter here - when(optOutStore.getLatestEntry(any(FirstLevelHashIdentity.class))) + when(optOutStore.getLatestEntry(any(FirstLevelHash.class))) .thenReturn(now.minus(1, ChronoUnit.HOURS)); JsonObject req = new JsonObject(); @@ -891,7 +891,7 @@ void identityMapNewClientWrongPolicySpecified(String apiVersion, String policyPa setupSalts(); setupKeys(); // the clock value shouldn't matter here - when(optOutStore.getLatestEntry(any(FirstLevelHashIdentity.class))) + when(optOutStore.getLatestEntry(any(FirstLevelHash.class))) .thenReturn(now.minus(1, ChronoUnit.HOURS)); JsonObject req = new JsonObject(); JsonArray emails = new JsonArray(); @@ -1171,8 +1171,8 @@ void tokenGenerateOptOutToken(String policyParameterKey, String identity, Identi firstLevelSalt, rotatingSalt123.getSalt()); final byte[] firstLevelHash = TokenUtils.getFirstLevelHashFromIdentity(optOutTokenInput.getNormalized(), firstLevelSalt); - assertArrayEquals(rawUid, advertisingTokenRequest.rawUidIdentity.rawUid); - assertArrayEquals(firstLevelHash, tokenRefreshRequest.firstLevelHashIdentity.firstLevelHash); + assertArrayEquals(rawUid, advertisingTokenRequest.rawUid.rawUid()); + assertArrayEquals(firstLevelHash, tokenRefreshRequest.firstLevelHash.firstLevelHash()); String advertisingTokenString = body.getString("advertising_token"); final Instant now = Instant.now(); @@ -1244,7 +1244,7 @@ public void assertAdvertisingTokenRefreshTokenRequests(AdvertisingTokenRequest a assertEquals(expectedClientSiteId, advertisingTokenRequest.sourcePublisher.siteId); assertEquals(expectedClientSiteId, tokenRefreshRequest.sourcePublisher.siteId); - assertArrayEquals(expectedRawUidIdentity, advertisingTokenRequest.rawUidIdentity.rawUid); + assertArrayEquals(expectedRawUidIdentity, advertisingTokenRequest.rawUid.rawUid()); verifyPrivacyBits(expectedPrivacyBits, advertisingTokenRequest, tokenRefreshRequest); verifyFirstLevelHashIdentityAndEstablishedAt(firstLevelHashIdentity, tokenRefreshRequest, identityResponse, advertisingTokenRequest.establishedAt); @@ -1316,7 +1316,7 @@ void tokenGenerateThenRefresh(String apiVersion, Vertx vertx, VertxTestContext t TokenRefreshRequest firstTokenRefreshRequest = decodeRefreshToken(encoder, bodyJson.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), IdentityType.Email); - assertEquals(firstAdvertisingTokenRequest.establishedAt, firstTokenRefreshRequest.firstLevelHashIdentity.establishedAt); + assertEquals(firstAdvertisingTokenRequest.establishedAt, firstTokenRefreshRequest.firstLevelHash.establishedAt()); when(this.optOutStore.getLatestEntry(any())).thenReturn(null); @@ -1416,13 +1416,13 @@ void tokenGenerateThenRefreshSaltsExpired(String apiVersion, Vertx vertx, VertxT assertFalse(advertisingTokenRequest.privacyBits.isClientSideTokenGenerated()); assertFalse(advertisingTokenRequest.privacyBits.isClientSideTokenOptedOut()); assertEquals(clientSiteId, advertisingTokenRequest.sourcePublisher.siteId); - assertArrayEquals(getRawUidFromIdentity(IdentityType.Email, emailAddress, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenRequest.rawUidIdentity.rawUid); + assertArrayEquals(getRawUidFromIdentity(IdentityType.Email, emailAddress, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenRequest.rawUid.rawUid()); String refreshTokenStringNew = refreshBody.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"); assertNotEquals(genRefreshToken, refreshTokenStringNew); TokenRefreshRequest tokenRefreshRequest = decodeRefreshToken(encoder, refreshTokenStringNew, IdentityType.Email); assertEquals(clientSiteId, tokenRefreshRequest.sourcePublisher.siteId); - assertArrayEquals(TokenUtils.getFirstLevelHashFromIdentity(emailAddress, firstLevelSalt), tokenRefreshRequest.firstLevelHashIdentity.firstLevelHash); + assertArrayEquals(TokenUtils.getFirstLevelHashFromIdentity(emailAddress, firstLevelSalt), tokenRefreshRequest.firstLevelHash.firstLevelHash()); assertEqualsClose(now.plusMillis(identityExpiresAfter.toMillis()), Instant.ofEpochMilli(refreshBody.getLong("identity_expires")), 10); assertEqualsClose(now.plusMillis(refreshExpiresAfter.toMillis()), Instant.ofEpochMilli(refreshBody.getLong("refresh_expires")), 10); @@ -1591,11 +1591,11 @@ void tokenGenerateUsingCustomSiteKey(String apiVersion, Vertx vertx, VertxTestCo AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, body, IdentityType.Email); assertEquals(clientSiteId, advertisingTokenRequest.sourcePublisher.siteId); - assertArrayEquals(getRawUidFromIdentity(IdentityType.Email, emailAddress, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenRequest.rawUidIdentity.rawUid); + assertArrayEquals(getRawUidFromIdentity(IdentityType.Email, emailAddress, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenRequest.rawUid.rawUid()); TokenRefreshRequest tokenRefreshRequest = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), IdentityType.Email); assertEquals(clientSiteId, tokenRefreshRequest.sourcePublisher.siteId); - assertArrayEquals(TokenUtils.getFirstLevelHashFromIdentity(emailAddress, firstLevelSalt), tokenRefreshRequest.firstLevelHashIdentity.firstLevelHash); + assertArrayEquals(TokenUtils.getFirstLevelHashFromIdentity(emailAddress, firstLevelSalt), tokenRefreshRequest.firstLevelHash.firstLevelHash()); testContext.completeNow(); }); @@ -1630,11 +1630,11 @@ void tokenGenerateSaltsExpired(String apiVersion, Vertx vertx, VertxTestContext assertFalse(advertisingTokenRequest.privacyBits.isClientSideTokenGenerated()); assertFalse(advertisingTokenRequest.privacyBits.isClientSideTokenOptedOut()); assertEquals(clientSiteId, advertisingTokenRequest.sourcePublisher.siteId); - assertArrayEquals(getRawUidFromIdentity(IdentityType.Email, emailAddress, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenRequest.rawUidIdentity.rawUid); + assertArrayEquals(getRawUidFromIdentity(IdentityType.Email, emailAddress, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenRequest.rawUid.rawUid()); TokenRefreshRequest tokenRefreshRequest = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), IdentityType.Email); assertEquals(clientSiteId, tokenRefreshRequest.sourcePublisher.siteId); - assertArrayEquals(TokenUtils.getFirstLevelHashFromIdentity(emailAddress, firstLevelSalt), tokenRefreshRequest.firstLevelHashIdentity.firstLevelHash); + assertArrayEquals(TokenUtils.getFirstLevelHashFromIdentity(emailAddress, firstLevelSalt), tokenRefreshRequest.firstLevelHash.firstLevelHash()); assertEqualsClose(now.plusMillis(identityExpiresAfter.toMillis()), Instant.ofEpochMilli(body.getLong("identity_expires")), 10); assertEqualsClose(now.plusMillis(refreshExpiresAfter.toMillis()), Instant.ofEpochMilli(body.getLong("refresh_expires")), 10); @@ -2555,11 +2555,11 @@ void verifyFirstLevelHashIdentityAndEstablishedAt(byte[] expectedFirstLevelHash, JsonObject receivedJsonBody, Instant expectedEstablishedTime) { - assertArrayEquals(expectedFirstLevelHash, tokenRefreshRequest.firstLevelHashIdentity.firstLevelHash); - assertEquals(expectedEstablishedTime, tokenRefreshRequest.firstLevelHashIdentity.establishedAt); - assertTrue(tokenRefreshRequest.firstLevelHashIdentity.establishedAt.toEpochMilli() < receivedJsonBody.getLong("identity_expires") ); - assertTrue(tokenRefreshRequest.firstLevelHashIdentity.establishedAt.toEpochMilli() < receivedJsonBody.getLong("refresh_expires") ); - assertTrue(tokenRefreshRequest.firstLevelHashIdentity.establishedAt.toEpochMilli() < receivedJsonBody.getLong("refresh_from") ); + assertArrayEquals(expectedFirstLevelHash, tokenRefreshRequest.firstLevelHash.firstLevelHash()); + assertEquals(expectedEstablishedTime, tokenRefreshRequest.firstLevelHash.establishedAt()); + assertTrue(tokenRefreshRequest.firstLevelHash.establishedAt().toEpochMilli() < receivedJsonBody.getLong("identity_expires") ); + assertTrue(tokenRefreshRequest.firstLevelHash.establishedAt().toEpochMilli() < receivedJsonBody.getLong("refresh_expires") ); + assertTrue(tokenRefreshRequest.firstLevelHash.establishedAt().toEpochMilli() < receivedJsonBody.getLong("refresh_from") ); } @ParameterizedTest @@ -3101,7 +3101,7 @@ void tokenGenerateRespectOptOutOption(String policyParameterKey, Vertx vertx, Ve setupKeys(); // the clock value shouldn't matter here - when(optOutStore.getLatestEntry(any(FirstLevelHashIdentity.class))) + when(optOutStore.getLatestEntry(any(FirstLevelHash.class))) .thenReturn(now.minus(1, ChronoUnit.HOURS)); JsonObject req = new JsonObject(); @@ -3132,7 +3132,7 @@ void identityMapDefaultOption(String apiVersion, Vertx vertx, VertxTestContext t setupKeys(); // the clock value shouldn't matter here - when(optOutStore.getLatestEntry(any(FirstLevelHashIdentity.class))) + when(optOutStore.getLatestEntry(any(FirstLevelHash.class))) .thenReturn(now.minus(1, ChronoUnit.HOURS)); JsonObject req = new JsonObject(); @@ -3173,7 +3173,7 @@ void identityMapRespectOptOutOption(String apiVersion, String policyParameterKey setupKeys(); // the clock value shouldn't matter here - when(optOutStore.getLatestEntry(any(FirstLevelHashIdentity.class))) + when(optOutStore.getLatestEntry(any(FirstLevelHash.class))) .thenReturn(now.minus(1, ChronoUnit.HOURS)); JsonObject req = new JsonObject(); @@ -4064,11 +4064,11 @@ void cstgUserOptsOutAfterTokenGenerate(String id, IdentityType identityType, Ver final Tuple.Tuple2 data = createClientSideTokenGenerateRequest(identityType, id, Instant.now().toEpochMilli()); // When we generate the token the user hasn't opted out. - when(optOutStore.getLatestEntry(any(FirstLevelHashIdentity.class))) + when(optOutStore.getLatestEntry(any(FirstLevelHash.class))) .thenReturn(null); final EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); - final ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(FirstLevelHashIdentity.class); + final ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(FirstLevelHash.class); sendCstg(vertx, "v2/token/client-generate", @@ -4080,7 +4080,7 @@ void cstgUserOptsOutAfterTokenGenerate(String id, IdentityType identityType, Ver response -> { verify(optOutStore, times(1)).getLatestEntry(argumentCaptor.capture()); assertArrayEquals(TokenUtils.getFirstLevelHashFromIdentity(id, firstLevelSalt), - argumentCaptor.getValue().firstLevelHash); + argumentCaptor.getValue().firstLevelHash()); assertEquals("success", response.getString("status")); final JsonObject genBody = response.getJsonObject("body"); @@ -4091,7 +4091,7 @@ void cstgUserOptsOutAfterTokenGenerate(String id, IdentityType identityType, Ver assertAreClientSideGeneratedTokens(advertisingTokenRequest, tokenRefreshRequest, clientSideTokenGenerateSiteId, identityType, id); // When we refresh the token the user has opted out. - when(optOutStore.getLatestEntry(any(FirstLevelHashIdentity.class))) + when(optOutStore.getLatestEntry(any(FirstLevelHash.class))) .thenReturn(advertisingTokenRequest.establishedAt.plusSeconds(1)); sendTokenRefresh("v2", vertx, testContext, genBody.getString("refresh_token"), genBody.getString("refresh_response_key"), 200, refreshRespJson -> { @@ -4120,11 +4120,11 @@ void cstgSuccessForBothOptedAndNonOptedOutTest(boolean optOutExpected, String id if(optOutExpected) { - when(optOutStore.getLatestEntry(any(FirstLevelHashIdentity.class))) + when(optOutStore.getLatestEntry(any(FirstLevelHash.class))) .thenReturn(Instant.now().minus(1, ChronoUnit.HOURS)); } else { //not expectedOptedOut - when(optOutStore.getLatestEntry(any(FirstLevelHashIdentity.class))) + when(optOutStore.getLatestEntry(any(FirstLevelHash.class))) .thenReturn(null); } @@ -4325,8 +4325,8 @@ private void assertAreClientSideGeneratedTokens(AdvertisingTokenRequest advertis () -> assertEquals(siteId, advertisingTokenRequest.sourcePublisher.siteId, "Advertising token site ID is incorrect"), () -> assertEquals(siteId, tokenRefreshRequest.sourcePublisher.siteId, "Refresh token site ID is incorrect"), - () -> assertArrayEquals(rawUid, advertisingTokenRequest.rawUidIdentity.rawUid, "Advertising token ID is incorrect"), - () -> assertArrayEquals(firstLevelHash, tokenRefreshRequest.firstLevelHashIdentity.firstLevelHash, "Refresh token ID is incorrect") + () -> assertArrayEquals(rawUid, advertisingTokenRequest.rawUid.rawUid(), "Advertising token ID is incorrect"), + () -> assertArrayEquals(firstLevelHash, tokenRefreshRequest.firstLevelHash.firstLevelHash(), "Refresh token ID is incorrect") ); } diff --git a/src/test/java/com/uid2/operator/benchmark/BenchmarkCommon.java b/src/test/java/com/uid2/operator/benchmark/BenchmarkCommon.java index c5b062ad8..6ab2447e7 100644 --- a/src/test/java/com/uid2/operator/benchmark/BenchmarkCommon.java +++ b/src/test/java/com/uid2/operator/benchmark/BenchmarkCommon.java @@ -3,8 +3,8 @@ import com.uid2.operator.Const; import com.uid2.operator.Main; import com.uid2.operator.model.*; -import com.uid2.operator.model.userIdentity.FirstLevelHashIdentity; -import com.uid2.operator.model.userIdentity.HashedDiiIdentity; +import com.uid2.operator.model.identities.FirstLevelHash; +import com.uid2.operator.model.identities.HashedDii; import com.uid2.operator.service.EncryptedTokenEncoder; import com.uid2.operator.service.IUIDOperatorService; import com.uid2.operator.service.UIDOperatorService; @@ -150,12 +150,12 @@ static ICloudStorage make1mOptOutEntryStorage(String salt, List out_gene return storage; } - static HashedDiiIdentity[] createHashedDiiIdentities() { - HashedDiiIdentity[] arr = new HashedDiiIdentity[65536]; + static HashedDii[] createHashedDiiIdentities() { + HashedDii[] arr = new HashedDii[65536]; for (int i = 0; i < 65536; i++) { final byte[] diiHash = new byte[33]; new Random().nextBytes(diiHash); - arr[i] = new HashedDiiIdentity(IdentityScope.UID2, IdentityType.Email, diiHash); + arr[i] = new HashedDii(IdentityScope.UID2, IdentityType.Email, diiHash); } return arr; } @@ -188,14 +188,14 @@ public StaticOptOutStore(ICloudStorage storage, JsonObject jsonConfig, Collectio } @Override - public Instant getLatestEntry(FirstLevelHashIdentity firstLevelHashIdentity) { - long epochSecond = this.snapshot.getOptOutTimestamp(firstLevelHashIdentity.firstLevelHash); + public Instant getLatestEntry(FirstLevelHash firstLevelHash) { + long epochSecond = this.snapshot.getOptOutTimestamp(firstLevelHash.firstLevelHash()); Instant instant = epochSecond > 0 ? Instant.ofEpochSecond(epochSecond) : null; return instant; } @Override - public void addEntry(FirstLevelHashIdentity firstLevelHashIdentity, byte[] advertisingId, Handler> handler) { + public void addEntry(FirstLevelHash firstLevelHash, byte[] advertisingId, Handler> handler) { // noop } diff --git a/src/test/java/com/uid2/operator/benchmark/IdentityMapBenchmark.java b/src/test/java/com/uid2/operator/benchmark/IdentityMapBenchmark.java index 63652ab09..7151d8c7e 100644 --- a/src/test/java/com/uid2/operator/benchmark/IdentityMapBenchmark.java +++ b/src/test/java/com/uid2/operator/benchmark/IdentityMapBenchmark.java @@ -1,7 +1,7 @@ package com.uid2.operator.benchmark; import com.uid2.operator.model.*; -import com.uid2.operator.model.userIdentity.HashedDiiIdentity; +import com.uid2.operator.model.identities.HashedDii; import com.uid2.operator.service.IUIDOperatorService; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; @@ -11,7 +11,7 @@ public class IdentityMapBenchmark { private static final IUIDOperatorService uidService; - private static final HashedDiiIdentity[] hashedDiiIdentities; + private static final HashedDii[] hashedDiiIdentities; private static int idx = 0; static { diff --git a/src/test/java/com/uid2/operator/benchmark/TokenEndecBenchmark.java b/src/test/java/com/uid2/operator/benchmark/TokenEndecBenchmark.java index b736c1a44..6b7e2c01c 100644 --- a/src/test/java/com/uid2/operator/benchmark/TokenEndecBenchmark.java +++ b/src/test/java/com/uid2/operator/benchmark/TokenEndecBenchmark.java @@ -1,7 +1,7 @@ package com.uid2.operator.benchmark; import com.uid2.operator.model.*; -import com.uid2.operator.model.userIdentity.HashedDiiIdentity; +import com.uid2.operator.model.identities.HashedDii; import com.uid2.operator.service.EncryptedTokenEncoder; import com.uid2.operator.service.IUIDOperatorService; import org.openjdk.jmh.annotations.Benchmark; @@ -14,7 +14,7 @@ public class TokenEndecBenchmark { private static final IUIDOperatorService uidService; - private static final HashedDiiIdentity[] hashedDiiIdentities; + private static final HashedDii[] hashedDiiIdentities; private static final SourcePublisher publisher; private static final EncryptedTokenEncoder encoder; private static final TokenGenerateResponse[] generatedTokens; From a8f0915b2c65bf043ee5a7b0787d29e04c6f086b Mon Sep 17 00:00:00 2001 From: Sunny Wu Date: Mon, 9 Dec 2024 19:04:22 +1100 Subject: [PATCH 154/431] 1. Renamed IdentityType to DiiType 2. In InputUtil, renamed Identity* member variable/methods to Dii* to indicate it's Dii 3. Change TokenUtils class getIdentityHash to getDiiHash and same for getDiiHashString --- .../DiiType.java} | 8 +- .../model/identities/FirstLevelHash.java | 7 +- .../operator/model/identities/HashedDii.java | 8 +- .../{ => model/identities}/IdentityConst.java | 2 +- .../model/{ => identities}/IdentityScope.java | 2 +- .../operator/model/identities/RawUid.java | 7 +- .../service/EncryptedTokenEncoder.java | 32 +-- .../com/uid2/operator/service/InputUtil.java | 57 ++--- .../com/uid2/operator/service/TokenUtils.java | 22 +- .../operator/service/UIDOperatorService.java | 44 ++-- .../uid2/operator/service/V2RequestUtil.java | 4 +- .../operator/vertx/UIDOperatorVerticle.java | 79 +++---- .../uid2/operator/vertx/V2PayloadHandler.java | 2 +- .../operator/EUIDOperatorVerticleTest.java | 2 +- .../uid2/operator/InputNormalizationTest.java | 4 +- .../com/uid2/operator/TokenEncodingTest.java | 8 +- .../uid2/operator/UIDOperatorServiceTest.java | 78 +++---- .../operator/UIDOperatorVerticleTest.java | 219 +++++++++--------- .../com/uid2/operator/V2RequestUtilTest.java | 2 +- .../operator/benchmark/BenchmarkCommon.java | 4 +- 20 files changed, 293 insertions(+), 298 deletions(-) rename src/main/java/com/uid2/operator/model/{IdentityType.java => identities/DiiType.java} (67%) rename src/main/java/com/uid2/operator/{ => model/identities}/IdentityConst.java (97%) rename src/main/java/com/uid2/operator/model/{ => identities}/IdentityScope.java (94%) diff --git a/src/main/java/com/uid2/operator/model/IdentityType.java b/src/main/java/com/uid2/operator/model/identities/DiiType.java similarity index 67% rename from src/main/java/com/uid2/operator/model/IdentityType.java rename to src/main/java/com/uid2/operator/model/identities/DiiType.java index b64817df5..062b55d35 100644 --- a/src/main/java/com/uid2/operator/model/IdentityType.java +++ b/src/main/java/com/uid2/operator/model/identities/DiiType.java @@ -1,15 +1,15 @@ -package com.uid2.operator.model; +package com.uid2.operator.model.identities; import com.uid2.operator.vertx.ClientInputValidationException; -public enum IdentityType { +public enum DiiType { Email(0), Phone(1); public final int value; - IdentityType(int value) { this.value = value; } + DiiType(int value) { this.value = value; } - public static IdentityType fromValue(int value) { + public static DiiType fromValue(int value) { switch (value) { case 0: return Email; case 1: return Phone; diff --git a/src/main/java/com/uid2/operator/model/identities/FirstLevelHash.java b/src/main/java/com/uid2/operator/model/identities/FirstLevelHash.java index d46cc0ad5..49b2728f4 100644 --- a/src/main/java/com/uid2/operator/model/identities/FirstLevelHash.java +++ b/src/main/java/com/uid2/operator/model/identities/FirstLevelHash.java @@ -1,8 +1,5 @@ package com.uid2.operator.model.identities; -import com.uid2.operator.model.IdentityScope; -import com.uid2.operator.model.IdentityType; - import java.time.Instant; import java.util.Arrays; @@ -10,13 +7,13 @@ * Contains a first level salted hash computed from Hashed DII (email/phone number) * @param establishedAt for brand new token generation, it should be the time it is generated if the first level hash is from token/refresh call, it will be when the raw UID was originally created in the earliest token generation */ -public record FirstLevelHash(IdentityScope identityScope, IdentityType identityType, byte[] firstLevelHash, +public record FirstLevelHash(IdentityScope identityScope, DiiType diiType, byte[] firstLevelHash, Instant establishedAt) { // explicitly not checking establishedAt - this is only for making sure the first level hash matches a new input public boolean matches(FirstLevelHash that) { return this.identityScope.equals(that.identityScope) && - this.identityType.equals(that.identityType) && + this.diiType.equals(that.diiType) && Arrays.equals(this.firstLevelHash, that.firstLevelHash); } } diff --git a/src/main/java/com/uid2/operator/model/identities/HashedDii.java b/src/main/java/com/uid2/operator/model/identities/HashedDii.java index da040c066..64c7bbf0f 100644 --- a/src/main/java/com/uid2/operator/model/identities/HashedDii.java +++ b/src/main/java/com/uid2/operator/model/identities/HashedDii.java @@ -1,9 +1,7 @@ package com.uid2.operator.model.identities; -import com.uid2.operator.model.IdentityScope; -import com.uid2.operator.model.IdentityType; - -// Contains a hash DII, +// Contains a hash Directly Identifying Information (DII) (email or phone) see https://unifiedid.com/docs/ref-info/glossary-uid#gl-dii // This hash can either be computed from a raw email/phone number DII input or provided by the UID Participant directly -public record HashedDii(IdentityScope identityScope, IdentityType identityType, byte[] hashedDii) { +// +public record HashedDii(IdentityScope identityScope, DiiType diiType, byte[] hashedDii) { } diff --git a/src/main/java/com/uid2/operator/IdentityConst.java b/src/main/java/com/uid2/operator/model/identities/IdentityConst.java similarity index 97% rename from src/main/java/com/uid2/operator/IdentityConst.java rename to src/main/java/com/uid2/operator/model/identities/IdentityConst.java index 402a99d88..63fa62f96 100644 --- a/src/main/java/com/uid2/operator/IdentityConst.java +++ b/src/main/java/com/uid2/operator/model/identities/IdentityConst.java @@ -1,4 +1,4 @@ -package com.uid2.operator; +package com.uid2.operator.model.identities; import com.uid2.operator.service.EncodingUtils; diff --git a/src/main/java/com/uid2/operator/model/IdentityScope.java b/src/main/java/com/uid2/operator/model/identities/IdentityScope.java similarity index 94% rename from src/main/java/com/uid2/operator/model/IdentityScope.java rename to src/main/java/com/uid2/operator/model/identities/IdentityScope.java index 0bff1edc1..3dc19a764 100644 --- a/src/main/java/com/uid2/operator/model/IdentityScope.java +++ b/src/main/java/com/uid2/operator/model/identities/IdentityScope.java @@ -1,4 +1,4 @@ -package com.uid2.operator.model; +package com.uid2.operator.model.identities; import com.uid2.operator.vertx.ClientInputValidationException; diff --git a/src/main/java/com/uid2/operator/model/identities/RawUid.java b/src/main/java/com/uid2/operator/model/identities/RawUid.java index 775c7c46a..4ae619d00 100644 --- a/src/main/java/com/uid2/operator/model/identities/RawUid.java +++ b/src/main/java/com/uid2/operator/model/identities/RawUid.java @@ -1,16 +1,13 @@ package com.uid2.operator.model.identities; -import com.uid2.operator.model.IdentityScope; -import com.uid2.operator.model.IdentityType; - import java.util.Arrays; // A raw UID is stored inside -public record RawUid(IdentityScope identityScope, IdentityType identityType, byte[] rawUid) { +public record RawUid(IdentityScope identityScope, DiiType diiType, byte[] rawUid) { public boolean matches(RawUid that) { return this.identityScope.equals(that.identityScope) && - this.identityType.equals(that.identityType) && + this.diiType.equals(that.diiType) && Arrays.equals(this.rawUid, that.rawUid); } } diff --git a/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java b/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java index 87f86b244..d3f6ad9cf 100644 --- a/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java +++ b/src/main/java/com/uid2/operator/service/EncryptedTokenEncoder.java @@ -1,7 +1,9 @@ package com.uid2.operator.service; import com.uid2.operator.model.*; +import com.uid2.operator.model.identities.DiiType; import com.uid2.operator.model.identities.FirstLevelHash; +import com.uid2.operator.model.identities.IdentityScope; import com.uid2.operator.model.identities.RawUid; import com.uid2.operator.util.PrivacyBits; import com.uid2.operator.vertx.ClientInputValidationException; @@ -69,7 +71,7 @@ private byte[] encodeIntoAdvertisingTokenV3(AdvertisingTokenRequest t, KeysetKey masterPayload.appendBytes(AesGcm.encrypt(sitePayload.getBytes(), siteKey).getPayload()); final Buffer b = Buffer.buffer(164); - b.appendByte(encodeIdentityTypeV3(t.rawUid.identityScope(), t.rawUid.identityType())); + b.appendByte(encodeIdentityTypeV3(t.rawUid.identityScope(), t.rawUid.diiType())); b.appendByte((byte) t.version.rawVersion); b.appendInt(masterKey.getId()); b.appendBytes(AesGcm.encrypt(masterPayload.getBytes(), masterKey).getPayload()); @@ -129,7 +131,7 @@ private TokenRefreshRequest decodeRefreshTokenV2(Buffer b) { TokenVersion.V2, createdAt, validTill, new OperatorIdentity(0, OperatorType.Service, 0, 0), new SourcePublisher(siteId), - new FirstLevelHash(IdentityScope.UID2, IdentityType.Email, identity, + new FirstLevelHash(IdentityScope.UID2, DiiType.Email, identity, Instant.ofEpochMilli(establishedMillis)), privacyBits); } @@ -152,19 +154,19 @@ private TokenRefreshRequest decodeRefreshTokenV3(Buffer b, byte[] bytes) { final PrivacyBits privacyBits = PrivacyBits.fromInt(b2.getInt(45)); final Instant establishedAt = Instant.ofEpochMilli(b2.getLong(49)); final IdentityScope identityScope = decodeIdentityScopeV3(b2.getByte(57)); - final IdentityType identityType = decodeIdentityTypeV3(b2.getByte(57)); + final DiiType diiType = decodeIdentityTypeV3(b2.getByte(57)); final byte[] firstLevelHash = b2.getBytes(58, 90); if (identityScope != decodeIdentityScopeV3(b.getByte(0))) { throw new ClientInputValidationException("Failed to decode refreshTokenV3: Identity scope mismatch"); } - if (identityType != decodeIdentityTypeV3(b.getByte(0))) { + if (diiType != decodeIdentityTypeV3(b.getByte(0))) { throw new ClientInputValidationException("Failed to decode refreshTokenV3: Identity type mismatch"); } return new TokenRefreshRequest( TokenVersion.V3, createdAt, expiresAt, operatorIdentity, sourcePublisher, - new FirstLevelHash(identityScope, identityType, firstLevelHash, establishedAt), + new FirstLevelHash(identityScope, diiType, firstLevelHash, establishedAt), privacyBits); } @@ -233,7 +235,7 @@ public AdvertisingTokenRequest decodeAdvertisingTokenV2(Buffer b) { Instant.ofEpochMilli(expiresMillis), new OperatorIdentity(0, OperatorType.Service, 0, masterKeyId), new SourcePublisher(siteId, siteKeyId, 0), - new RawUid(IdentityScope.UID2, IdentityType.Email, rawUid), + new RawUid(IdentityScope.UID2, DiiType.Email, rawUid), privacyBits, Instant.ofEpochMilli(establishedMillis) ); @@ -262,21 +264,21 @@ public AdvertisingTokenRequest decodeAdvertisingTokenV3orV4(Buffer b, byte[] byt final Instant refreshedAt = Instant.ofEpochMilli(sitePayload.getLong(28)); final byte[] rawUid = sitePayload.slice(36, sitePayload.length()).getBytes(); final IdentityScope identityScope = rawUid.length == 32 ? IdentityScope.UID2 : decodeIdentityScopeV3(rawUid[0]); - final IdentityType identityType = rawUid.length == 32 ? IdentityType.Email : decodeIdentityTypeV3(rawUid[0]); + final DiiType diiType = rawUid.length == 32 ? DiiType.Email : decodeIdentityTypeV3(rawUid[0]); if (rawUid.length > 32) { if (identityScope != decodeIdentityScopeV3(b.getByte(0))) { throw new ClientInputValidationException("Failed decoding advertisingTokenV3: Identity scope mismatch"); } - if (identityType != decodeIdentityTypeV3(b.getByte(0))) { + if (diiType != decodeIdentityTypeV3(b.getByte(0))) { throw new ClientInputValidationException("Failed decoding advertisingTokenV3: Identity type mismatch"); } } return new AdvertisingTokenRequest( tokenVersion, createdAt, expiresAt, operatorIdentity, sourcePublisher, - new RawUid(identityScope, identityType, rawUid), + new RawUid(identityScope, diiType, rawUid), privacyBits, establishedAt ); } @@ -326,11 +328,11 @@ public byte[] encodeIntoRefreshTokenV3(TokenRefreshRequest t, KeysetKey serviceK encodePublisherRequesterV3(refreshPayload, t.sourcePublisher); refreshPayload.appendInt(t.privacyBits.getAsInt()); refreshPayload.appendLong(t.firstLevelHash.establishedAt().toEpochMilli()); - refreshPayload.appendByte(encodeIdentityTypeV3(t.firstLevelHash.identityScope(), t.firstLevelHash.identityType())); + refreshPayload.appendByte(encodeIdentityTypeV3(t.firstLevelHash.identityScope(), t.firstLevelHash.diiType())); refreshPayload.appendBytes(t.firstLevelHash.firstLevelHash()); final Buffer b = Buffer.buffer(124); - b.appendByte(encodeIdentityTypeV3(t.firstLevelHash.identityScope(), t.firstLevelHash.identityType())); + b.appendByte(encodeIdentityTypeV3(t.firstLevelHash.identityScope(), t.firstLevelHash.diiType())); b.appendByte((byte) t.version.rawVersion); b.appendInt(serviceKey.getId()); b.appendBytes(AesGcm.encrypt(refreshPayload.getBytes(), serviceKey).getPayload()); @@ -402,8 +404,8 @@ private byte[] encryptIdentityV2(SourcePublisher sourcePublisher, byte[] id, Pri } } - static private byte encodeIdentityTypeV3(IdentityScope identityScope, IdentityType identityType) { - return (byte) (TokenUtils.encodeIdentityScope(identityScope) | (identityType.value << 2) | 3); + static private byte encodeIdentityTypeV3(IdentityScope identityScope, DiiType diiType) { + return (byte) (TokenUtils.encodeIdentityScope(identityScope) | (diiType.value << 2) | 3); // "| 3" is used so that the 2nd char matches the version when V3 or higher. Eg "3" for V3 and "4" for V4 } @@ -411,8 +413,8 @@ static private IdentityScope decodeIdentityScopeV3(byte value) { return IdentityScope.fromValue((value & 0x10) >> 4); } - static private IdentityType decodeIdentityTypeV3(byte value) { - return IdentityType.fromValue((value & 0xf) >> 2); + static private DiiType decodeIdentityTypeV3(byte value) { + return DiiType.fromValue((value & 0xf) >> 2); } static void encodePublisherRequesterV3(Buffer b, SourcePublisher sourcePublisher) { diff --git a/src/main/java/com/uid2/operator/service/InputUtil.java b/src/main/java/com/uid2/operator/service/InputUtil.java index 9610d20f3..ab2b573b5 100644 --- a/src/main/java/com/uid2/operator/service/InputUtil.java +++ b/src/main/java/com/uid2/operator/service/InputUtil.java @@ -1,7 +1,7 @@ package com.uid2.operator.service; -import com.uid2.operator.model.IdentityScope; -import com.uid2.operator.model.IdentityType; +import com.uid2.operator.model.identities.IdentityScope; +import com.uid2.operator.model.identities.DiiType; import com.uid2.operator.model.identities.HashedDii; public class InputUtil { @@ -167,7 +167,7 @@ public static String normalizeEmailString(String email) { return addressPartToUse.append('@').append(domainPart).toString(); } - public enum IdentityInputType { + public enum DiiInputType { Raw, Hash } @@ -183,62 +183,63 @@ private static enum EmailParsingState { public static class InputVal { private final String provided; private final String normalized; - private final IdentityType identityType; - private final IdentityInputType inputType; + //Directly Identifying Information (DII) (email or phone) see https://unifiedid.com/docs/ref-info/glossary-uid#gl-dii + private final DiiType diiType; + private final DiiInputType inputType; private final boolean valid; - private final byte[] identityInput; + private final byte[] diiInput; - public InputVal(String provided, String normalized, IdentityType identityType, IdentityInputType inputType, boolean valid) { + public InputVal(String provided, String normalized, DiiType diiType, DiiInputType inputType, boolean valid) { this.provided = provided; this.normalized = normalized; - this.identityType = identityType; + this.diiType = diiType; this.inputType = inputType; this.valid = valid; if (valid) { - if (this.inputType == IdentityInputType.Raw) { - this.identityInput = TokenUtils.getIdentityHash(this.normalized); + if (this.inputType == DiiInputType.Raw) { + this.diiInput = TokenUtils.getDiiHash(this.normalized); } else { - this.identityInput = EncodingUtils.fromBase64(this.normalized); + this.diiInput = EncodingUtils.fromBase64(this.normalized); } } else { - this.identityInput = null; + this.diiInput = null; } } public static InputVal validEmail(String input, String normalized) { - return new InputVal(input, normalized, IdentityType.Email, IdentityInputType.Raw, true); + return new InputVal(input, normalized, DiiType.Email, DiiInputType.Raw, true); } public static InputVal invalidEmail(String input) { - return new InputVal(input, null, IdentityType.Email, IdentityInputType.Raw, false); + return new InputVal(input, null, DiiType.Email, DiiInputType.Raw, false); } public static InputVal validEmailHash(String input, String normalized) { - return new InputVal(input, normalized, IdentityType.Email, IdentityInputType.Hash, true); + return new InputVal(input, normalized, DiiType.Email, DiiInputType.Hash, true); } public static InputVal invalidEmailHash(String input) { - return new InputVal(input, null, IdentityType.Email, IdentityInputType.Hash, false); + return new InputVal(input, null, DiiType.Email, DiiInputType.Hash, false); } public static InputVal validPhone(String input, String normalized) { - return new InputVal(input, normalized, IdentityType.Phone, IdentityInputType.Raw, true); + return new InputVal(input, normalized, DiiType.Phone, DiiInputType.Raw, true); } public static InputVal invalidPhone(String input) { - return new InputVal(input, null, IdentityType.Phone, IdentityInputType.Raw, false); + return new InputVal(input, null, DiiType.Phone, DiiInputType.Raw, false); } public static InputVal validPhoneHash(String input, String normalized) { - return new InputVal(input, normalized, IdentityType.Phone, IdentityInputType.Hash, true); + return new InputVal(input, normalized, DiiType.Phone, DiiInputType.Hash, true); } public static InputVal invalidPhoneHash(String input) { - return new InputVal(input, null, IdentityType.Phone, IdentityInputType.Hash, false); + return new InputVal(input, null, DiiType.Phone, DiiInputType.Hash, false); } - public byte[] getIdentityInput() { - return this.identityInput; + public byte[] getHashedDiiInput() { + return this.diiInput; } public String getProvided() { @@ -249,21 +250,21 @@ public String getNormalized() { return normalized; } - public IdentityType getIdentityType() { - return identityType; + public DiiType getDiiType() { + return diiType; } - public IdentityInputType getInputType() { return inputType; } + public DiiInputType getInputType() { return inputType; } public boolean isValid() { return valid; } - public HashedDii toHashedDiiIdentity(IdentityScope identityScope) { + public HashedDii toHashedDii(IdentityScope identityScope) { return new HashedDii( identityScope, - this.identityType, - getIdentityInput()); + this.diiType, + getHashedDiiInput()); } } diff --git a/src/main/java/com/uid2/operator/service/TokenUtils.java b/src/main/java/com/uid2/operator/service/TokenUtils.java index ef532e578..0860db016 100644 --- a/src/main/java/com/uid2/operator/service/TokenUtils.java +++ b/src/main/java/com/uid2/operator/service/TokenUtils.java @@ -1,18 +1,18 @@ package com.uid2.operator.service; -import com.uid2.operator.model.IdentityScope; -import com.uid2.operator.model.IdentityType; +import com.uid2.operator.model.identities.IdentityScope; +import com.uid2.operator.model.identities.DiiType; import java.util.HashSet; import java.util.Set; public class TokenUtils { - public static byte[] getIdentityHash(String identityString) { + public static byte[] getDiiHash(String identityString) { return EncodingUtils.getSha256Bytes(identityString); } - public static String getIdentityHashString(String identityString) { - return EncodingUtils.toBase64String(getIdentityHash(identityString)); + public static String getDiiHashString(String identityString) { + return EncodingUtils.toBase64String(getDiiHash(identityString)); } public static byte[] getFirstLevelHash(byte[] identityHash, String firstLevelSalt) { @@ -20,7 +20,7 @@ public static byte[] getFirstLevelHash(byte[] identityHash, String firstLevelSal } public static byte[] getFirstLevelHashFromIdentity(String identityString, String firstLevelSalt) { - return getFirstLevelHash(getIdentityHash(identityString), firstLevelSalt); + return getFirstLevelHash(getDiiHash(identityString), firstLevelSalt); } public static byte[] getFirstLevelHashFromIdentityHash(String identityHash, String firstLevelSalt) { @@ -39,7 +39,7 @@ public static byte[] getRawUidV2FromIdentityHash(String identityString, String f return getRawUidV2(getFirstLevelHashFromIdentityHash(identityString, firstLevelSalt), rotatingSalt); } - public static byte[] getRawUidV3(IdentityScope scope, IdentityType type, byte[] firstLevelHash, String rotatingSalt) { + public static byte[] getRawUidV3(IdentityScope scope, DiiType type, byte[] firstLevelHash, String rotatingSalt) { final byte[] sha = EncodingUtils.getSha256Bytes(EncodingUtils.toBase64String(firstLevelHash), rotatingSalt); final byte[] rawUid = new byte[33]; rawUid[0] = (byte)(encodeIdentityScope(scope) | encodeIdentityType(type)); @@ -47,11 +47,11 @@ public static byte[] getRawUidV3(IdentityScope scope, IdentityType type, byte[] return rawUid; } - public static byte[] getRawUidV3FromIdentity(IdentityScope scope, IdentityType type, String identityString, String firstLevelSalt, String rotatingSalt) { + public static byte[] getRawUidV3FromIdentity(IdentityScope scope, DiiType type, String identityString, String firstLevelSalt, String rotatingSalt) { return getRawUidV3(scope, type, getFirstLevelHashFromIdentity(identityString, firstLevelSalt), rotatingSalt); } - public static byte[] getRawUidV3FromIdentityHash(IdentityScope scope, IdentityType type, String identityString, String firstLevelSalt, String rotatingSalt) { + public static byte[] getRawUidV3FromIdentityHash(IdentityScope scope, DiiType type, String identityString, String firstLevelSalt, String rotatingSalt) { return getRawUidV3(scope, type, getFirstLevelHashFromIdentityHash(identityString, firstLevelSalt), rotatingSalt); } @@ -59,8 +59,8 @@ public static byte encodeIdentityScope(IdentityScope identityScope) { return (byte) (identityScope.value << 4); } - public static byte encodeIdentityType(IdentityType identityType) { - return (byte) (identityType.value << 2); + public static byte encodeIdentityType(DiiType diiType) { + return (byte) (diiType.value << 2); } public static Set getSiteIdsUsingV4Tokens(String siteIdsUsingV4TokensInString) { diff --git a/src/main/java/com/uid2/operator/service/UIDOperatorService.java b/src/main/java/com/uid2/operator/service/UIDOperatorService.java index 703735e49..c42bde293 100644 --- a/src/main/java/com/uid2/operator/service/UIDOperatorService.java +++ b/src/main/java/com/uid2/operator/service/UIDOperatorService.java @@ -1,9 +1,7 @@ package com.uid2.operator.service; import com.uid2.operator.model.*; -import com.uid2.operator.model.identities.FirstLevelHash; -import com.uid2.operator.model.identities.HashedDii; -import com.uid2.operator.model.identities.RawUid; +import com.uid2.operator.model.identities.*; import com.uid2.operator.util.PrivacyBits; import com.uid2.shared.model.SaltEntry; import com.uid2.operator.store.IOptOutStore; @@ -24,7 +22,7 @@ import java.time.format.DateTimeFormatter; import java.util.*; -import static com.uid2.operator.IdentityConst.*; +import static com.uid2.operator.model.identities.IdentityConst.*; import static com.uid2.operator.service.TokenUtils.getSiteIdsUsingV4Tokens; public class UIDOperatorService implements IUIDOperatorService { @@ -67,18 +65,18 @@ public UIDOperatorService(JsonObject config, IOptOutStore optOutStore, ISaltProv this.identityScope = identityScope; this.saltRetrievalResponseHandler = saltRetrievalResponseHandler; - this.testOptOutIdentityForEmail = getFirstLevelHashIdentity(identityScope, IdentityType.Email, - InputUtil.normalizeEmail(OptOutIdentityForEmail).getIdentityInput(), Instant.now()); - this.testOptOutIdentityForPhone = getFirstLevelHashIdentity(identityScope, IdentityType.Phone, - InputUtil.normalizePhone(OptOutIdentityForPhone).getIdentityInput(), Instant.now()); - this.testValidateIdentityForEmail = getFirstLevelHashIdentity(identityScope, IdentityType.Email, - InputUtil.normalizeEmail(ValidateIdentityForEmail).getIdentityInput(), Instant.now()); - this.testValidateIdentityForPhone = getFirstLevelHashIdentity(identityScope, IdentityType.Phone, - InputUtil.normalizePhone(ValidateIdentityForPhone).getIdentityInput(), Instant.now()); - this.testRefreshOptOutIdentityForEmail = getFirstLevelHashIdentity(identityScope, IdentityType.Email, - InputUtil.normalizeEmail(RefreshOptOutIdentityForEmail).getIdentityInput(), Instant.now()); - this.testRefreshOptOutIdentityForPhone = getFirstLevelHashIdentity(identityScope, IdentityType.Phone, - InputUtil.normalizePhone(RefreshOptOutIdentityForPhone).getIdentityInput(), Instant.now()); + this.testOptOutIdentityForEmail = getFirstLevelHashIdentity(identityScope, DiiType.Email, + InputUtil.normalizeEmail(OptOutIdentityForEmail).getHashedDiiInput(), Instant.now()); + this.testOptOutIdentityForPhone = getFirstLevelHashIdentity(identityScope, DiiType.Phone, + InputUtil.normalizePhone(OptOutIdentityForPhone).getHashedDiiInput(), Instant.now()); + this.testValidateIdentityForEmail = getFirstLevelHashIdentity(identityScope, DiiType.Email, + InputUtil.normalizeEmail(ValidateIdentityForEmail).getHashedDiiInput(), Instant.now()); + this.testValidateIdentityForPhone = getFirstLevelHashIdentity(identityScope, DiiType.Phone, + InputUtil.normalizePhone(ValidateIdentityForPhone).getHashedDiiInput(), Instant.now()); + this.testRefreshOptOutIdentityForEmail = getFirstLevelHashIdentity(identityScope, DiiType.Email, + InputUtil.normalizeEmail(RefreshOptOutIdentityForEmail).getHashedDiiInput(), Instant.now()); + this.testRefreshOptOutIdentityForPhone = getFirstLevelHashIdentity(identityScope, DiiType.Phone, + InputUtil.normalizePhone(RefreshOptOutIdentityForPhone).getHashedDiiInput(), Instant.now()); this.operatorIdentity = new OperatorIdentity(0, OperatorType.Service, 0, 0); @@ -109,7 +107,7 @@ public TokenGenerateResponse generateIdentity(TokenGenerateRequest request) { final Instant now = EncodingUtils.NowUTCMillis(this.clock); final byte[] firstLevelHash = getFirstLevelHash(request.hashedDii.hashedDii(), now); final FirstLevelHash firstLevelHashIdentity = new FirstLevelHash( - request.hashedDii.identityScope(), request.hashedDii.identityType(), firstLevelHash, + request.hashedDii.identityScope(), request.hashedDii.diiType(), firstLevelHash, request.establishedAt); if (request.shouldCheckOptOut() && getGlobalOptOutResult(firstLevelHashIdentity, false).isOptedOut()) { @@ -227,12 +225,12 @@ public Duration getIdentityExpiryDuration() { } private FirstLevelHash getFirstLevelHashIdentity(HashedDii hashedDii, Instant asOf) { - return getFirstLevelHashIdentity(hashedDii.identityScope(), hashedDii.identityType(), hashedDii.hashedDii(), asOf); + return getFirstLevelHashIdentity(hashedDii.identityScope(), hashedDii.diiType(), hashedDii.hashedDii(), asOf); } - private FirstLevelHash getFirstLevelHashIdentity(IdentityScope identityScope, IdentityType identityType, byte[] identityHash, Instant asOf) { - final byte[] firstLevelHash = getFirstLevelHash(identityHash, asOf); - return new FirstLevelHash(identityScope, identityType, firstLevelHash, null); + private FirstLevelHash getFirstLevelHashIdentity(IdentityScope identityScope, DiiType diiType, byte[] hashedDii, Instant asOf) { + final byte[] firstLevelHash = getFirstLevelHash(hashedDii, asOf); + return new FirstLevelHash(identityScope, diiType, firstLevelHash, null); } private byte[] getFirstLevelHash(byte[] identityHash, Instant asOf) { @@ -245,7 +243,7 @@ private IdentityMapResponseItem generateRawUid(FirstLevelHash firstLevelHash, In return new IdentityMapResponseItem( this.identityV3Enabled ? TokenUtils.getRawUidV3(firstLevelHash.identityScope(), - firstLevelHash.identityType(), firstLevelHash.firstLevelHash(), rotatingSalt.getSalt()) + firstLevelHash.diiType(), firstLevelHash.firstLevelHash(), rotatingSalt.getSalt()) : TokenUtils.getRawUidV2(firstLevelHash.firstLevelHash(), rotatingSalt.getSalt()), rotatingSalt.getHashedId()); } @@ -256,7 +254,7 @@ private TokenGenerateResponse generateIdentity(SourcePublisher sourcePublisher, final IdentityMapResponseItem identityMapResponseItem = generateRawUid(firstLevelHash, nowUtc); final RawUid rawUid = new RawUid(firstLevelHash.identityScope(), - firstLevelHash.identityType(), + firstLevelHash.diiType(), identityMapResponseItem.rawUid); return this.encoder.encodeIntoIdentityResponse( diff --git a/src/main/java/com/uid2/operator/service/V2RequestUtil.java b/src/main/java/com/uid2/operator/service/V2RequestUtil.java index cd4983865..a83849e79 100644 --- a/src/main/java/com/uid2/operator/service/V2RequestUtil.java +++ b/src/main/java/com/uid2/operator/service/V2RequestUtil.java @@ -1,8 +1,7 @@ package com.uid2.operator.service; -import com.uid2.operator.model.IdentityScope; +import com.uid2.operator.model.identities.IdentityScope; import com.uid2.operator.model.KeyManager; -import com.uid2.operator.vertx.ClientInputValidationException; import com.uid2.shared.IClock; import com.uid2.shared.Utils; import com.uid2.shared.auth.ClientKey; @@ -15,7 +14,6 @@ import org.slf4j.LoggerFactory; import java.nio.charset.StandardCharsets; -import java.time.Clock; import java.time.Duration; import java.time.Instant; diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index a61af9790..045a58484 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -3,7 +3,8 @@ import com.uid2.operator.Const; import com.uid2.operator.model.*; import com.uid2.operator.model.TokenGenerateResponse; -import com.uid2.operator.model.IdentityScope; +import com.uid2.operator.model.identities.IdentityScope; +import com.uid2.operator.model.identities.DiiType; import com.uid2.operator.model.identities.HashedDii; import com.uid2.operator.monitoring.IStatsCollectorQueue; import com.uid2.operator.monitoring.StatsCollectorHandler; @@ -69,7 +70,7 @@ import java.util.function.Supplier; import java.util.stream.Collectors; -import static com.uid2.operator.IdentityConst.*; +import static com.uid2.operator.model.identities.IdentityConst.*; import static com.uid2.operator.service.ResponseUtil.*; import static com.uid2.operator.vertx.Endpoints.*; @@ -469,7 +470,7 @@ else if(emailHash != null) { tokenGenerateResponse = this.idService.generateIdentity( new TokenGenerateRequest( new SourcePublisher(clientSideKeypair.getSiteId()), - input.toHashedDiiIdentity(this.identityScope), + input.toHashedDii(this.identityScope), OptoutCheckPolicy.RespectOptOut, privacyBits, Instant.now())); } catch (KeyManager.NoActiveKeyException e){ SendServerErrorResponseAndRecordStats(rc, "No active encryption key available", clientSideKeypair.getSiteId(), TokenResponseStatsCollector.Endpoint.ClientSideTokenGenerateV2, TokenResponseStatsCollector.ResponseStatus.NoActiveKey, siteProvider, e, platformType); @@ -878,11 +879,11 @@ private void handleTokenValidateV1(RoutingContext rc) { if (!isTokenInputValid(input, rc)) { return; } - if ((Arrays.equals(ValidateIdentityForEmailHash, input.getIdentityInput()) && input.getIdentityType() == IdentityType.Email) - || (Arrays.equals(ValidateIdentityForPhoneHash, input.getIdentityInput()) && input.getIdentityType() == IdentityType.Phone)) { + if ((Arrays.equals(ValidateIdentityForEmailHash, input.getHashedDiiInput()) && input.getDiiType() == DiiType.Email) + || (Arrays.equals(ValidateIdentityForPhoneHash, input.getHashedDiiInput()) && input.getDiiType() == DiiType.Phone)) { try { final Instant now = Instant.now(); - if (this.idService.advertisingTokenMatches(rc.queryParam("token").get(0), input.toHashedDiiIdentity(this.identityScope), now)) { + if (this.idService.advertisingTokenMatches(rc.queryParam("token").get(0), input.toHashedDii(this.identityScope), now)) { ResponseUtil.Success(rc, Boolean.TRUE); } else { ResponseUtil.Success(rc, Boolean.FALSE); @@ -909,13 +910,13 @@ private void handleTokenValidateV2(RoutingContext rc) { if (!isTokenInputValid(input, rc)) { return; } - if ((input.getIdentityType() == IdentityType.Email && Arrays.equals(ValidateIdentityForEmailHash, input.getIdentityInput())) - || (input.getIdentityType() == IdentityType.Phone && Arrays.equals(ValidateIdentityForPhoneHash, input.getIdentityInput()))) { + if ((input.getDiiType() == DiiType.Email && Arrays.equals(ValidateIdentityForEmailHash, input.getHashedDiiInput())) + || (input.getDiiType() == DiiType.Phone && Arrays.equals(ValidateIdentityForPhoneHash, input.getHashedDiiInput()))) { try { final Instant now = Instant.now(); final String token = req.getString("token"); - if (this.idService.advertisingTokenMatches(token, input.toHashedDiiIdentity(this.identityScope), now)) { + if (this.idService.advertisingTokenMatches(token, input.toHashedDii(this.identityScope), now)) { ResponseUtil.SuccessV2(rc, Boolean.TRUE); } else { ResponseUtil.SuccessV2(rc, Boolean.FALSE); @@ -942,7 +943,7 @@ private void handleTokenGenerateV1(RoutingContext rc) { final TokenGenerateResponse response = this.idService.generateIdentity( new TokenGenerateRequest( new SourcePublisher(siteId), - input.toHashedDiiIdentity(this.identityScope), + input.toHashedDii(this.identityScope), OptoutCheckPolicy.defaultPolicy())); ResponseUtil.Success(rc, response.toJsonV1()); recordTokenResponseStats(siteId, TokenResponseStatsCollector.Endpoint.GenerateV1, TokenResponseStatsCollector.ResponseStatus.Success, siteProvider, response.getAdvertisingTokenVersion(), platformType); @@ -994,12 +995,12 @@ private void handleTokenGenerateV2(RoutingContext rc) { final TokenGenerateResponse response = this.idService.generateIdentity( new TokenGenerateRequest( new SourcePublisher(siteId), - input.toHashedDiiIdentity(this.identityScope), + input.toHashedDii(this.identityScope), OptoutCheckPolicy.respectOptOut())); if (response.isOptedOut()) { if (optoutCheckPolicy.getItem1() == OptoutCheckPolicy.DoNotRespect) { // only legacy can use this policy - final InputUtil.InputVal optOutTokenInput = input.getIdentityType() == IdentityType.Email + final InputUtil.InputVal optOutTokenInput = input.getDiiType() == DiiType.Email ? InputUtil.InputVal.validEmail(OptOutTokenIdentityForEmail, OptOutTokenIdentityForEmail) : InputUtil.InputVal.validPhone(OptOutTokenIdentityForPhone, OptOutTokenIdentityForPhone); @@ -1010,7 +1011,7 @@ private void handleTokenGenerateV2(RoutingContext rc) { final TokenGenerateResponse optOutTokens = this.idService.generateIdentity( new TokenGenerateRequest( new SourcePublisher(siteId), - optOutTokenInput.toHashedDiiIdentity(this.identityScope), + optOutTokenInput.toHashedDii(this.identityScope), OptoutCheckPolicy.DoNotRespect, pb, Instant.now())); ResponseUtil.SuccessV2(rc, optOutTokens.toJsonV1()); @@ -1050,7 +1051,7 @@ else if (!input.isValid()) { final TokenGenerateResponse response = this.idService.generateIdentity( new TokenGenerateRequest( new SourcePublisher(siteId), - input.toHashedDiiIdentity(this.identityScope), + input.toHashedDii(this.identityScope), OptoutCheckPolicy.defaultPolicy())); recordTokenResponseStats(siteId, TokenResponseStatsCollector.Endpoint.GenerateV0, TokenResponseStatsCollector.ResponseStatus.Success, siteProvider, response.getAdvertisingTokenVersion(), TokenResponseStatsCollector.PlatformType.Other); @@ -1087,10 +1088,10 @@ private void handleTokenRefresh(RoutingContext rc) { private void handleValidate(RoutingContext rc) { try { final InputUtil.InputVal input = getTokenInput(rc); - if (input != null && input.isValid() && Arrays.equals(ValidateIdentityForEmailHash, input.getIdentityInput())) { + if (input != null && input.isValid() && Arrays.equals(ValidateIdentityForEmailHash, input.getHashedDiiInput())) { try { final Instant now = Instant.now(); - if (this.idService.advertisingTokenMatches(rc.queryParam("token").get(0), input.toHashedDiiIdentity(this.identityScope), now)) { + if (this.idService.advertisingTokenMatches(rc.queryParam("token").get(0), input.toHashedDii(this.identityScope), now)) { rc.response().end("true"); } else { rc.response().end("false"); @@ -1111,7 +1112,7 @@ private void handleLogoutAsync(RoutingContext rc) { final InputUtil.InputVal input = this.phoneSupport ? getTokenInputV1(rc) : getTokenInput(rc); if (input.isValid()) { final Instant now = Instant.now(); - this.idService.invalidateTokensAsync(input.toHashedDiiIdentity(this.identityScope), now, ar -> { + this.idService.invalidateTokensAsync(input.toHashedDii(this.identityScope), now, ar -> { if (ar.succeeded()) { rc.response().end("OK"); } else { @@ -1130,7 +1131,7 @@ private Future handleLogoutAsyncV2(RoutingContext rc) { final Instant now = Instant.now(); Promise promise = Promise.promise(); - this.idService.invalidateTokensAsync(input.toHashedDiiIdentity(this.identityScope), now, ar -> { + this.idService.invalidateTokensAsync(input.toHashedDii(this.identityScope), now, ar -> { if (ar.succeeded()) { JsonObject body = new JsonObject(); body.put("optout", "OK"); @@ -1152,7 +1153,7 @@ private void handleOptOutGet(RoutingContext rc) { if (input.isValid()) { try { final Instant now = Instant.now(); - final HashedDii hashedDii = input.toHashedDiiIdentity(this.identityScope); + final HashedDii hashedDii = input.toHashedDii(this.identityScope); final Instant result = this.idService.getLatestOptoutEntry(hashedDii, now); long timestamp = result == null ? -1 : result.getEpochSecond(); rc.response().setStatusCode(200) @@ -1237,7 +1238,7 @@ private void handleIdentityMapV1(RoutingContext rc) { } try { final Instant now = Instant.now(); - final IdentityMapResponseItem identityMapResponseItem = this.idService.map(input.toHashedDiiIdentity(this.identityScope), now); + final IdentityMapResponseItem identityMapResponseItem = this.idService.map(input.toHashedDii(this.identityScope), now); final JsonObject jsonObject = new JsonObject(); jsonObject.put("identifier", input.getProvided()); jsonObject.put("advertising_id", EncodingUtils.toBase64String(identityMapResponseItem.rawUid)); @@ -1254,7 +1255,7 @@ private void handleIdentityMap(RoutingContext rc) { try { if (isTokenInputValid(input, rc)) { final Instant now = Instant.now(); - final IdentityMapResponseItem identityMapResponseItem = this.idService.map(input.toHashedDiiIdentity(this.identityScope), now); + final IdentityMapResponseItem identityMapResponseItem = this.idService.map(input.toHashedDii(this.identityScope), now); rc.response().end(EncodingUtils.toBase64String(identityMapResponseItem.rawUid)); } } catch (Exception ex) { @@ -1427,16 +1428,16 @@ private InputUtil.InputVal[] getIdentityBulkInputV1(RoutingContext rc) { } if (emails != null && !emails.isEmpty()) { - return createInputListV1(emails, IdentityType.Email, InputUtil.IdentityInputType.Raw); + return createInputListV1(emails, DiiType.Email, InputUtil.DiiInputType.Raw); } else if (emailHashes != null && !emailHashes.isEmpty()) { - return createInputListV1(emailHashes, IdentityType.Email, InputUtil.IdentityInputType.Hash); + return createInputListV1(emailHashes, DiiType.Email, InputUtil.DiiInputType.Hash); } else if (phones != null && !phones.isEmpty()) { - return createInputListV1(phones, IdentityType.Phone, InputUtil.IdentityInputType.Raw); + return createInputListV1(phones, DiiType.Phone, InputUtil.DiiInputType.Raw); } else if (phoneHashes != null && !phoneHashes.isEmpty()) { - return createInputListV1(phoneHashes, IdentityType.Phone, InputUtil.IdentityInputType.Hash); + return createInputListV1(phoneHashes, DiiType.Phone, InputUtil.DiiInputType.Hash); } else { // handle empty array - return createInputListV1(null, IdentityType.Email, InputUtil.IdentityInputType.Raw); + return createInputListV1(null, DiiType.Email, InputUtil.DiiInputType.Raw); } } @@ -1452,7 +1453,7 @@ private JsonObject handleIdentityMapCommon(RoutingContext rc, InputUtil.InputVal if (input != null && input.isValid()) { final IdentityMapResponseItem identityMapResponseItem = idService.mapHashedDii( new IdentityMapRequestItem( - input.toHashedDiiIdentity(this.identityScope), + input.toHashedDii(this.identityScope), OptoutCheckPolicy.respectOptOut(), now)); @@ -1528,7 +1529,7 @@ private InputUtil.InputVal[] getIdentityMapV2Input(RoutingContext rc) { Supplier getInputList = null; final JsonArray emails = JsonParseUtils.parseArray(obj, "email", rc); if (emails != null && !emails.isEmpty()) { - getInputList = () -> createInputListV1(emails, IdentityType.Email, InputUtil.IdentityInputType.Raw); + getInputList = () -> createInputListV1(emails, DiiType.Email, InputUtil.DiiInputType.Raw); } final JsonArray emailHashes = JsonParseUtils.parseArray(obj, "email_hash", rc); @@ -1536,7 +1537,7 @@ private InputUtil.InputVal[] getIdentityMapV2Input(RoutingContext rc) { if (getInputList != null) { return null; // only one type of input is allowed } - getInputList = () -> createInputListV1(emailHashes, IdentityType.Email, InputUtil.IdentityInputType.Hash); + getInputList = () -> createInputListV1(emailHashes, DiiType.Email, InputUtil.DiiInputType.Hash); } final JsonArray phones = this.phoneSupport ? JsonParseUtils.parseArray(obj,"phone", rc) : null; @@ -1544,7 +1545,7 @@ private InputUtil.InputVal[] getIdentityMapV2Input(RoutingContext rc) { if (getInputList != null) { return null; // only one type of input is allowed } - getInputList = () -> createInputListV1(phones, IdentityType.Phone, InputUtil.IdentityInputType.Raw); + getInputList = () -> createInputListV1(phones, DiiType.Phone, InputUtil.DiiInputType.Raw); } final JsonArray phoneHashes = this.phoneSupport ? JsonParseUtils.parseArray(obj,"phone_hash", rc) : null; @@ -1552,7 +1553,7 @@ private InputUtil.InputVal[] getIdentityMapV2Input(RoutingContext rc) { if (getInputList != null) { return null; // only one type of input is allowed } - getInputList = () -> createInputListV1(phoneHashes, IdentityType.Phone, InputUtil.IdentityInputType.Hash); + getInputList = () -> createInputListV1(phoneHashes, DiiType.Phone, InputUtil.DiiInputType.Hash); } if (emails == null && emailHashes == null && phones == null && phoneHashes == null) { @@ -1560,7 +1561,7 @@ private InputUtil.InputVal[] getIdentityMapV2Input(RoutingContext rc) { } return getInputList == null ? - createInputListV1(null, IdentityType.Email, InputUtil.IdentityInputType.Raw) : // handle empty array + createInputListV1(null, DiiType.Email, InputUtil.DiiInputType.Raw) : // handle empty array getInputList.get(); } @@ -1854,31 +1855,31 @@ private InputUtil.InputVal[] createInputList(JsonArray a, boolean inputAsHash) { } - private InputUtil.InputVal[] createInputListV1(JsonArray a, IdentityType identityType, InputUtil.IdentityInputType inputType) { + private InputUtil.InputVal[] createInputListV1(JsonArray a, DiiType diiType, InputUtil.DiiInputType inputType) { if (a == null || a.isEmpty()) { return new InputUtil.InputVal[0]; } final int size = a.size(); final InputUtil.InputVal[] resp = new InputUtil.InputVal[size]; - if (identityType == IdentityType.Email) { - if (inputType == InputUtil.IdentityInputType.Raw) { + if (diiType == DiiType.Email) { + if (inputType == InputUtil.DiiInputType.Raw) { for (int i = 0; i < size; ++i) { resp[i] = InputUtil.normalizeEmail(a.getString(i)); } - } else if (inputType == InputUtil.IdentityInputType.Hash) { + } else if (inputType == InputUtil.DiiInputType.Hash) { for (int i = 0; i < size; ++i) { resp[i] = InputUtil.normalizeEmailHash(a.getString(i)); } } else { throw new IllegalStateException("inputType"); } - } else if (identityType == IdentityType.Phone) { - if (inputType == InputUtil.IdentityInputType.Raw) { + } else if (diiType == DiiType.Phone) { + if (inputType == InputUtil.DiiInputType.Raw) { for (int i = 0; i < size; ++i) { resp[i] = InputUtil.normalizePhone(a.getString(i)); } - } else if (inputType == InputUtil.IdentityInputType.Hash) { + } else if (inputType == InputUtil.DiiInputType.Hash) { for (int i = 0; i < size; ++i) { resp[i] = InputUtil.normalizePhoneHash(a.getString(i)); } diff --git a/src/main/java/com/uid2/operator/vertx/V2PayloadHandler.java b/src/main/java/com/uid2/operator/vertx/V2PayloadHandler.java index 07ab3ff58..b55fca5a3 100644 --- a/src/main/java/com/uid2/operator/vertx/V2PayloadHandler.java +++ b/src/main/java/com/uid2/operator/vertx/V2PayloadHandler.java @@ -1,6 +1,6 @@ package com.uid2.operator.vertx; -import com.uid2.operator.model.IdentityScope; +import com.uid2.operator.model.identities.IdentityScope; import com.uid2.operator.model.KeyManager; import com.uid2.operator.monitoring.TokenResponseStatsCollector; import com.uid2.operator.service.EncodingUtils; diff --git a/src/test/java/com/uid2/operator/EUIDOperatorVerticleTest.java b/src/test/java/com/uid2/operator/EUIDOperatorVerticleTest.java index 8d168d42e..c4a765179 100644 --- a/src/test/java/com/uid2/operator/EUIDOperatorVerticleTest.java +++ b/src/test/java/com/uid2/operator/EUIDOperatorVerticleTest.java @@ -3,7 +3,7 @@ import com.uid2.shared.model.TokenVersion; import org.junit.jupiter.api.Test; -import com.uid2.operator.model.IdentityScope; +import com.uid2.operator.model.identities.IdentityScope; import com.uid2.shared.auth.Role; import io.vertx.core.Vertx; diff --git a/src/test/java/com/uid2/operator/InputNormalizationTest.java b/src/test/java/com/uid2/operator/InputNormalizationTest.java index adbefbeae..6f825608a 100644 --- a/src/test/java/com/uid2/operator/InputNormalizationTest.java +++ b/src/test/java/com/uid2/operator/InputNormalizationTest.java @@ -71,7 +71,7 @@ public void testValidEmailNormalization() { Assert.assertEquals(normalization.getProvided(), testCase[0]); Assert.assertTrue(normalization.isValid()); Assert.assertEquals(testCase[1], normalization.getNormalized()); - Assert.assertEquals(testCase[2], EncodingUtils.toBase64String(normalization.getIdentityInput())); + Assert.assertEquals(testCase[2], EncodingUtils.toBase64String(normalization.getHashedDiiInput())); } } @@ -90,7 +90,7 @@ public void testValidHashNormalization() { Assert.assertEquals(s, normalization.getProvided()); Assert.assertTrue(normalization.isValid()); Assert.assertEquals(masterHash, normalization.getNormalized()); - Assert.assertEquals(masterHash, EncodingUtils.toBase64String(normalization.getIdentityInput())); + Assert.assertEquals(masterHash, EncodingUtils.toBase64String(normalization.getHashedDiiInput())); } } diff --git a/src/test/java/com/uid2/operator/TokenEncodingTest.java b/src/test/java/com/uid2/operator/TokenEncodingTest.java index 039ebd17f..d216ad04b 100644 --- a/src/test/java/com/uid2/operator/TokenEncodingTest.java +++ b/src/test/java/com/uid2/operator/TokenEncodingTest.java @@ -1,7 +1,9 @@ package com.uid2.operator; import com.uid2.operator.model.*; +import com.uid2.operator.model.identities.DiiType; import com.uid2.operator.model.identities.FirstLevelHash; +import com.uid2.operator.model.identities.IdentityScope; import com.uid2.operator.model.identities.RawUid; import com.uid2.operator.service.EncodingUtils; import com.uid2.operator.service.EncryptedTokenEncoder; @@ -60,7 +62,7 @@ public void testRefreshTokenEncoding(TokenVersion tokenVersion) { now.plusSeconds(360), new OperatorIdentity(101, OperatorType.Service, 102, 103), new SourcePublisher(111, 112, 113), - new FirstLevelHash(IdentityScope.UID2, IdentityType.Email, firstLevelHash, now), + new FirstLevelHash(IdentityScope.UID2, DiiType.Email, firstLevelHash, now), PrivacyBits.fromInt(121) ); @@ -95,7 +97,7 @@ public void testAdvertisingTokenEncodings(TokenVersion tokenVersion) { final EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(this.keyManager); final Instant now = EncodingUtils.NowUTCMillis(); - final byte[] rawUid = UIDOperatorVerticleTest.getRawUid(IdentityType.Email, "test@example.com", IdentityScope.UID2, tokenVersion != TokenVersion.V2); + final byte[] rawUid = UIDOperatorVerticleTest.getRawUid(DiiType.Email, "test@example.com", IdentityScope.UID2, tokenVersion != TokenVersion.V2); final AdvertisingTokenRequest adTokenRequest = new AdvertisingTokenRequest( tokenVersion, @@ -103,7 +105,7 @@ public void testAdvertisingTokenEncodings(TokenVersion tokenVersion) { now.plusSeconds(60), new OperatorIdentity(101, OperatorType.Service, 102, 103), new SourcePublisher(111, 112, 113), - new RawUid(IdentityScope.UID2, IdentityType.Email, rawUid), + new RawUid(IdentityScope.UID2, DiiType.Email, rawUid), PrivacyBits.fromInt(121), now ); diff --git a/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java b/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java index bd4874c6e..5c2e24e86 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java @@ -1,9 +1,7 @@ package com.uid2.operator; import com.uid2.operator.model.*; -import com.uid2.operator.model.identities.FirstLevelHash; -import com.uid2.operator.model.identities.HashedDii; -import com.uid2.operator.model.identities.RawUid; +import com.uid2.operator.model.identities.*; import com.uid2.operator.service.*; import com.uid2.operator.service.EncodingUtils; import com.uid2.operator.service.EncryptedTokenEncoder; @@ -146,7 +144,7 @@ private void setNow(Instant now) { when(clock.instant()).thenAnswer(i -> this.now); } - private HashedDii createHashedDiiIdentity(String rawIdentityHash, IdentityScope scope, IdentityType type) { + private HashedDii createHashedDiiIdentity(String rawIdentityHash, IdentityScope scope, DiiType type) { return new HashedDii( scope, type, @@ -154,28 +152,28 @@ private HashedDii createHashedDiiIdentity(String rawIdentityHash, IdentityScope ); } - private AdvertisingTokenRequest validateAndGetToken(EncryptedTokenEncoder tokenEncoder, String advertisingTokenString, IdentityScope scope, IdentityType type, int siteId) { + private AdvertisingTokenRequest validateAndGetToken(EncryptedTokenEncoder tokenEncoder, String advertisingTokenString, IdentityScope scope, DiiType type, int siteId) { TokenVersion tokenVersion = (scope == IdentityScope.UID2) ? uid2Service.getAdvertisingTokenVersionForTests(siteId) : euidService.getAdvertisingTokenVersionForTests(siteId); UIDOperatorVerticleTest.validateAdvertisingToken(advertisingTokenString, tokenVersion, scope, type); return tokenEncoder.decodeAdvertisingToken(advertisingTokenString); } - private void assertIdentityScopeIdentityType(IdentityScope expectedScope, IdentityType expectedIdentityType, + private void assertIdentityScopeIdentityType(IdentityScope expectedScope, DiiType expectedDiiType, HashedDii hashedDii) { assertEquals(expectedScope, hashedDii.identityScope()); - assertEquals(expectedIdentityType, hashedDii.identityType()); + assertEquals(expectedDiiType, hashedDii.diiType()); } - private void assertIdentityScopeIdentityType(IdentityScope expectedScope, IdentityType expectedIdentityType, + private void assertIdentityScopeIdentityType(IdentityScope expectedScope, DiiType expectedDiiType, RawUid rawUid) { assertEquals(expectedScope, rawUid.identityScope()); - assertEquals(expectedIdentityType, rawUid.identityType()); + assertEquals(expectedDiiType, rawUid.diiType()); } - private void assertIdentityScopeIdentityType(IdentityScope expectedScope, IdentityType expectedIdentityType, + private void assertIdentityScopeIdentityType(IdentityScope expectedScope, DiiType expectedDiiType, FirstLevelHash firstLevelHash) { assertEquals(expectedScope, firstLevelHash.identityScope()); - assertEquals(expectedIdentityType, firstLevelHash.identityType()); + assertEquals(expectedDiiType, firstLevelHash.diiType()); } @@ -186,12 +184,12 @@ private void assertIdentityScopeIdentityType(IdentityScope expectedScope, Identi @CsvSource({"123, V2","127, V4","128, V4"}) //site id 127 and 128 is for testing "site_ids_using_v4_tokens" public void testGenerateAndRefresh(int siteId, TokenVersion tokenVersion) { IdentityScope expectedIdentityScope = IdentityScope.UID2; - IdentityType expectedIdentityType = IdentityType.Email; + DiiType expectedDiiType = DiiType.Email; final TokenGenerateRequest tokenGenerateRequest = new TokenGenerateRequest( new SourcePublisher(siteId, 124, 125), - createHashedDiiIdentity("test-email-hash", expectedIdentityScope, expectedIdentityType), + createHashedDiiIdentity("test-email-hash", expectedIdentityScope, expectedDiiType), OptoutCheckPolicy.DoNotRespect, PrivacyBits.fromInt(0), this.now.minusSeconds(234) ); @@ -200,11 +198,11 @@ public void testGenerateAndRefresh(int siteId, TokenVersion tokenVersion) { verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); assertNotNull(tokenGenerateResponse); - UIDOperatorVerticleTest.validateAdvertisingToken(tokenGenerateResponse.getAdvertisingToken(), tokenVersion, IdentityScope.UID2, IdentityType.Email); + UIDOperatorVerticleTest.validateAdvertisingToken(tokenGenerateResponse.getAdvertisingToken(), tokenVersion, IdentityScope.UID2, DiiType.Email); AdvertisingTokenRequest advertisingTokenRequest = tokenEncoder.decodeAdvertisingToken(tokenGenerateResponse.getAdvertisingToken()); assertEquals(this.now.plusSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS), advertisingTokenRequest.expiresAt); assertEquals(tokenGenerateRequest.sourcePublisher.siteId, advertisingTokenRequest.sourcePublisher.siteId); - assertIdentityScopeIdentityType(expectedIdentityScope, expectedIdentityType, + assertIdentityScopeIdentityType(expectedIdentityScope, expectedDiiType, advertisingTokenRequest.rawUid); assertEquals(tokenGenerateRequest.establishedAt, advertisingTokenRequest.establishedAt); assertEquals(tokenGenerateRequest.privacyBits, advertisingTokenRequest.privacyBits); @@ -213,7 +211,7 @@ public void testGenerateAndRefresh(int siteId, TokenVersion tokenVersion) { assertEquals(this.now, tokenRefreshRequest.createdAt); assertEquals(this.now.plusSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), tokenRefreshRequest.expiresAt); assertEquals(tokenGenerateRequest.sourcePublisher.siteId, tokenRefreshRequest.sourcePublisher.siteId); - assertIdentityScopeIdentityType(expectedIdentityScope, expectedIdentityType, tokenRefreshRequest.firstLevelHash); + assertIdentityScopeIdentityType(expectedIdentityScope, expectedDiiType, tokenRefreshRequest.firstLevelHash); assertEquals(tokenGenerateRequest.establishedAt, tokenRefreshRequest.firstLevelHash.establishedAt()); final byte[] firstLevelHash = getFirstLevelHash(tokenGenerateRequest.hashedDii.hashedDii(), @@ -231,11 +229,11 @@ public void testGenerateAndRefresh(int siteId, TokenVersion tokenVersion) { assertEquals(TokenRefreshResponse.Status.Refreshed, refreshResponse.getStatus()); assertNotNull(refreshResponse.getIdentityResponse()); - UIDOperatorVerticleTest.validateAdvertisingToken(refreshResponse.getIdentityResponse().getAdvertisingToken(), tokenVersion, IdentityScope.UID2, IdentityType.Email); + UIDOperatorVerticleTest.validateAdvertisingToken(refreshResponse.getIdentityResponse().getAdvertisingToken(), tokenVersion, IdentityScope.UID2, DiiType.Email); AdvertisingTokenRequest advertisingTokenRequest2 = tokenEncoder.decodeAdvertisingToken(refreshResponse.getIdentityResponse().getAdvertisingToken()); assertEquals(this.now.plusSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS), advertisingTokenRequest2.expiresAt); assertEquals(advertisingTokenRequest.sourcePublisher.siteId, advertisingTokenRequest2.sourcePublisher.siteId); - assertIdentityScopeIdentityType(expectedIdentityScope, expectedIdentityType, + assertIdentityScopeIdentityType(expectedIdentityScope, expectedDiiType, advertisingTokenRequest2.rawUid); assertEquals(advertisingTokenRequest.establishedAt, advertisingTokenRequest2.establishedAt); assertArrayEquals(advertisingTokenRequest.rawUid.rawUid(), @@ -246,7 +244,7 @@ public void testGenerateAndRefresh(int siteId, TokenVersion tokenVersion) { assertEquals(this.now, tokenRefreshRequest2.createdAt); assertEquals(this.now.plusSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), tokenRefreshRequest2.expiresAt); assertEquals(tokenRefreshRequest.sourcePublisher.siteId, tokenRefreshRequest2.sourcePublisher.siteId); - assertIdentityScopeIdentityType(expectedIdentityScope, expectedIdentityType, tokenRefreshRequest2.firstLevelHash); + assertIdentityScopeIdentityType(expectedIdentityScope, expectedDiiType, tokenRefreshRequest2.firstLevelHash); assertEquals(tokenRefreshRequest.firstLevelHash.establishedAt(), tokenRefreshRequest2.firstLevelHash.establishedAt()); assertArrayEquals(tokenRefreshRequest.firstLevelHash.firstLevelHash(), tokenRefreshRequest2.firstLevelHash.firstLevelHash()); assertArrayEquals(firstLevelHash, tokenRefreshRequest2.firstLevelHash.firstLevelHash()); @@ -258,7 +256,7 @@ public void testTestOptOutKey_DoNotRespectOptout() { final TokenGenerateRequest tokenGenerateRequest = new TokenGenerateRequest( new SourcePublisher(123, 124, 125), - inputVal.toHashedDiiIdentity(IdentityScope.UID2), + inputVal.toHashedDii(IdentityScope.UID2), OptoutCheckPolicy.DoNotRespect, PrivacyBits.fromInt(0), this.now ); final TokenGenerateResponse tokenGenerateResponse = uid2Service.generateIdentity(tokenGenerateRequest); @@ -277,7 +275,7 @@ public void testTestOptOutKey_RespectOptout() { final TokenGenerateRequest tokenGenerateRequest = new TokenGenerateRequest( new SourcePublisher(123, 124, 125), - inputVal.toHashedDiiIdentity(IdentityScope.UID2), + inputVal.toHashedDii(IdentityScope.UID2), OptoutCheckPolicy.RespectOptOut, PrivacyBits.fromInt(0), this.now ); final TokenGenerateResponse tokenGenerateResponse = uid2Service.generateIdentity(tokenGenerateRequest); @@ -293,7 +291,7 @@ public void testTestOptOutKeyIdentityScopeMismatch() { final TokenGenerateRequest tokenGenerateRequest = new TokenGenerateRequest( new SourcePublisher(123, 124, 125), - inputVal.toHashedDiiIdentity(IdentityScope.EUID), + inputVal.toHashedDii(IdentityScope.EUID), OptoutCheckPolicy.DoNotRespect, PrivacyBits.fromInt(0), this.now ); final TokenGenerateResponse tokenGenerateResponse = euidService.generateIdentity(tokenGenerateRequest); @@ -312,8 +310,8 @@ public void testTestOptOutKeyIdentityScopeMismatch() { "Email,test@example.com,EUID", "Phone,+01010101010,UID2", "Phone,+01010101010,EUID"}) - public void testGenerateTokenForOptOutUser(IdentityType type, String id, IdentityScope scope) { - final HashedDii hashedDii = createHashedDiiIdentity(TokenUtils.getIdentityHashString(id), + public void testGenerateTokenForOptOutUser(DiiType type, String id, IdentityScope scope) { + final HashedDii hashedDii = createHashedDiiIdentity(TokenUtils.getDiiHashString(id), scope, type); final TokenGenerateRequest tokenGenerateRequestForceGenerate = new TokenGenerateRequest( @@ -339,7 +337,7 @@ public void testGenerateTokenForOptOutUser(IdentityType type, String id, Identit tokenGenerateResponse = uid2Service.generateIdentity(tokenGenerateRequestForceGenerate); verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); - advertisingTokenRequest = validateAndGetToken(tokenEncoder, tokenGenerateResponse.getAdvertisingToken(), IdentityScope.UID2, hashedDii.identityType(), tokenGenerateRequestRespectOptOut.sourcePublisher.siteId); + advertisingTokenRequest = validateAndGetToken(tokenEncoder, tokenGenerateResponse.getAdvertisingToken(), IdentityScope.UID2, hashedDii.diiType(), tokenGenerateRequestRespectOptOut.sourcePublisher.siteId); reset(shutdownHandler); tokenGenerateResponseAfterOptOut = uid2Service.generateIdentity(tokenGenerateRequestRespectOptOut); @@ -347,7 +345,7 @@ public void testGenerateTokenForOptOutUser(IdentityType type, String id, Identit tokenGenerateResponse = euidService.generateIdentity(tokenGenerateRequestForceGenerate); verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); - advertisingTokenRequest = validateAndGetToken(tokenEncoder, tokenGenerateResponse.getAdvertisingToken(), IdentityScope.EUID, hashedDii.identityType(), tokenGenerateRequestRespectOptOut.sourcePublisher.siteId); + advertisingTokenRequest = validateAndGetToken(tokenEncoder, tokenGenerateResponse.getAdvertisingToken(), IdentityScope.EUID, hashedDii.diiType(), tokenGenerateRequestRespectOptOut.sourcePublisher.siteId); reset(shutdownHandler); tokenGenerateResponseAfterOptOut = euidService.generateIdentity(tokenGenerateRequestRespectOptOut); } @@ -365,7 +363,7 @@ public void testGenerateTokenForOptOutUser(IdentityType type, String id, Identit "Email,test@example.com,EUID", "Phone,+01010101010,UID2", "Phone,+01010101010,EUID"}) - public void testIdentityMapForOptOutUser(IdentityType type, String identity, IdentityScope scope) { + public void testIdentityMapForOptOutUser(DiiType type, String identity, IdentityScope scope) { final HashedDii hashedDii = createHashedDiiIdentity(identity, scope, type); final Instant now = Instant.now(); @@ -451,7 +449,7 @@ void testSpecialIdentityOptOutTokenGenerate(TestIdentityInputType type, String i final TokenGenerateRequest tokenGenerateRequest = new TokenGenerateRequest( new SourcePublisher(123, 124, 125), - inputVal.toHashedDiiIdentity(scope), + inputVal.toHashedDii(scope), OptoutCheckPolicy.RespectOptOut, PrivacyBits.fromInt(0), this.now ); @@ -483,7 +481,7 @@ void testSpecialIdentityOptOutIdentityMap(TestIdentityInputType type, String id, InputUtil.InputVal inputVal = generateInputVal(type, id); final IdentityMapRequestItem identityMapRequestItemRespectOptOut = new IdentityMapRequestItem( - inputVal.toHashedDiiIdentity(scope), + inputVal.toHashedDii(scope), OptoutCheckPolicy.RespectOptOut, now); @@ -517,7 +515,7 @@ void testSpecialIdentityOptOutTokenRefresh(TestIdentityInputType type, String id final TokenGenerateRequest tokenGenerateRequest = new TokenGenerateRequest( new SourcePublisher(123, 124, 125), - inputVal.toHashedDiiIdentity(scope), + inputVal.toHashedDii(scope), OptoutCheckPolicy.DoNotRespect, PrivacyBits.fromInt(0), this.now ); @@ -556,7 +554,7 @@ void testSpecialIdentityRefreshOptOutGenerate(TestIdentityInputType type, String final TokenGenerateRequest tokenGenerateRequest = new TokenGenerateRequest( new SourcePublisher(123, 124, 125), - inputVal.toHashedDiiIdentity(scope), + inputVal.toHashedDii(scope), OptoutCheckPolicy.RespectOptOut, PrivacyBits.fromInt(0), this.now ); @@ -597,7 +595,7 @@ void testSpecialIdentityRefreshOptOutIdentityMap(TestIdentityInputType type, Str InputUtil.InputVal inputVal = generateInputVal(type, id); final IdentityMapRequestItem identityMapRequestItemRespectOptOut = new IdentityMapRequestItem( - inputVal.toHashedDiiIdentity(scope), + inputVal.toHashedDii(scope), OptoutCheckPolicy.RespectOptOut, now); @@ -631,7 +629,7 @@ void testSpecialIdentityValidateGenerate(TestIdentityInputType type, String id, final TokenGenerateRequest tokenGenerateRequest = new TokenGenerateRequest( new SourcePublisher(123, 124, 125), - inputVal.toHashedDiiIdentity(scope), + inputVal.toHashedDii(scope), OptoutCheckPolicy.RespectOptOut, PrivacyBits.fromInt(0), this.now ); @@ -646,7 +644,7 @@ void testSpecialIdentityValidateGenerate(TestIdentityInputType type, String id, else { tokenGenerateResponse = uid2Service.generateIdentity(tokenGenerateRequest); } - advertisingTokenRequest = validateAndGetToken(tokenEncoder, tokenGenerateResponse.getAdvertisingToken(), scope, tokenGenerateRequest.hashedDii.identityType(), tokenGenerateRequest.sourcePublisher.siteId); + advertisingTokenRequest = validateAndGetToken(tokenEncoder, tokenGenerateResponse.getAdvertisingToken(), scope, tokenGenerateRequest.hashedDii.diiType(), tokenGenerateRequest.sourcePublisher.siteId); verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); assertNotNull(tokenGenerateResponse); @@ -668,7 +666,7 @@ void testSpecialIdentityValidateIdentityMap(TestIdentityInputType type, String i InputUtil.InputVal inputVal = generateInputVal(type, id); final IdentityMapRequestItem identityMapRequestItemRespectOptOut = new IdentityMapRequestItem( - inputVal.toHashedDiiIdentity(scope), + inputVal.toHashedDii(scope), OptoutCheckPolicy.RespectOptOut, now); @@ -699,7 +697,7 @@ void testNormalIdentityOptIn(TestIdentityInputType type, String id, IdentityScop InputUtil.InputVal inputVal = generateInputVal(type, id); final TokenGenerateRequest tokenGenerateRequest = new TokenGenerateRequest( new SourcePublisher(123, 124, 125), - inputVal.toHashedDiiIdentity(scope), + inputVal.toHashedDii(scope), OptoutCheckPolicy.DoNotRespect ); TokenGenerateResponse tokenGenerateResponse; @@ -760,7 +758,7 @@ void testExpiredSaltsNotifiesShutdownHandler(TestIdentityInputType type, String final TokenGenerateRequest tokenGenerateRequest = new TokenGenerateRequest( new SourcePublisher(123, 124, 125), - inputVal.toHashedDiiIdentity(scope), + inputVal.toHashedDii(scope), OptoutCheckPolicy.RespectOptOut, PrivacyBits.fromInt(0), this.now); TokenGenerateResponse tokenGenerateResponse; @@ -768,11 +766,11 @@ void testExpiredSaltsNotifiesShutdownHandler(TestIdentityInputType type, String reset(shutdownHandler); if(scope == IdentityScope.EUID) { tokenGenerateResponse = euidService.generateIdentity(tokenGenerateRequest); - advertisingTokenRequest = validateAndGetToken(tokenEncoder, tokenGenerateResponse.getAdvertisingToken(), IdentityScope.EUID, tokenGenerateRequest.hashedDii.identityType(), tokenGenerateRequest.sourcePublisher.siteId); + advertisingTokenRequest = validateAndGetToken(tokenEncoder, tokenGenerateResponse.getAdvertisingToken(), IdentityScope.EUID, tokenGenerateRequest.hashedDii.diiType(), tokenGenerateRequest.sourcePublisher.siteId); } else { tokenGenerateResponse = uid2Service.generateIdentity(tokenGenerateRequest); - advertisingTokenRequest = validateAndGetToken(tokenEncoder, tokenGenerateResponse.getAdvertisingToken(), IdentityScope.UID2, tokenGenerateRequest.hashedDii.identityType(), tokenGenerateRequest.sourcePublisher.siteId); + advertisingTokenRequest = validateAndGetToken(tokenEncoder, tokenGenerateResponse.getAdvertisingToken(), IdentityScope.UID2, tokenGenerateRequest.hashedDii.diiType(), tokenGenerateRequest.sourcePublisher.siteId); } verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(true); verify(shutdownHandler, never()).handleSaltRetrievalResponse(false); @@ -790,7 +788,7 @@ void testExpiredSaltsNotifiesShutdownHandler(TestIdentityInputType type, String assertNotEquals(TokenRefreshResponse.Optout, refreshResponse); final IdentityMapRequestItem identityMapRequestItem = new IdentityMapRequestItem( - inputVal.toHashedDiiIdentity(scope), + inputVal.toHashedDii(scope), OptoutCheckPolicy.RespectOptOut, now); final IdentityMapResponseItem identityMapResponseItem; diff --git a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java index 6ee61fe3c..5e19ddbcd 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java @@ -4,7 +4,8 @@ import ch.qos.logback.classic.spi.ILoggingEvent; import ch.qos.logback.core.read.ListAppender; import com.uid2.operator.model.*; -import com.uid2.operator.model.IdentityScope; +import com.uid2.operator.model.identities.IdentityScope; +import com.uid2.operator.model.identities.DiiType; import com.uid2.operator.model.identities.FirstLevelHash; import com.uid2.operator.monitoring.IStatsCollectorQueue; import com.uid2.operator.monitoring.TokenResponseStatsCollector; @@ -69,7 +70,7 @@ import java.util.stream.Stream; import static com.uid2.operator.ClientSideTokenGenerateTestUtil.decrypt; -import static com.uid2.operator.IdentityConst.*; +import static com.uid2.operator.model.identities.IdentityConst.*; import static com.uid2.operator.service.EncodingUtils.getSha256; import static com.uid2.operator.vertx.UIDOperatorVerticle.*; import static com.uid2.shared.Const.Data.*; @@ -622,26 +623,26 @@ private void assertTokenStatusMetrics(Integer siteId, TokenResponseStatsCollecto assertEquals(1, actual); } - private byte[] getRawUidFromIdentity(IdentityType identityType, String identityString, String firstLevelSalt, String rotatingSalt) { - return getRawUid(identityType, identityString, firstLevelSalt, rotatingSalt, getIdentityScope(), useIdentityV3()); + private byte[] getRawUidFromIdentity(DiiType diiType, String identityString, String firstLevelSalt, String rotatingSalt) { + return getRawUid(diiType, identityString, firstLevelSalt, rotatingSalt, getIdentityScope(), useIdentityV3()); } - private static byte[] getRawUid(IdentityType identityType, String identityString, String firstLevelSalt, String rotatingSalt, IdentityScope identityScope, boolean useIdentityV3) { + private static byte[] getRawUid(DiiType diiType, String identityString, String firstLevelSalt, String rotatingSalt, IdentityScope identityScope, boolean useIdentityV3) { return !useIdentityV3 ? TokenUtils.getRawUidV2FromIdentity(identityString, firstLevelSalt, rotatingSalt) - : TokenUtils.getRawUidV3FromIdentity(identityScope, identityType, identityString, firstLevelSalt, rotatingSalt); + : TokenUtils.getRawUidV3FromIdentity(identityScope, diiType, identityString, firstLevelSalt, rotatingSalt); } - public static byte[] getRawUid(IdentityType identityType, String identityString, IdentityScope identityScope, boolean useIdentityV3) { + public static byte[] getRawUid(DiiType diiType, String identityString, IdentityScope identityScope, boolean useIdentityV3) { return !useIdentityV3 ? TokenUtils.getRawUidV2FromIdentity(identityString, firstLevelSalt, rotatingSalt123.getSalt()) - : TokenUtils.getRawUidV3FromIdentity(identityScope, identityType, identityString, firstLevelSalt, rotatingSalt123.getSalt()); + : TokenUtils.getRawUidV3FromIdentity(identityScope, diiType, identityString, firstLevelSalt, rotatingSalt123.getSalt()); } - private byte[] getRawUidFromIdentityHash(IdentityType identityType, String identityString, String firstLevelSalt, String rotatingSalt) { + private byte[] getRawUidFromIdentityHash(DiiType diiType, String identityString, String firstLevelSalt, String rotatingSalt) { return !useIdentityV3() ? TokenUtils.getRawUidV2FromIdentityHash(identityString, firstLevelSalt, rotatingSalt) - : TokenUtils.getRawUidV3FromIdentityHash(getIdentityScope(), identityType, identityString, firstLevelSalt, rotatingSalt); + : TokenUtils.getRawUidV3FromIdentityHash(getIdentityScope(), diiType, identityString, firstLevelSalt, rotatingSalt); } private JsonObject createBatchEmailsRequestPayload() { @@ -765,7 +766,7 @@ void keyLatestHideRefreshKey(String apiVersion, Vertx vertx, VertxTestContext te void tokenGenerateBothEmailAndHashSpecified(String apiVersion, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; final String emailAddress = "test@uid2.com"; - final String emailHash = TokenUtils.getIdentityHashString(emailAddress); + final String emailHash = TokenUtils.getDiiHashString(emailAddress); fakeAuth(clientSiteId, Role.GENERATOR); setupSalts(); setupKeys(); @@ -813,25 +814,25 @@ private void assertStatsCollector(String path, String referer, String apiContact assertEquals(siteId, messageItem.getSiteId()); } - private AdvertisingTokenRequest validateAndGetToken(EncryptedTokenEncoder encoder, JsonObject body, IdentityType identityType) { //See UID2-79+Token+and+ID+format+v3 + private AdvertisingTokenRequest validateAndGetToken(EncryptedTokenEncoder encoder, JsonObject body, DiiType diiType) { //See UID2-79+Token+and+ID+format+v3 final String advertisingTokenString = body.getString("advertising_token"); - validateAdvertisingToken(advertisingTokenString, getTokenVersion(), getIdentityScope(), identityType); + validateAdvertisingToken(advertisingTokenString, getTokenVersion(), getIdentityScope(), diiType); AdvertisingTokenRequest advertisingTokenRequest = encoder.decodeAdvertisingToken(advertisingTokenString); if (getTokenVersion() == TokenVersion.V4) { - assertEquals(identityType, advertisingTokenRequest.rawUid.identityType()); + assertEquals(diiType, advertisingTokenRequest.rawUid.diiType()); } return advertisingTokenRequest; } - public static void validateAdvertisingToken(String advertisingTokenString, TokenVersion tokenVersion, IdentityScope identityScope, IdentityType identityType) { + public static void validateAdvertisingToken(String advertisingTokenString, TokenVersion tokenVersion, IdentityScope identityScope, DiiType diiType) { if (tokenVersion == TokenVersion.V2) { assertEquals("Ag", advertisingTokenString.substring(0, 2)); } else { String firstChar = advertisingTokenString.substring(0, 1); if (identityScope == IdentityScope.UID2) { - assertEquals(identityType == IdentityType.Email ? "A" : "B", firstChar); + assertEquals(diiType == DiiType.Email ? "A" : "B", firstChar); } else { - assertEquals(identityType == IdentityType.Email ? "E" : "F", firstChar); + assertEquals(diiType == DiiType.Email ? "E" : "F", firstChar); } String secondChar = advertisingTokenString.substring(1, 2); @@ -848,10 +849,10 @@ public static void validateAdvertisingToken(String advertisingTokenString, Token } } - TokenRefreshRequest decodeRefreshToken(EncryptedTokenEncoder encoder, String refreshTokenString, IdentityType identityType) { + TokenRefreshRequest decodeRefreshToken(EncryptedTokenEncoder encoder, String refreshTokenString, DiiType diiType) { TokenRefreshRequest tokenRefreshRequest = encoder.decodeRefreshToken(refreshTokenString); assertEquals(getIdentityScope(), tokenRefreshRequest.firstLevelHash.identityScope()); - assertEquals(identityType, tokenRefreshRequest.firstLevelHash.identityType()); + assertEquals(diiType, tokenRefreshRequest.firstLevelHash.diiType()); return tokenRefreshRequest; } @@ -1125,7 +1126,7 @@ void tokenGenerateNewClientWrongPolicySpecifiedOlderKeySuccessful(String policyP "policy,+01234567890,Phone", "optout_check,someoptout@example.com,Email", "optout_check,+01234567890,Phone"}) - void tokenGenerateOptOutToken(String policyParameterKey, String identity, IdentityType identityType, + void tokenGenerateOptOutToken(String policyParameterKey, String identity, DiiType diiType, Vertx vertx, VertxTestContext testContext) { ClientKey oldClientKey = new ClientKey( null, @@ -1145,13 +1146,13 @@ void tokenGenerateOptOutToken(String policyParameterKey, String identity, Identi setupKeys(); JsonObject v2Payload = new JsonObject(); - v2Payload.put(identityType.name().toLowerCase(), identity); + v2Payload.put(diiType.name().toLowerCase(), identity); v2Payload.put(policyParameterKey, OptoutCheckPolicy.DoNotRespect.policy); sendTokenGenerate("v2", vertx, "", v2Payload, 200, json -> { - InputUtil.InputVal optOutTokenInput = identityType == IdentityType.Email ? + InputUtil.InputVal optOutTokenInput = diiType == DiiType.Email ? InputUtil.InputVal.validEmail(OptOutTokenIdentityForEmail, OptOutTokenIdentityForEmail) : InputUtil.InputVal.validPhone(OptOutIdentityForPhone, OptOutTokenIdentityForPhone); @@ -1164,9 +1165,9 @@ void tokenGenerateOptOutToken(String policyParameterKey, String identity, Identi EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); - AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, body, identityType); + AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, body, diiType); TokenRefreshRequest tokenRefreshRequest = encoder.decodeRefreshToken(body.getString("decrypted_refresh_token")); - final byte[] rawUid = getRawUidFromIdentity(identityType, + final byte[] rawUid = getRawUidFromIdentity(diiType, optOutTokenInput.getNormalized(), firstLevelSalt, rotatingSalt123.getSalt()); @@ -1177,7 +1178,7 @@ void tokenGenerateOptOutToken(String policyParameterKey, String identity, Identi String advertisingTokenString = body.getString("advertising_token"); final Instant now = Instant.now(); final String token = advertisingTokenString; - final boolean matchedOptedOutIdentity = this.uidOperatorVerticle.getIdService().advertisingTokenMatches(token, optOutTokenInput.toHashedDiiIdentity(getIdentityScope()), now); + final boolean matchedOptedOutIdentity = this.uidOperatorVerticle.getIdService().advertisingTokenMatches(token, optOutTokenInput.toHashedDii(getIdentityScope()), now); assertTrue(matchedOptedOutIdentity); assertFalse(advertisingTokenRequest.privacyBits.isClientSideTokenGenerated()); assertTrue(advertisingTokenRequest.privacyBits.isClientSideTokenOptedOut()); @@ -1224,11 +1225,11 @@ void tokenGenerateForEmail(String apiVersion, Vertx vertx, VertxTestContext test assertNotNull(body); EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); - AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, body, IdentityType.Email); - TokenRefreshRequest tokenRefreshRequest = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), IdentityType.Email); + AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, body, DiiType.Email); + TokenRefreshRequest tokenRefreshRequest = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), DiiType.Email); assertAdvertisingTokenRefreshTokenRequests(advertisingTokenRequest, tokenRefreshRequest, clientSiteId, - getRawUidFromIdentity(IdentityType.Email, emailAddress, firstLevelSalt, rotatingSalt123.getSalt()), + getRawUidFromIdentity(DiiType.Email, emailAddress, firstLevelSalt, rotatingSalt123.getSalt()), PrivacyBits.DEFAULT, body, TokenUtils.getFirstLevelHashFromIdentity(emailAddress, firstLevelSalt)); @@ -1264,7 +1265,7 @@ public void verifyPrivacyBits(PrivacyBits expectedValue, AdvertisingTokenRequest @ValueSource(strings = {"v1", "v2"}) void tokenGenerateForEmailHash(String apiVersion, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; - final String emailHash = TokenUtils.getIdentityHashString("test@uid2.com"); + final String emailHash = TokenUtils.getDiiHashString("test@uid2.com"); fakeAuth(clientSiteId, Role.GENERATOR); setupSalts(); setupKeys(); @@ -1281,11 +1282,11 @@ void tokenGenerateForEmailHash(String apiVersion, Vertx vertx, VertxTestContext assertNotNull(body); EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); - AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, body, IdentityType.Email); - TokenRefreshRequest tokenRefreshRequest = decodeRefreshToken(encoder, apiVersion.equals("v2") ? body.getString("decrypted_refresh_token") : body.getString("refresh_token"), IdentityType.Email); + AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, body, DiiType.Email); + TokenRefreshRequest tokenRefreshRequest = decodeRefreshToken(encoder, apiVersion.equals("v2") ? body.getString("decrypted_refresh_token") : body.getString("refresh_token"), DiiType.Email); assertAdvertisingTokenRefreshTokenRequests(advertisingTokenRequest, tokenRefreshRequest, clientSiteId, - getRawUidFromIdentityHash(IdentityType.Email, emailHash, firstLevelSalt, rotatingSalt123.getSalt()), + getRawUidFromIdentityHash(DiiType.Email, emailHash, firstLevelSalt, rotatingSalt123.getSalt()), PrivacyBits.DEFAULT, body, TokenUtils.getFirstLevelHashFromIdentityHash(emailHash, firstLevelSalt)); @@ -1312,16 +1313,16 @@ void tokenGenerateThenRefresh(String apiVersion, Vertx vertx, VertxTestContext t EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); AdvertisingTokenRequest firstAdvertisingTokenRequest = validateAndGetToken(encoder, bodyJson, - IdentityType.Email); + DiiType.Email); - TokenRefreshRequest firstTokenRefreshRequest = decodeRefreshToken(encoder, bodyJson.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), IdentityType.Email); + TokenRefreshRequest firstTokenRefreshRequest = decodeRefreshToken(encoder, bodyJson.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), DiiType.Email); assertEquals(firstAdvertisingTokenRequest.establishedAt, firstTokenRefreshRequest.firstLevelHash.establishedAt()); when(this.optOutStore.getLatestEntry(any())).thenReturn(null); - byte[] expectedRawUidIdentity = getRawUidFromIdentity(IdentityType.Email, emailAddress, firstLevelSalt, rotatingSalt123.getSalt()); + byte[] expectedRawUidIdentity = getRawUidFromIdentity(DiiType.Email, emailAddress, firstLevelSalt, rotatingSalt123.getSalt()); byte[] expectedFirstLevelHashIdentity = TokenUtils.getFirstLevelHashFromIdentity(emailAddress, firstLevelSalt); assertAdvertisingTokenRefreshTokenRequests(firstAdvertisingTokenRequest, firstTokenRefreshRequest, clientSiteId, @@ -1338,10 +1339,10 @@ void tokenGenerateThenRefresh(String apiVersion, Vertx vertx, VertxTestContext t assertNotNull(refreshBody); - AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, refreshBody, IdentityType.Email); + AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, refreshBody, DiiType.Email); String refreshTokenStringNew = refreshBody.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"); assertNotEquals(genRefreshToken, refreshTokenStringNew); - TokenRefreshRequest tokenRefreshRequest = decodeRefreshToken(encoder, refreshTokenStringNew, IdentityType.Email); + TokenRefreshRequest tokenRefreshRequest = decodeRefreshToken(encoder, refreshTokenStringNew, DiiType.Email); // assert if the ad/refresh tokens from original token/generate is same as the ad/refresh tokens from token/refresh assertAdvertisingTokenRefreshTokenRequests( @@ -1411,16 +1412,16 @@ void tokenGenerateThenRefreshSaltsExpired(String apiVersion, Vertx vertx, VertxT assertNotNull(refreshBody); EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); - AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, refreshBody, IdentityType.Email); + AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, refreshBody, DiiType.Email); assertFalse(advertisingTokenRequest.privacyBits.isClientSideTokenGenerated()); assertFalse(advertisingTokenRequest.privacyBits.isClientSideTokenOptedOut()); assertEquals(clientSiteId, advertisingTokenRequest.sourcePublisher.siteId); - assertArrayEquals(getRawUidFromIdentity(IdentityType.Email, emailAddress, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenRequest.rawUid.rawUid()); + assertArrayEquals(getRawUidFromIdentity(DiiType.Email, emailAddress, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenRequest.rawUid.rawUid()); String refreshTokenStringNew = refreshBody.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"); assertNotEquals(genRefreshToken, refreshTokenStringNew); - TokenRefreshRequest tokenRefreshRequest = decodeRefreshToken(encoder, refreshTokenStringNew, IdentityType.Email); + TokenRefreshRequest tokenRefreshRequest = decodeRefreshToken(encoder, refreshTokenStringNew, DiiType.Email); assertEquals(clientSiteId, tokenRefreshRequest.sourcePublisher.siteId); assertArrayEquals(TokenUtils.getFirstLevelHashFromIdentity(emailAddress, firstLevelSalt), tokenRefreshRequest.firstLevelHash.firstLevelHash()); @@ -1589,11 +1590,11 @@ void tokenGenerateUsingCustomSiteKey(String apiVersion, Vertx vertx, VertxTestCo assertNotNull(body); EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); - AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, body, IdentityType.Email); + AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, body, DiiType.Email); assertEquals(clientSiteId, advertisingTokenRequest.sourcePublisher.siteId); - assertArrayEquals(getRawUidFromIdentity(IdentityType.Email, emailAddress, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenRequest.rawUid.rawUid()); + assertArrayEquals(getRawUidFromIdentity(DiiType.Email, emailAddress, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenRequest.rawUid.rawUid()); - TokenRefreshRequest tokenRefreshRequest = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), IdentityType.Email); + TokenRefreshRequest tokenRefreshRequest = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), DiiType.Email); assertEquals(clientSiteId, tokenRefreshRequest.sourcePublisher.siteId); assertArrayEquals(TokenUtils.getFirstLevelHashFromIdentity(emailAddress, firstLevelSalt), tokenRefreshRequest.firstLevelHash.firstLevelHash()); @@ -1623,16 +1624,16 @@ void tokenGenerateSaltsExpired(String apiVersion, Vertx vertx, VertxTestContext assertNotNull(body); EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); - AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, body, IdentityType.Email); + AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, body, DiiType.Email); assertTrue(advertisingTokenRequest.privacyBits.isLegacyBitSet()); assertEquals(advertisingTokenRequest.privacyBits, PrivacyBits.DEFAULT); assertFalse(advertisingTokenRequest.privacyBits.isClientSideTokenGenerated()); assertFalse(advertisingTokenRequest.privacyBits.isClientSideTokenOptedOut()); assertEquals(clientSiteId, advertisingTokenRequest.sourcePublisher.siteId); - assertArrayEquals(getRawUidFromIdentity(IdentityType.Email, emailAddress, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenRequest.rawUid.rawUid()); + assertArrayEquals(getRawUidFromIdentity(DiiType.Email, emailAddress, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenRequest.rawUid.rawUid()); - TokenRefreshRequest tokenRefreshRequest = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), IdentityType.Email); + TokenRefreshRequest tokenRefreshRequest = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), DiiType.Email); assertEquals(clientSiteId, tokenRefreshRequest.sourcePublisher.siteId); assertArrayEquals(TokenUtils.getFirstLevelHashFromIdentity(emailAddress, firstLevelSalt), tokenRefreshRequest.firstLevelHash.firstLevelHash()); @@ -1941,7 +1942,7 @@ void tokenValidateWithEmailHash_Mismatch(String apiVersion, Vertx vertx, VertxTe void identityMapBothEmailAndHashSpecified(Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; final String emailAddress = "test@uid2.com"; - final String emailHash = TokenUtils.getIdentityHashString(emailAddress); + final String emailHash = TokenUtils.getDiiHashString(emailAddress); fakeAuth(clientSiteId, Role.MAPPER); setupSalts(); setupKeys(); @@ -2029,7 +2030,7 @@ void identityMapForSaltsExpired(Vertx vertx, VertxTestContext testContext) { @Test void identityMapForEmailHash(Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; - final String emailHash = TokenUtils.getIdentityHashString("test@uid2.com"); + final String emailHash = TokenUtils.getDiiHashString("test@uid2.com"); fakeAuth(clientSiteId, Role.MAPPER); setupSalts(); setupKeys(); @@ -2085,7 +2086,7 @@ void identityMapBatchBothEmailAndHashSpecified(String apiVersion, Vertx vertx, V req.put("email_hash", emailHashes); emails.add("test1@uid2.com"); - emailHashes.add(TokenUtils.getIdentityHashString("test2@uid2.com")); + emailHashes.add(TokenUtils.getDiiHashString("test2@uid2.com")); send(apiVersion, vertx, apiVersion + "/identity/map", false, null, req, 400, respJson -> { assertFalse(respJson.containsKey("body")); @@ -2224,8 +2225,8 @@ void identityMapBatchEmailHashes(String apiVersion, Vertx vertx, VertxTestContex JsonArray hashes = new JsonArray(); req.put("email_hash", hashes); final String[] email_hashes = { - TokenUtils.getIdentityHashString("test1@uid2.com"), - TokenUtils.getIdentityHashString("test2@uid2.com"), + TokenUtils.getDiiHashString("test1@uid2.com"), + TokenUtils.getDiiHashString("test2@uid2.com"), }; for (String email_hash : email_hashes) { @@ -2452,7 +2453,7 @@ void LogoutV2SaltsExpired(Vertx vertx, VertxTestContext testContext) { void tokenGenerateBothPhoneAndHashSpecified(String apiVersion, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; final String phone = "+15555555555"; - final String phoneHash = TokenUtils.getIdentityHashString(phone); + final String phoneHash = TokenUtils.getDiiHashString(phone); fakeAuth(clientSiteId, Role.GENERATOR); setupSalts(); setupKeys(); @@ -2498,9 +2499,9 @@ void tokenGenerateBothPhoneAndEmailSpecified(String apiVersion, Vertx vertx, Ver void tokenGenerateBothPhoneHashAndEmailHashSpecified(String apiVersion, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; final String phone = "+15555555555"; - final String phoneHash = TokenUtils.getIdentityHashString(phone); + final String phoneHash = TokenUtils.getDiiHashString(phone); final String emailAddress = "test@uid2.com"; - final String emailHash = TokenUtils.getIdentityHashString(emailAddress); + final String emailHash = TokenUtils.getDiiHashString(emailAddress); fakeAuth(clientSiteId, Role.GENERATOR); setupSalts(); setupKeys(); @@ -2537,11 +2538,11 @@ void tokenGenerateForPhone(String apiVersion, Vertx vertx, VertxTestContext test assertNotNull(body); EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); - AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, body, IdentityType.Phone); - TokenRefreshRequest tokenRefreshRequest = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), IdentityType.Phone); + AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, body, DiiType.Phone); + TokenRefreshRequest tokenRefreshRequest = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), DiiType.Phone); assertAdvertisingTokenRefreshTokenRequests(advertisingTokenRequest, tokenRefreshRequest, clientSiteId, - getRawUidFromIdentity(IdentityType.Phone, phone, firstLevelSalt, rotatingSalt123.getSalt()), + getRawUidFromIdentity(DiiType.Phone, phone, firstLevelSalt, rotatingSalt123.getSalt()), PrivacyBits.DEFAULT, body, TokenUtils.getFirstLevelHashFromIdentity(phone, firstLevelSalt)); @@ -2567,7 +2568,7 @@ void verifyFirstLevelHashIdentityAndEstablishedAt(byte[] expectedFirstLevelHash, void tokenGenerateForPhoneHash(String apiVersion, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; final String phone = "+15555555555"; - final String phoneHash = TokenUtils.getIdentityHashString(phone); + final String phoneHash = TokenUtils.getDiiHashString(phone); fakeAuth(clientSiteId, Role.GENERATOR); setupSalts(); setupKeys(); @@ -2582,11 +2583,11 @@ void tokenGenerateForPhoneHash(String apiVersion, Vertx vertx, VertxTestContext assertNotNull(body); EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); - AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, body, IdentityType.Phone); - TokenRefreshRequest tokenRefreshRequest = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), IdentityType.Phone); + AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, body, DiiType.Phone); + TokenRefreshRequest tokenRefreshRequest = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), DiiType.Phone); assertAdvertisingTokenRefreshTokenRequests(advertisingTokenRequest, tokenRefreshRequest, clientSiteId, - getRawUidFromIdentity(IdentityType.Phone, phone, firstLevelSalt, rotatingSalt123.getSalt()), + getRawUidFromIdentity(DiiType.Phone, phone, firstLevelSalt, rotatingSalt123.getSalt()), PrivacyBits.DEFAULT, body, TokenUtils.getFirstLevelHashFromIdentity(phone, firstLevelSalt)); @@ -2612,13 +2613,13 @@ void tokenGenerateThenRefreshForPhone(String apiVersion, Vertx vertx, VertxTestC EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); AdvertisingTokenRequest firstAdvertisingTokenRequest = validateAndGetToken(encoder, bodyJson, - IdentityType.Phone); + DiiType.Phone); String genRefreshToken = bodyJson.getString("refresh_token"); - TokenRefreshRequest firstTokenRefreshRequest = decodeRefreshToken(encoder, bodyJson.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), IdentityType.Phone); + TokenRefreshRequest firstTokenRefreshRequest = decodeRefreshToken(encoder, bodyJson.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), DiiType.Phone); when(this.optOutStore.getLatestEntry(any())).thenReturn(null); - byte[] expectedRawUidIdentity = getRawUidFromIdentity(IdentityType.Phone, phone, firstLevelSalt, rotatingSalt123.getSalt()); + byte[] expectedRawUidIdentity = getRawUidFromIdentity(DiiType.Phone, phone, firstLevelSalt, rotatingSalt123.getSalt()); byte[] expectedFirstLevelHashIdentity = TokenUtils.getFirstLevelHashFromIdentity(phone, firstLevelSalt); assertAdvertisingTokenRefreshTokenRequests(firstAdvertisingTokenRequest, firstTokenRefreshRequest, clientSiteId, @@ -2634,10 +2635,10 @@ void tokenGenerateThenRefreshForPhone(String apiVersion, Vertx vertx, VertxTestC assertNotNull(refreshBody); - AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, refreshBody, IdentityType.Phone); + AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, refreshBody, DiiType.Phone); String refreshTokenStringNew = refreshBody.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"); assertNotEquals(genRefreshToken, refreshTokenStringNew); - TokenRefreshRequest tokenRefreshRequest = decodeRefreshToken(encoder, refreshTokenStringNew, IdentityType.Phone); + TokenRefreshRequest tokenRefreshRequest = decodeRefreshToken(encoder, refreshTokenStringNew, DiiType.Phone); // assert if the ad/refresh tokens from original token/generate is same as the ad/refresh tokens from token/refresh assertAdvertisingTokenRefreshTokenRequests( @@ -2858,7 +2859,7 @@ void tokenRefreshOptOutBeforeLoginForPhone(String apiVersion, Vertx vertx, Vertx void identityMapBothPhoneAndHashSpecified(Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; final String phone = "+15555555555"; - final String phoneHash = TokenUtils.getIdentityHashString(phone); + final String phoneHash = TokenUtils.getDiiHashString(phone); fakeAuth(clientSiteId, Role.MAPPER); setupSalts(); setupKeys(); @@ -2902,7 +2903,7 @@ void identityMapForPhone(Vertx vertx, VertxTestContext testContext) { void identityMapForPhoneHash(Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; final String phone = "+15555555555"; - final String phonneHash = TokenUtils.getIdentityHashString(phone); + final String phonneHash = TokenUtils.getDiiHashString(phone); fakeAuth(clientSiteId, Role.MAPPER); setupSalts(); setupKeys(); @@ -2977,7 +2978,7 @@ void identityMapBatchBothPhoneAndHashSpecified(String apiVersion, Vertx vertx, V req.put("phone_hash", phoneHashes); phones.add("+15555555555"); - phoneHashes.add(TokenUtils.getIdentityHashString("+15555555555")); + phoneHashes.add(TokenUtils.getDiiHashString("+15555555555")); send(apiVersion, vertx, apiVersion + "/identity/map", false, null, req, 400, respJson -> { assertFalse(respJson.containsKey("body")); @@ -3019,8 +3020,8 @@ void identityMapBatchPhoneHashes(String apiVersion, Vertx vertx, VertxTestContex JsonArray hashes = new JsonArray(); req.put("phone_hash", hashes); final String[] email_hashes = { - TokenUtils.getIdentityHashString("+15555555555"), - TokenUtils.getIdentityHashString("+15555555556"), + TokenUtils.getDiiHashString("+15555555555"), + TokenUtils.getDiiHashString("+15555555556"), }; for (String email_hash : email_hashes) { @@ -3309,7 +3310,7 @@ void cstgNoIdentityHashProvided(Vertx vertx, VertxTestContext testContext) throw }) void cstgDomainNameCheckFails(String httpOrigin, Vertx vertx, VertxTestContext testContext) throws NoSuchAlgorithmException, InvalidKeyException { setupCstgBackend(); - Tuple.Tuple2 data = createClientSideTokenGenerateRequest(IdentityType.Email, "random@unifiedid.com", Instant.now().toEpochMilli()); + Tuple.Tuple2 data = createClientSideTokenGenerateRequest(DiiType.Email, "random@unifiedid.com", Instant.now().toEpochMilli()); sendCstg(vertx, "v2/token/client-generate", httpOrigin, @@ -3338,7 +3339,7 @@ void cstgDomainNameCheckFails(String httpOrigin, Vertx vertx, VertxTestContext t }) void cstgAppNameCheckFails(String appName, Vertx vertx, VertxTestContext testContext) throws NoSuchAlgorithmException, InvalidKeyException { setupCstgBackend(Collections.emptyList(), List.of("com.123.Game.App.android")); - Tuple.Tuple2 data = createClientSideTokenGenerateRequest(IdentityType.Email, "random@unifiedid.com", Instant.now().toEpochMilli(), appName); + Tuple.Tuple2 data = createClientSideTokenGenerateRequest(DiiType.Email, "random@unifiedid.com", Instant.now().toEpochMilli(), appName); sendCstg(vertx, "v2/token/client-generate", null, @@ -3373,7 +3374,7 @@ void cstgDomainNameCheckFailsAndLogInvalidHttpOrigin(String httpOrigin, Vertx ve this.uidOperatorVerticle.setLastInvalidOriginProcessTime(Instant.now().minusSeconds(3600)); setupCstgBackend(); - Tuple.Tuple2 data = createClientSideTokenGenerateRequest(IdentityType.Email, "random@unifiedid.com", Instant.now().toEpochMilli()); + Tuple.Tuple2 data = createClientSideTokenGenerateRequest(DiiType.Email, "random@unifiedid.com", Instant.now().toEpochMilli()); sendCstg(vertx, "v2/token/client-generate", httpOrigin, @@ -3404,7 +3405,7 @@ void cstgLogsInvalidAppName(String appName, Vertx vertx, VertxTestContext testCo this.uidOperatorVerticle.setLastInvalidOriginProcessTime(Instant.now().minusSeconds(3600)); setupCstgBackend(); - Tuple.Tuple2 data = createClientSideTokenGenerateRequest(IdentityType.Email, "random@unifiedid.com", Instant.now().toEpochMilli(), appName); + Tuple.Tuple2 data = createClientSideTokenGenerateRequest(DiiType.Email, "random@unifiedid.com", Instant.now().toEpochMilli(), appName); sendCstg(vertx, "v2/token/client-generate", null, @@ -3450,7 +3451,7 @@ void cstgDisabledAsUnauthorized(Vertx vertx, VertxTestContext testContext) throw requestJson.put("timestamp", timestamp); requestJson.put("subscription_id", subscriptionID); - Tuple.Tuple2 data = createClientSideTokenGenerateRequest(IdentityType.Email, "random@unifiedid.com", Instant.now().toEpochMilli(), null); + Tuple.Tuple2 data = createClientSideTokenGenerateRequest(DiiType.Email, "random@unifiedid.com", Instant.now().toEpochMilli(), null); sendCstg(vertx, "v2/token/client-generate", null, @@ -3488,7 +3489,7 @@ void cstgDomainNameCheckFailsAndLogSeveralInvalidHttpOrigin(String httpOrigin, V setupCstgBackend(); when(siteProvider.getSite(124)).thenReturn(new Site(124, "test2", true, new HashSet<>())); - Tuple.Tuple2 data = createClientSideTokenGenerateRequest(IdentityType.Email, "random@unifiedid.com", Instant.now().toEpochMilli()); + Tuple.Tuple2 data = createClientSideTokenGenerateRequest(DiiType.Email, "random@unifiedid.com", Instant.now().toEpochMilli()); sendCstg(vertx, "v2/token/client-generate", httpOrigin, @@ -3518,7 +3519,7 @@ void cstgDomainNameCheckFailsAndLogSeveralInvalidHttpOrigin(String httpOrigin, V }) void cstgDomainNameCheckPasses(String httpOrigin, Vertx vertx, VertxTestContext testContext) throws NoSuchAlgorithmException, InvalidKeyException { setupCstgBackend("cstg.co.uk", "cstg2.com", "localhost"); - Tuple.Tuple2 data = createClientSideTokenGenerateRequest(IdentityType.Email, "random@unifiedid.com", Instant.now().toEpochMilli()); + Tuple.Tuple2 data = createClientSideTokenGenerateRequest(DiiType.Email, "random@unifiedid.com", Instant.now().toEpochMilli()); sendCstg(vertx, "v2/token/client-generate", httpOrigin, @@ -3532,7 +3533,7 @@ void cstgDomainNameCheckPasses(String httpOrigin, Vertx vertx, VertxTestContext JsonObject refreshBody = respJson.getJsonObject("body"); assertNotNull(refreshBody); var encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); - validateAndGetToken(encoder, refreshBody, IdentityType.Email); //to validate token version is correct + validateAndGetToken(encoder, refreshBody, DiiType.Email); //to validate token version is correct testContext.completeNow(); }); } @@ -3545,7 +3546,7 @@ void cstgDomainNameCheckPasses(String httpOrigin, Vertx vertx, VertxTestContext }) void cstgAppNameCheckPasses(String appName, Vertx vertx, VertxTestContext testContext) throws NoSuchAlgorithmException, InvalidKeyException { setupCstgBackend(Collections.emptyList(), List.of("com.123.Game.App.android", "123456789")); - Tuple.Tuple2 data = createClientSideTokenGenerateRequest(IdentityType.Email, "random@unifiedid.com", Instant.now().toEpochMilli(), appName); + Tuple.Tuple2 data = createClientSideTokenGenerateRequest(DiiType.Email, "random@unifiedid.com", Instant.now().toEpochMilli(), appName); sendCstg(vertx, "v2/token/client-generate", null, @@ -3559,7 +3560,7 @@ void cstgAppNameCheckPasses(String appName, Vertx vertx, VertxTestContext testCo JsonObject refreshBody = respJson.getJsonObject("body"); assertNotNull(refreshBody); var encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); - validateAndGetToken(encoder, refreshBody, IdentityType.Email); //to validate token version is correct + validateAndGetToken(encoder, refreshBody, DiiType.Email); //to validate token version is correct assertTokenStatusMetrics( clientSideTokenGenerateSiteId, TokenResponseStatsCollector.Endpoint.ClientSideTokenGenerateV2, @@ -4026,18 +4027,18 @@ private Tuple.Tuple2 createClientSideTokenGenerateRequest return new Tuple.Tuple2<>(requestJson, secretKey); } - private Tuple.Tuple2 createClientSideTokenGenerateRequest(IdentityType identityType, String rawId, long timestamp) throws NoSuchAlgorithmException, InvalidKeyException { - return createClientSideTokenGenerateRequest(identityType, rawId, timestamp, null); + private Tuple.Tuple2 createClientSideTokenGenerateRequest(DiiType diiType, String rawId, long timestamp) throws NoSuchAlgorithmException, InvalidKeyException { + return createClientSideTokenGenerateRequest(diiType, rawId, timestamp, null); } - private Tuple.Tuple2 createClientSideTokenGenerateRequest(IdentityType identityType, String rawId, long timestamp, String appName) throws NoSuchAlgorithmException, InvalidKeyException { + private Tuple.Tuple2 createClientSideTokenGenerateRequest(DiiType diiType, String rawId, long timestamp, String appName) throws NoSuchAlgorithmException, InvalidKeyException { JsonObject identity = new JsonObject(); - if(identityType == IdentityType.Email) { + if(diiType == DiiType.Email) { identity.put("email_hash", getSha256(rawId)); } - else if(identityType == IdentityType.Phone) { + else if(diiType == DiiType.Phone) { identity.put("phone_hash", getSha256(rawId)); } else { //can't be other types @@ -4058,10 +4059,10 @@ private Tuple.Tuple2 createClientSideTokenGenerateRequest "test@example.com,Email", "+61400000000,Phone" }) - void cstgUserOptsOutAfterTokenGenerate(String id, IdentityType identityType, Vertx vertx, VertxTestContext testContext) throws NoSuchAlgorithmException, InvalidKeyException { + void cstgUserOptsOutAfterTokenGenerate(String id, DiiType diiType, Vertx vertx, VertxTestContext testContext) throws NoSuchAlgorithmException, InvalidKeyException { setupCstgBackend("cstg.co.uk"); - final Tuple.Tuple2 data = createClientSideTokenGenerateRequest(identityType, id, Instant.now().toEpochMilli()); + final Tuple.Tuple2 data = createClientSideTokenGenerateRequest(diiType, id, Instant.now().toEpochMilli()); // When we generate the token the user hasn't opted out. when(optOutStore.getLatestEntry(any(FirstLevelHash.class))) @@ -4085,10 +4086,10 @@ void cstgUserOptsOutAfterTokenGenerate(String id, IdentityType identityType, Ver assertEquals("success", response.getString("status")); final JsonObject genBody = response.getJsonObject("body"); - final AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, genBody, identityType); - final TokenRefreshRequest tokenRefreshRequest = decodeRefreshToken(encoder, decodeV2RefreshToken(response), identityType); + final AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, genBody, diiType); + final TokenRefreshRequest tokenRefreshRequest = decodeRefreshToken(encoder, decodeV2RefreshToken(response), diiType); - assertAreClientSideGeneratedTokens(advertisingTokenRequest, tokenRefreshRequest, clientSideTokenGenerateSiteId, identityType, id); + assertAreClientSideGeneratedTokens(advertisingTokenRequest, tokenRefreshRequest, clientSideTokenGenerateSiteId, diiType, id); // When we refresh the token the user has opted out. when(optOutStore.getLatestEntry(any(FirstLevelHash.class))) @@ -4112,11 +4113,11 @@ void cstgUserOptsOutAfterTokenGenerate(String id, IdentityType identityType, Ver "false,abc@abc.com,Email", "false,+61400000000,Phone", }) - void cstgSuccessForBothOptedAndNonOptedOutTest(boolean optOutExpected, String id, IdentityType identityType, + void cstgSuccessForBothOptedAndNonOptedOutTest(boolean optOutExpected, String id, DiiType diiType, Vertx vertx, VertxTestContext testContext) throws NoSuchAlgorithmException, InvalidKeyException { setupCstgBackend("cstg.co.uk"); - Tuple.Tuple2 data = createClientSideTokenGenerateRequest(identityType, id, Instant.now().toEpochMilli()); + Tuple.Tuple2 data = createClientSideTokenGenerateRequest(diiType, id, Instant.now().toEpochMilli()); if(optOutExpected) { @@ -4149,13 +4150,13 @@ void cstgSuccessForBothOptedAndNonOptedOutTest(boolean optOutExpected, String id decodeV2RefreshToken(respJson); EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); - AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, genBody, identityType); + AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, genBody, diiType); - TokenRefreshRequest tokenRefreshRequest = decodeRefreshToken(encoder, genBody.getString("decrypted_refresh_token"), identityType); + TokenRefreshRequest tokenRefreshRequest = decodeRefreshToken(encoder, genBody.getString("decrypted_refresh_token"), diiType); - byte[] expectedRawUidIdentity = getRawUidFromIdentity(identityType, id, firstLevelSalt, rotatingSalt123.getSalt()); + byte[] expectedRawUidIdentity = getRawUidFromIdentity(diiType, id, firstLevelSalt, rotatingSalt123.getSalt()); byte[] expectedFirstLevelHashIdentity = TokenUtils.getFirstLevelHashFromIdentity(id, firstLevelSalt); PrivacyBits expectedPrivacyBits = new PrivacyBits(); @@ -4168,7 +4169,7 @@ void cstgSuccessForBothOptedAndNonOptedOutTest(boolean optOutExpected, String id expectedPrivacyBits, genBody, expectedFirstLevelHashIdentity); - assertAreClientSideGeneratedTokens(advertisingTokenRequest, tokenRefreshRequest, clientSideTokenGenerateSiteId, identityType, id); + assertAreClientSideGeneratedTokens(advertisingTokenRequest, tokenRefreshRequest, clientSideTokenGenerateSiteId, diiType, id); assertEqualsClose(Instant.now().plusMillis(identityExpiresAfter.toMillis()), Instant.ofEpochMilli(genBody.getLong("identity_expires")), 10); assertEqualsClose(Instant.now().plusMillis(refreshExpiresAfter.toMillis()), Instant.ofEpochMilli(genBody.getLong("refresh_expires")), 10); assertEqualsClose(Instant.now().plusMillis(refreshIdentityAfter.toMillis()), Instant.ofEpochMilli(genBody.getLong("refresh_from")), 10); @@ -4189,11 +4190,11 @@ void cstgSuccessForBothOptedAndNonOptedOutTest(boolean optOutExpected, String id EncryptedTokenEncoder encoder2 = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); //make sure the new advertising token from refresh looks right - AdvertisingTokenRequest adTokenFromRefresh = validateAndGetToken(encoder2, refreshBody, identityType); + AdvertisingTokenRequest adTokenFromRefresh = validateAndGetToken(encoder2, refreshBody, diiType); String refreshTokenStringNew = refreshBody.getString("decrypted_refresh_token"); assertNotEquals(genRefreshToken, refreshTokenStringNew); - TokenRefreshRequest refreshTokenAfterRefreshSource = decodeRefreshToken(encoder, refreshTokenStringNew, identityType); + TokenRefreshRequest refreshTokenAfterRefreshSource = decodeRefreshToken(encoder, refreshTokenStringNew, diiType); assertAdvertisingTokenRefreshTokenRequests(adTokenFromRefresh, refreshTokenAfterRefreshSource, clientSideTokenGenerateSiteId, @@ -4201,7 +4202,7 @@ void cstgSuccessForBothOptedAndNonOptedOutTest(boolean optOutExpected, String id expectedPrivacyBits, genBody, expectedFirstLevelHashIdentity); - assertAreClientSideGeneratedTokens(adTokenFromRefresh, refreshTokenAfterRefreshSource, clientSideTokenGenerateSiteId, identityType, id); + assertAreClientSideGeneratedTokens(adTokenFromRefresh, refreshTokenAfterRefreshSource, clientSideTokenGenerateSiteId, diiType, id); assertEqualsClose(Instant.now().plusMillis(identityExpiresAfter.toMillis()), Instant.ofEpochMilli(refreshBody.getLong("identity_expires")), 10); assertEqualsClose(Instant.now().plusMillis(refreshExpiresAfter.toMillis()), Instant.ofEpochMilli(refreshBody.getLong("refresh_expires")), 10); assertEqualsClose(Instant.now().plusMillis(refreshIdentityAfter.toMillis()), Instant.ofEpochMilli(refreshBody.getLong("refresh_from")), 10); @@ -4226,7 +4227,7 @@ void cstgSuccessForBothOptedAndNonOptedOutTest(boolean optOutExpected, String id void cstgSaltsExpired(String httpOrigin, Vertx vertx, VertxTestContext testContext) throws NoSuchAlgorithmException, InvalidKeyException { when(saltProviderSnapshot.getExpires()).thenReturn(Instant.now().minus(1, ChronoUnit.HOURS)); setupCstgBackend("cstg.co.uk", "cstg2.com", "localhost"); - Tuple.Tuple2 data = createClientSideTokenGenerateRequest(IdentityType.Email, "random@unifiedid.com", Instant.now().toEpochMilli()); + Tuple.Tuple2 data = createClientSideTokenGenerateRequest(DiiType.Email, "random@unifiedid.com", Instant.now().toEpochMilli()); sendCstg(vertx, "v2/token/client-generate", httpOrigin, @@ -4240,7 +4241,7 @@ void cstgSaltsExpired(String httpOrigin, Vertx vertx, VertxTestContext testConte JsonObject refreshBody = respJson.getJsonObject("body"); assertNotNull(refreshBody); var encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); - validateAndGetToken(encoder, refreshBody, IdentityType.Email); //to validate token version is correct + validateAndGetToken(encoder, refreshBody, DiiType.Email); //to validate token version is correct verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(true); @@ -4252,7 +4253,7 @@ void cstgSaltsExpired(String httpOrigin, Vertx vertx, VertxTestContext testConte void cstgNoActiveKey(Vertx vertx, VertxTestContext testContext) throws NoSuchAlgorithmException, InvalidKeyException { setupCstgBackend("cstg.co.uk"); setupKeys(true); - Tuple.Tuple2 data = createClientSideTokenGenerateRequest(IdentityType.Email, "random@unifiedid.com", Instant.now().toEpochMilli()); + Tuple.Tuple2 data = createClientSideTokenGenerateRequest(DiiType.Email, "random@unifiedid.com", Instant.now().toEpochMilli()); sendCstg(vertx, "v2/token/client-generate", "http://cstg.co.uk", @@ -4295,20 +4296,20 @@ void cstgInvalidInput(String identityType, String rawUID, Vertx vertx, VertxTest }); } - private void assertAreClientSideGeneratedTokens(AdvertisingTokenRequest advertisingTokenRequest, TokenRefreshRequest tokenRefreshRequest, int siteId, IdentityType identityType, String identity) { + private void assertAreClientSideGeneratedTokens(AdvertisingTokenRequest advertisingTokenRequest, TokenRefreshRequest tokenRefreshRequest, int siteId, DiiType diiType, String identity) { assertAreClientSideGeneratedTokens(advertisingTokenRequest, tokenRefreshRequest, siteId, - identityType, + diiType, identity, false); } - private void assertAreClientSideGeneratedTokens(AdvertisingTokenRequest advertisingTokenRequest, TokenRefreshRequest tokenRefreshRequest, int siteId, IdentityType identityType, String identity, boolean expectedOptOut) { + private void assertAreClientSideGeneratedTokens(AdvertisingTokenRequest advertisingTokenRequest, TokenRefreshRequest tokenRefreshRequest, int siteId, DiiType diiType, String identity, boolean expectedOptOut) { final PrivacyBits advertisingTokenPrivacyBits = advertisingTokenRequest.privacyBits; final PrivacyBits refreshTokenPrivacyBits = tokenRefreshRequest.privacyBits; - final byte[] rawUid = getRawUidFromIdentity(identityType, + final byte[] rawUid = getRawUidFromIdentity(diiType, identity, firstLevelSalt, rotatingSalt123.getSalt()); @@ -4500,7 +4501,7 @@ void getActiveKeyTest() { @ValueSource(strings = {"MultiKeysets", "AddKey", "RotateKey", "DisableActiveKey", "DisableDefaultKeyset"}) void tokenGenerateRotatingKeysets_GENERATOR(String testRun, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 101; - final String emailHash = TokenUtils.getIdentityHashString("test@uid2.com"); + final String emailHash = TokenUtils.getDiiHashString("test@uid2.com"); fakeAuth(clientSiteId, Role.GENERATOR); MultipleKeysetsTests test = new MultipleKeysetsTests(); //To read these tests, open the MultipleKeysetsTests() constructor in another window so you can see the keyset contents and validate expectations @@ -4556,7 +4557,7 @@ void tokenGenerateRotatingKeysets_GENERATOR(String testRun, Vertx vertx, VertxTe assertNotNull(body); EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); - AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, body, IdentityType.Email); + AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, body, DiiType.Email); assertEquals(clientSiteId, advertisingTokenRequest.sourcePublisher.siteId); //Uses a key from default keyset int clientKeyId; diff --git a/src/test/java/com/uid2/operator/V2RequestUtilTest.java b/src/test/java/com/uid2/operator/V2RequestUtilTest.java index f296411e0..ee42d6a21 100644 --- a/src/test/java/com/uid2/operator/V2RequestUtilTest.java +++ b/src/test/java/com/uid2/operator/V2RequestUtilTest.java @@ -3,7 +3,7 @@ import ch.qos.logback.classic.Level; import ch.qos.logback.classic.Logger; import ch.qos.logback.classic.LoggerContext; -import com.uid2.operator.model.IdentityScope; +import com.uid2.operator.model.identities.IdentityScope; import com.uid2.operator.model.KeyManager; import com.uid2.operator.service.V2RequestUtil; import com.uid2.shared.IClock; diff --git a/src/test/java/com/uid2/operator/benchmark/BenchmarkCommon.java b/src/test/java/com/uid2/operator/benchmark/BenchmarkCommon.java index 6ab2447e7..b11c97f09 100644 --- a/src/test/java/com/uid2/operator/benchmark/BenchmarkCommon.java +++ b/src/test/java/com/uid2/operator/benchmark/BenchmarkCommon.java @@ -3,8 +3,10 @@ import com.uid2.operator.Const; import com.uid2.operator.Main; import com.uid2.operator.model.*; +import com.uid2.operator.model.identities.DiiType; import com.uid2.operator.model.identities.FirstLevelHash; import com.uid2.operator.model.identities.HashedDii; +import com.uid2.operator.model.identities.IdentityScope; import com.uid2.operator.service.EncryptedTokenEncoder; import com.uid2.operator.service.IUIDOperatorService; import com.uid2.operator.service.UIDOperatorService; @@ -155,7 +157,7 @@ static HashedDii[] createHashedDiiIdentities() { for (int i = 0; i < 65536; i++) { final byte[] diiHash = new byte[33]; new Random().nextBytes(diiHash); - arr[i] = new HashedDii(IdentityScope.UID2, IdentityType.Email, diiHash); + arr[i] = new HashedDii(IdentityScope.UID2, DiiType.Email, diiHash); } return arr; } From 2ae26b1534dac56c3f6bebe701c5acbcc159d60e Mon Sep 17 00:00:00 2001 From: Sunny Wu Date: Mon, 9 Dec 2024 19:50:33 +1100 Subject: [PATCH 155/431] 1. Renamed TokenUtils.java's functions to be blatently clear on the input value type 2. Fixed the old testIdentityMapForOptOutUser test that was using raw DII as a hash 3. Renamed all uses of IdentityString/IdentityHash variables to rawDii and hashedDii --- .../com/uid2/operator/service/InputUtil.java | 2 +- .../com/uid2/operator/service/TokenUtils.java | 36 +++--- .../operator/service/UIDOperatorService.java | 8 +- .../com/uid2/operator/TokenEncodingTest.java | 2 +- .../uid2/operator/UIDOperatorServiceTest.java | 14 +-- .../operator/UIDOperatorVerticleTest.java | 110 +++++++++--------- 6 files changed, 86 insertions(+), 86 deletions(-) diff --git a/src/main/java/com/uid2/operator/service/InputUtil.java b/src/main/java/com/uid2/operator/service/InputUtil.java index ab2b573b5..81ed4c6d3 100644 --- a/src/main/java/com/uid2/operator/service/InputUtil.java +++ b/src/main/java/com/uid2/operator/service/InputUtil.java @@ -197,7 +197,7 @@ public InputVal(String provided, String normalized, DiiType diiType, DiiInputTyp this.valid = valid; if (valid) { if (this.inputType == DiiInputType.Raw) { - this.diiInput = TokenUtils.getDiiHash(this.normalized); + this.diiInput = TokenUtils.getHashedDii(this.normalized); } else { this.diiInput = EncodingUtils.fromBase64(this.normalized); } diff --git a/src/main/java/com/uid2/operator/service/TokenUtils.java b/src/main/java/com/uid2/operator/service/TokenUtils.java index 0860db016..d6a0940e0 100644 --- a/src/main/java/com/uid2/operator/service/TokenUtils.java +++ b/src/main/java/com/uid2/operator/service/TokenUtils.java @@ -7,36 +7,36 @@ import java.util.Set; public class TokenUtils { - public static byte[] getDiiHash(String identityString) { - return EncodingUtils.getSha256Bytes(identityString); + public static byte[] getHashedDii(String rawDii) { + return EncodingUtils.getSha256Bytes(rawDii); } - public static String getDiiHashString(String identityString) { - return EncodingUtils.toBase64String(getDiiHash(identityString)); + public static String getHashedDiiString(String rawDii) { + return EncodingUtils.toBase64String(getHashedDii(rawDii)); } - public static byte[] getFirstLevelHash(byte[] identityHash, String firstLevelSalt) { - return getFirstLevelHashFromIdentityHash(EncodingUtils.toBase64String(identityHash), firstLevelSalt); + public static byte[] getFirstLevelHashFromHashedDii(byte[] hashedDii, String firstLevelSalt) { + return getFirstLevelHashFromHashedDii(EncodingUtils.toBase64String(hashedDii), firstLevelSalt); } - public static byte[] getFirstLevelHashFromIdentity(String identityString, String firstLevelSalt) { - return getFirstLevelHash(getDiiHash(identityString), firstLevelSalt); + public static byte[] getFirstLevelHashFromRawDii(String rawDii, String firstLevelSalt) { + return getFirstLevelHashFromHashedDii(getHashedDii(rawDii), firstLevelSalt); } - public static byte[] getFirstLevelHashFromIdentityHash(String identityHash, String firstLevelSalt) { - return EncodingUtils.getSha256Bytes(identityHash, firstLevelSalt); + public static byte[] getFirstLevelHashFromHashedDii(String hashedDii, String firstLevelSalt) { + return EncodingUtils.getSha256Bytes(hashedDii, firstLevelSalt); } public static byte[] getRawUidV2(byte[] firstLevelHash, String rotatingSalt) { return EncodingUtils.getSha256Bytes(EncodingUtils.toBase64String(firstLevelHash), rotatingSalt); } - public static byte[] getRawUidV2FromIdentity(String identityString, String firstLevelSalt, String rotatingSalt) { - return getRawUidV2(getFirstLevelHashFromIdentity(identityString, firstLevelSalt), rotatingSalt); + public static byte[] getRawUidV2FromRawDii(String rawDii, String firstLevelSalt, String rotatingSalt) { + return getRawUidV2(getFirstLevelHashFromRawDii(rawDii, firstLevelSalt), rotatingSalt); } - public static byte[] getRawUidV2FromIdentityHash(String identityString, String firstLevelSalt, String rotatingSalt) { - return getRawUidV2(getFirstLevelHashFromIdentityHash(identityString, firstLevelSalt), rotatingSalt); + public static byte[] getRawUidV2FromHashedDii(String hashedDii, String firstLevelSalt, String rotatingSalt) { + return getRawUidV2(getFirstLevelHashFromHashedDii(hashedDii, firstLevelSalt), rotatingSalt); } public static byte[] getRawUidV3(IdentityScope scope, DiiType type, byte[] firstLevelHash, String rotatingSalt) { @@ -47,12 +47,12 @@ public static byte[] getRawUidV3(IdentityScope scope, DiiType type, byte[] first return rawUid; } - public static byte[] getRawUidV3FromIdentity(IdentityScope scope, DiiType type, String identityString, String firstLevelSalt, String rotatingSalt) { - return getRawUidV3(scope, type, getFirstLevelHashFromIdentity(identityString, firstLevelSalt), rotatingSalt); + public static byte[] getRawUidV3FromRawDii(IdentityScope scope, DiiType type, String rawDii, String firstLevelSalt, String rotatingSalt) { + return getRawUidV3(scope, type, getFirstLevelHashFromRawDii(rawDii, firstLevelSalt), rotatingSalt); } - public static byte[] getRawUidV3FromIdentityHash(IdentityScope scope, DiiType type, String identityString, String firstLevelSalt, String rotatingSalt) { - return getRawUidV3(scope, type, getFirstLevelHashFromIdentityHash(identityString, firstLevelSalt), rotatingSalt); + public static byte[] getRawUidV3FromHashedDii(IdentityScope scope, DiiType type, String hashedDii, String firstLevelSalt, String rotatingSalt) { + return getRawUidV3(scope, type, getFirstLevelHashFromHashedDii(hashedDii, firstLevelSalt), rotatingSalt); } public static byte encodeIdentityScope(IdentityScope identityScope) { diff --git a/src/main/java/com/uid2/operator/service/UIDOperatorService.java b/src/main/java/com/uid2/operator/service/UIDOperatorService.java index c42bde293..96eb8fb5b 100644 --- a/src/main/java/com/uid2/operator/service/UIDOperatorService.java +++ b/src/main/java/com/uid2/operator/service/UIDOperatorService.java @@ -170,8 +170,8 @@ public IdentityMapResponseItem mapHashedDii(IdentityMapRequestItem request) { } @Override - public IdentityMapResponseItem map(HashedDii diiIdentity, Instant asOf) { - final FirstLevelHash firstLevelHash = getFirstLevelHashIdentity(diiIdentity, asOf); + public IdentityMapResponseItem map(HashedDii hashedDii, Instant asOf) { + final FirstLevelHash firstLevelHash = getFirstLevelHashIdentity(hashedDii, asOf); return generateRawUid(firstLevelHash, asOf); } @@ -233,8 +233,8 @@ private FirstLevelHash getFirstLevelHashIdentity(IdentityScope identityScope, Di return new FirstLevelHash(identityScope, diiType, firstLevelHash, null); } - private byte[] getFirstLevelHash(byte[] identityHash, Instant asOf) { - return TokenUtils.getFirstLevelHash(identityHash, getSaltProviderSnapshot(asOf).getFirstLevelSalt()); + private byte[] getFirstLevelHash(byte[] hashedDii, Instant asOf) { + return TokenUtils.getFirstLevelHashFromHashedDii(hashedDii, getSaltProviderSnapshot(asOf).getFirstLevelSalt()); } private IdentityMapResponseItem generateRawUid(FirstLevelHash firstLevelHash, Instant asOf) { diff --git a/src/test/java/com/uid2/operator/TokenEncodingTest.java b/src/test/java/com/uid2/operator/TokenEncodingTest.java index d216ad04b..f537a153f 100644 --- a/src/test/java/com/uid2/operator/TokenEncodingTest.java +++ b/src/test/java/com/uid2/operator/TokenEncodingTest.java @@ -55,7 +55,7 @@ public void testRefreshTokenEncoding(TokenVersion tokenVersion) { final EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(this.keyManager); final Instant now = EncodingUtils.NowUTCMillis(); - final byte[] firstLevelHash = TokenUtils.getFirstLevelHashFromIdentity("test@example.com", "some-salt"); + final byte[] firstLevelHash = TokenUtils.getFirstLevelHashFromRawDii("test@example.com", "some-salt"); final TokenRefreshRequest tokenRefreshRequest = new TokenRefreshRequest(tokenVersion, now, diff --git a/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java b/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java index 5c2e24e86..3f0d60b5a 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java @@ -24,7 +24,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import static com.uid2.operator.service.TokenUtils.getFirstLevelHash; +import static com.uid2.operator.service.TokenUtils.getFirstLevelHashFromHashedDii; import static org.junit.jupiter.api.Assertions.*; import org.junit.jupiter.params.ParameterizedTest; @@ -144,11 +144,11 @@ private void setNow(Instant now) { when(clock.instant()).thenAnswer(i -> this.now); } - private HashedDii createHashedDiiIdentity(String rawIdentityHash, IdentityScope scope, DiiType type) { + private HashedDii createHashedDii(String hashedDii, IdentityScope scope, DiiType type) { return new HashedDii( scope, type, - rawIdentityHash.getBytes(StandardCharsets.UTF_8) + hashedDii.getBytes(StandardCharsets.UTF_8) ); } @@ -189,7 +189,7 @@ public void testGenerateAndRefresh(int siteId, TokenVersion tokenVersion) { final TokenGenerateRequest tokenGenerateRequest = new TokenGenerateRequest( new SourcePublisher(siteId, 124, 125), - createHashedDiiIdentity("test-email-hash", expectedIdentityScope, expectedDiiType), + createHashedDii("test-email-hash", expectedIdentityScope, expectedDiiType), OptoutCheckPolicy.DoNotRespect, PrivacyBits.fromInt(0), this.now.minusSeconds(234) ); @@ -214,7 +214,7 @@ public void testGenerateAndRefresh(int siteId, TokenVersion tokenVersion) { assertIdentityScopeIdentityType(expectedIdentityScope, expectedDiiType, tokenRefreshRequest.firstLevelHash); assertEquals(tokenGenerateRequest.establishedAt, tokenRefreshRequest.firstLevelHash.establishedAt()); - final byte[] firstLevelHash = getFirstLevelHash(tokenGenerateRequest.hashedDii.hashedDii(), + final byte[] firstLevelHash = getFirstLevelHashFromHashedDii(tokenGenerateRequest.hashedDii.hashedDii(), saltProvider.getSnapshot(this.now).getFirstLevelSalt() ); assertArrayEquals(firstLevelHash, tokenRefreshRequest.firstLevelHash.firstLevelHash()); @@ -311,7 +311,7 @@ public void testTestOptOutKeyIdentityScopeMismatch() { "Phone,+01010101010,UID2", "Phone,+01010101010,EUID"}) public void testGenerateTokenForOptOutUser(DiiType type, String id, IdentityScope scope) { - final HashedDii hashedDii = createHashedDiiIdentity(TokenUtils.getDiiHashString(id), + final HashedDii hashedDii = createHashedDii(TokenUtils.getHashedDiiString(id), scope, type); final TokenGenerateRequest tokenGenerateRequestForceGenerate = new TokenGenerateRequest( @@ -364,7 +364,7 @@ public void testGenerateTokenForOptOutUser(DiiType type, String id, IdentityScop "Phone,+01010101010,UID2", "Phone,+01010101010,EUID"}) public void testIdentityMapForOptOutUser(DiiType type, String identity, IdentityScope scope) { - final HashedDii hashedDii = createHashedDiiIdentity(identity, scope, type); + final HashedDii hashedDii = createHashedDii(TokenUtils.getHashedDiiString(identity), scope, type); final Instant now = Instant.now(); final IdentityMapRequestItem mapRequestForceIdentityMapItem = new IdentityMapRequestItem( diff --git a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java index 5e19ddbcd..41783d5f2 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java @@ -623,26 +623,26 @@ private void assertTokenStatusMetrics(Integer siteId, TokenResponseStatsCollecto assertEquals(1, actual); } - private byte[] getRawUidFromIdentity(DiiType diiType, String identityString, String firstLevelSalt, String rotatingSalt) { - return getRawUid(diiType, identityString, firstLevelSalt, rotatingSalt, getIdentityScope(), useIdentityV3()); + private byte[] getRawUidFromRawDii(DiiType diiType, String rawDii, String firstLevelSalt, String rotatingSalt) { + return getRawUid(diiType, rawDii, firstLevelSalt, rotatingSalt, getIdentityScope(), useIdentityV3()); } - private static byte[] getRawUid(DiiType diiType, String identityString, String firstLevelSalt, String rotatingSalt, IdentityScope identityScope, boolean useIdentityV3) { + private static byte[] getRawUid(DiiType diiType, String rawDii, String firstLevelSalt, String rotatingSalt, IdentityScope identityScope, boolean useIdentityV3) { return !useIdentityV3 - ? TokenUtils.getRawUidV2FromIdentity(identityString, firstLevelSalt, rotatingSalt) - : TokenUtils.getRawUidV3FromIdentity(identityScope, diiType, identityString, firstLevelSalt, rotatingSalt); + ? TokenUtils.getRawUidV2FromRawDii(rawDii, firstLevelSalt, rotatingSalt) + : TokenUtils.getRawUidV3FromRawDii(identityScope, diiType, rawDii, firstLevelSalt, rotatingSalt); } - public static byte[] getRawUid(DiiType diiType, String identityString, IdentityScope identityScope, boolean useIdentityV3) { + public static byte[] getRawUid(DiiType diiType, String rawDii, IdentityScope identityScope, boolean useIdentityV3) { return !useIdentityV3 - ? TokenUtils.getRawUidV2FromIdentity(identityString, firstLevelSalt, rotatingSalt123.getSalt()) - : TokenUtils.getRawUidV3FromIdentity(identityScope, diiType, identityString, firstLevelSalt, rotatingSalt123.getSalt()); + ? TokenUtils.getRawUidV2FromRawDii(rawDii, firstLevelSalt, rotatingSalt123.getSalt()) + : TokenUtils.getRawUidV3FromRawDii(identityScope, diiType, rawDii, firstLevelSalt, rotatingSalt123.getSalt()); } - private byte[] getRawUidFromIdentityHash(DiiType diiType, String identityString, String firstLevelSalt, String rotatingSalt) { + private byte[] getRawUidFromHashedDii(DiiType diiType, String hashedDii, String firstLevelSalt, String rotatingSalt) { return !useIdentityV3() - ? TokenUtils.getRawUidV2FromIdentityHash(identityString, firstLevelSalt, rotatingSalt) - : TokenUtils.getRawUidV3FromIdentityHash(getIdentityScope(), diiType, identityString, firstLevelSalt, rotatingSalt); + ? TokenUtils.getRawUidV2FromHashedDii(hashedDii, firstLevelSalt, rotatingSalt) + : TokenUtils.getRawUidV3FromHashedDii(getIdentityScope(), diiType, hashedDii, firstLevelSalt, rotatingSalt); } private JsonObject createBatchEmailsRequestPayload() { @@ -766,7 +766,7 @@ void keyLatestHideRefreshKey(String apiVersion, Vertx vertx, VertxTestContext te void tokenGenerateBothEmailAndHashSpecified(String apiVersion, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; final String emailAddress = "test@uid2.com"; - final String emailHash = TokenUtils.getDiiHashString(emailAddress); + final String emailHash = TokenUtils.getHashedDiiString(emailAddress); fakeAuth(clientSiteId, Role.GENERATOR); setupSalts(); setupKeys(); @@ -1167,11 +1167,11 @@ void tokenGenerateOptOutToken(String policyParameterKey, String identity, DiiTyp AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, body, diiType); TokenRefreshRequest tokenRefreshRequest = encoder.decodeRefreshToken(body.getString("decrypted_refresh_token")); - final byte[] rawUid = getRawUidFromIdentity(diiType, + final byte[] rawUid = getRawUidFromRawDii(diiType, optOutTokenInput.getNormalized(), firstLevelSalt, rotatingSalt123.getSalt()); - final byte[] firstLevelHash = TokenUtils.getFirstLevelHashFromIdentity(optOutTokenInput.getNormalized(), firstLevelSalt); + final byte[] firstLevelHash = TokenUtils.getFirstLevelHashFromRawDii(optOutTokenInput.getNormalized(), firstLevelSalt); assertArrayEquals(rawUid, advertisingTokenRequest.rawUid.rawUid()); assertArrayEquals(firstLevelHash, tokenRefreshRequest.firstLevelHash.firstLevelHash()); @@ -1229,10 +1229,10 @@ void tokenGenerateForEmail(String apiVersion, Vertx vertx, VertxTestContext test TokenRefreshRequest tokenRefreshRequest = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), DiiType.Email); assertAdvertisingTokenRefreshTokenRequests(advertisingTokenRequest, tokenRefreshRequest, clientSiteId, - getRawUidFromIdentity(DiiType.Email, emailAddress, firstLevelSalt, rotatingSalt123.getSalt()), + getRawUidFromRawDii(DiiType.Email, emailAddress, firstLevelSalt, rotatingSalt123.getSalt()), PrivacyBits.DEFAULT, body, - TokenUtils.getFirstLevelHashFromIdentity(emailAddress, firstLevelSalt)); + TokenUtils.getFirstLevelHashFromRawDii(emailAddress, firstLevelSalt)); assertStatsCollector("/" + apiVersion + "/token/generate", null, "test-contact", clientSiteId); @@ -1265,7 +1265,7 @@ public void verifyPrivacyBits(PrivacyBits expectedValue, AdvertisingTokenRequest @ValueSource(strings = {"v1", "v2"}) void tokenGenerateForEmailHash(String apiVersion, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; - final String emailHash = TokenUtils.getDiiHashString("test@uid2.com"); + final String emailHash = TokenUtils.getHashedDiiString("test@uid2.com"); fakeAuth(clientSiteId, Role.GENERATOR); setupSalts(); setupKeys(); @@ -1286,10 +1286,10 @@ void tokenGenerateForEmailHash(String apiVersion, Vertx vertx, VertxTestContext TokenRefreshRequest tokenRefreshRequest = decodeRefreshToken(encoder, apiVersion.equals("v2") ? body.getString("decrypted_refresh_token") : body.getString("refresh_token"), DiiType.Email); assertAdvertisingTokenRefreshTokenRequests(advertisingTokenRequest, tokenRefreshRequest, clientSiteId, - getRawUidFromIdentityHash(DiiType.Email, emailHash, firstLevelSalt, rotatingSalt123.getSalt()), + getRawUidFromHashedDii(DiiType.Email, emailHash, firstLevelSalt, rotatingSalt123.getSalt()), PrivacyBits.DEFAULT, body, - TokenUtils.getFirstLevelHashFromIdentityHash(emailHash, firstLevelSalt)); + TokenUtils.getFirstLevelHashFromHashedDii(emailHash, firstLevelSalt)); testContext.completeNow(); }); @@ -1322,8 +1322,8 @@ void tokenGenerateThenRefresh(String apiVersion, Vertx vertx, VertxTestContext t when(this.optOutStore.getLatestEntry(any())).thenReturn(null); - byte[] expectedRawUidIdentity = getRawUidFromIdentity(DiiType.Email, emailAddress, firstLevelSalt, rotatingSalt123.getSalt()); - byte[] expectedFirstLevelHashIdentity = TokenUtils.getFirstLevelHashFromIdentity(emailAddress, firstLevelSalt); + byte[] expectedRawUidIdentity = getRawUidFromRawDii(DiiType.Email, emailAddress, firstLevelSalt, rotatingSalt123.getSalt()); + byte[] expectedFirstLevelHashIdentity = TokenUtils.getFirstLevelHashFromRawDii(emailAddress, firstLevelSalt); assertAdvertisingTokenRefreshTokenRequests(firstAdvertisingTokenRequest, firstTokenRefreshRequest, clientSiteId, expectedRawUidIdentity, @@ -1417,13 +1417,13 @@ void tokenGenerateThenRefreshSaltsExpired(String apiVersion, Vertx vertx, VertxT assertFalse(advertisingTokenRequest.privacyBits.isClientSideTokenGenerated()); assertFalse(advertisingTokenRequest.privacyBits.isClientSideTokenOptedOut()); assertEquals(clientSiteId, advertisingTokenRequest.sourcePublisher.siteId); - assertArrayEquals(getRawUidFromIdentity(DiiType.Email, emailAddress, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenRequest.rawUid.rawUid()); + assertArrayEquals(getRawUidFromRawDii(DiiType.Email, emailAddress, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenRequest.rawUid.rawUid()); String refreshTokenStringNew = refreshBody.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"); assertNotEquals(genRefreshToken, refreshTokenStringNew); TokenRefreshRequest tokenRefreshRequest = decodeRefreshToken(encoder, refreshTokenStringNew, DiiType.Email); assertEquals(clientSiteId, tokenRefreshRequest.sourcePublisher.siteId); - assertArrayEquals(TokenUtils.getFirstLevelHashFromIdentity(emailAddress, firstLevelSalt), tokenRefreshRequest.firstLevelHash.firstLevelHash()); + assertArrayEquals(TokenUtils.getFirstLevelHashFromRawDii(emailAddress, firstLevelSalt), tokenRefreshRequest.firstLevelHash.firstLevelHash()); assertEqualsClose(now.plusMillis(identityExpiresAfter.toMillis()), Instant.ofEpochMilli(refreshBody.getLong("identity_expires")), 10); assertEqualsClose(now.plusMillis(refreshExpiresAfter.toMillis()), Instant.ofEpochMilli(refreshBody.getLong("refresh_expires")), 10); @@ -1592,11 +1592,11 @@ void tokenGenerateUsingCustomSiteKey(String apiVersion, Vertx vertx, VertxTestCo AdvertisingTokenRequest advertisingTokenRequest = validateAndGetToken(encoder, body, DiiType.Email); assertEquals(clientSiteId, advertisingTokenRequest.sourcePublisher.siteId); - assertArrayEquals(getRawUidFromIdentity(DiiType.Email, emailAddress, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenRequest.rawUid.rawUid()); + assertArrayEquals(getRawUidFromRawDii(DiiType.Email, emailAddress, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenRequest.rawUid.rawUid()); TokenRefreshRequest tokenRefreshRequest = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), DiiType.Email); assertEquals(clientSiteId, tokenRefreshRequest.sourcePublisher.siteId); - assertArrayEquals(TokenUtils.getFirstLevelHashFromIdentity(emailAddress, firstLevelSalt), tokenRefreshRequest.firstLevelHash.firstLevelHash()); + assertArrayEquals(TokenUtils.getFirstLevelHashFromRawDii(emailAddress, firstLevelSalt), tokenRefreshRequest.firstLevelHash.firstLevelHash()); testContext.completeNow(); }); @@ -1631,11 +1631,11 @@ void tokenGenerateSaltsExpired(String apiVersion, Vertx vertx, VertxTestContext assertFalse(advertisingTokenRequest.privacyBits.isClientSideTokenGenerated()); assertFalse(advertisingTokenRequest.privacyBits.isClientSideTokenOptedOut()); assertEquals(clientSiteId, advertisingTokenRequest.sourcePublisher.siteId); - assertArrayEquals(getRawUidFromIdentity(DiiType.Email, emailAddress, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenRequest.rawUid.rawUid()); + assertArrayEquals(getRawUidFromRawDii(DiiType.Email, emailAddress, firstLevelSalt, rotatingSalt123.getSalt()), advertisingTokenRequest.rawUid.rawUid()); TokenRefreshRequest tokenRefreshRequest = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), DiiType.Email); assertEquals(clientSiteId, tokenRefreshRequest.sourcePublisher.siteId); - assertArrayEquals(TokenUtils.getFirstLevelHashFromIdentity(emailAddress, firstLevelSalt), tokenRefreshRequest.firstLevelHash.firstLevelHash()); + assertArrayEquals(TokenUtils.getFirstLevelHashFromRawDii(emailAddress, firstLevelSalt), tokenRefreshRequest.firstLevelHash.firstLevelHash()); assertEqualsClose(now.plusMillis(identityExpiresAfter.toMillis()), Instant.ofEpochMilli(body.getLong("identity_expires")), 10); assertEqualsClose(now.plusMillis(refreshExpiresAfter.toMillis()), Instant.ofEpochMilli(body.getLong("refresh_expires")), 10); @@ -1942,7 +1942,7 @@ void tokenValidateWithEmailHash_Mismatch(String apiVersion, Vertx vertx, VertxTe void identityMapBothEmailAndHashSpecified(Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; final String emailAddress = "test@uid2.com"; - final String emailHash = TokenUtils.getDiiHashString(emailAddress); + final String emailHash = TokenUtils.getHashedDiiString(emailAddress); fakeAuth(clientSiteId, Role.MAPPER); setupSalts(); setupKeys(); @@ -2030,7 +2030,7 @@ void identityMapForSaltsExpired(Vertx vertx, VertxTestContext testContext) { @Test void identityMapForEmailHash(Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; - final String emailHash = TokenUtils.getDiiHashString("test@uid2.com"); + final String emailHash = TokenUtils.getHashedDiiString("test@uid2.com"); fakeAuth(clientSiteId, Role.MAPPER); setupSalts(); setupKeys(); @@ -2086,7 +2086,7 @@ void identityMapBatchBothEmailAndHashSpecified(String apiVersion, Vertx vertx, V req.put("email_hash", emailHashes); emails.add("test1@uid2.com"); - emailHashes.add(TokenUtils.getDiiHashString("test2@uid2.com")); + emailHashes.add(TokenUtils.getHashedDiiString("test2@uid2.com")); send(apiVersion, vertx, apiVersion + "/identity/map", false, null, req, 400, respJson -> { assertFalse(respJson.containsKey("body")); @@ -2225,8 +2225,8 @@ void identityMapBatchEmailHashes(String apiVersion, Vertx vertx, VertxTestContex JsonArray hashes = new JsonArray(); req.put("email_hash", hashes); final String[] email_hashes = { - TokenUtils.getDiiHashString("test1@uid2.com"), - TokenUtils.getDiiHashString("test2@uid2.com"), + TokenUtils.getHashedDiiString("test1@uid2.com"), + TokenUtils.getHashedDiiString("test2@uid2.com"), }; for (String email_hash : email_hashes) { @@ -2453,7 +2453,7 @@ void LogoutV2SaltsExpired(Vertx vertx, VertxTestContext testContext) { void tokenGenerateBothPhoneAndHashSpecified(String apiVersion, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; final String phone = "+15555555555"; - final String phoneHash = TokenUtils.getDiiHashString(phone); + final String phoneHash = TokenUtils.getHashedDiiString(phone); fakeAuth(clientSiteId, Role.GENERATOR); setupSalts(); setupKeys(); @@ -2499,9 +2499,9 @@ void tokenGenerateBothPhoneAndEmailSpecified(String apiVersion, Vertx vertx, Ver void tokenGenerateBothPhoneHashAndEmailHashSpecified(String apiVersion, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; final String phone = "+15555555555"; - final String phoneHash = TokenUtils.getDiiHashString(phone); + final String phoneHash = TokenUtils.getHashedDiiString(phone); final String emailAddress = "test@uid2.com"; - final String emailHash = TokenUtils.getDiiHashString(emailAddress); + final String emailHash = TokenUtils.getHashedDiiString(emailAddress); fakeAuth(clientSiteId, Role.GENERATOR); setupSalts(); setupKeys(); @@ -2542,10 +2542,10 @@ void tokenGenerateForPhone(String apiVersion, Vertx vertx, VertxTestContext test TokenRefreshRequest tokenRefreshRequest = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), DiiType.Phone); assertAdvertisingTokenRefreshTokenRequests(advertisingTokenRequest, tokenRefreshRequest, clientSiteId, - getRawUidFromIdentity(DiiType.Phone, phone, firstLevelSalt, rotatingSalt123.getSalt()), + getRawUidFromRawDii(DiiType.Phone, phone, firstLevelSalt, rotatingSalt123.getSalt()), PrivacyBits.DEFAULT, body, - TokenUtils.getFirstLevelHashFromIdentity(phone, firstLevelSalt)); + TokenUtils.getFirstLevelHashFromRawDii(phone, firstLevelSalt)); testContext.completeNow(); }); @@ -2568,7 +2568,7 @@ void verifyFirstLevelHashIdentityAndEstablishedAt(byte[] expectedFirstLevelHash, void tokenGenerateForPhoneHash(String apiVersion, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; final String phone = "+15555555555"; - final String phoneHash = TokenUtils.getDiiHashString(phone); + final String phoneHash = TokenUtils.getHashedDiiString(phone); fakeAuth(clientSiteId, Role.GENERATOR); setupSalts(); setupKeys(); @@ -2587,10 +2587,10 @@ void tokenGenerateForPhoneHash(String apiVersion, Vertx vertx, VertxTestContext TokenRefreshRequest tokenRefreshRequest = decodeRefreshToken(encoder, body.getString(apiVersion.equals("v2") ? "decrypted_refresh_token" : "refresh_token"), DiiType.Phone); assertAdvertisingTokenRefreshTokenRequests(advertisingTokenRequest, tokenRefreshRequest, clientSiteId, - getRawUidFromIdentity(DiiType.Phone, phone, firstLevelSalt, rotatingSalt123.getSalt()), + getRawUidFromRawDii(DiiType.Phone, phone, firstLevelSalt, rotatingSalt123.getSalt()), PrivacyBits.DEFAULT, body, - TokenUtils.getFirstLevelHashFromIdentity(phone, firstLevelSalt)); + TokenUtils.getFirstLevelHashFromRawDii(phone, firstLevelSalt)); testContext.completeNow(); @@ -2619,8 +2619,8 @@ void tokenGenerateThenRefreshForPhone(String apiVersion, Vertx vertx, VertxTestC when(this.optOutStore.getLatestEntry(any())).thenReturn(null); - byte[] expectedRawUidIdentity = getRawUidFromIdentity(DiiType.Phone, phone, firstLevelSalt, rotatingSalt123.getSalt()); - byte[] expectedFirstLevelHashIdentity = TokenUtils.getFirstLevelHashFromIdentity(phone, firstLevelSalt); + byte[] expectedRawUidIdentity = getRawUidFromRawDii(DiiType.Phone, phone, firstLevelSalt, rotatingSalt123.getSalt()); + byte[] expectedFirstLevelHashIdentity = TokenUtils.getFirstLevelHashFromRawDii(phone, firstLevelSalt); assertAdvertisingTokenRefreshTokenRequests(firstAdvertisingTokenRequest, firstTokenRefreshRequest, clientSiteId, expectedRawUidIdentity, @@ -2859,7 +2859,7 @@ void tokenRefreshOptOutBeforeLoginForPhone(String apiVersion, Vertx vertx, Vertx void identityMapBothPhoneAndHashSpecified(Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; final String phone = "+15555555555"; - final String phoneHash = TokenUtils.getDiiHashString(phone); + final String phoneHash = TokenUtils.getHashedDiiString(phone); fakeAuth(clientSiteId, Role.MAPPER); setupSalts(); setupKeys(); @@ -2903,7 +2903,7 @@ void identityMapForPhone(Vertx vertx, VertxTestContext testContext) { void identityMapForPhoneHash(Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 201; final String phone = "+15555555555"; - final String phonneHash = TokenUtils.getDiiHashString(phone); + final String phonneHash = TokenUtils.getHashedDiiString(phone); fakeAuth(clientSiteId, Role.MAPPER); setupSalts(); setupKeys(); @@ -2978,7 +2978,7 @@ void identityMapBatchBothPhoneAndHashSpecified(String apiVersion, Vertx vertx, V req.put("phone_hash", phoneHashes); phones.add("+15555555555"); - phoneHashes.add(TokenUtils.getDiiHashString("+15555555555")); + phoneHashes.add(TokenUtils.getHashedDiiString("+15555555555")); send(apiVersion, vertx, apiVersion + "/identity/map", false, null, req, 400, respJson -> { assertFalse(respJson.containsKey("body")); @@ -3020,8 +3020,8 @@ void identityMapBatchPhoneHashes(String apiVersion, Vertx vertx, VertxTestContex JsonArray hashes = new JsonArray(); req.put("phone_hash", hashes); final String[] email_hashes = { - TokenUtils.getDiiHashString("+15555555555"), - TokenUtils.getDiiHashString("+15555555556"), + TokenUtils.getHashedDiiString("+15555555555"), + TokenUtils.getHashedDiiString("+15555555556"), }; for (String email_hash : email_hashes) { @@ -3278,9 +3278,9 @@ private void setupCstgBackend(List domainNames, List appNames) when(siteProvider.getSite(clientSideTokenGenerateSiteId)).thenReturn(site); } - //if no identity is provided will get an error + //if no hashed dii is provided will get an error @Test - void cstgNoIdentityHashProvided(Vertx vertx, VertxTestContext testContext) throws NoSuchAlgorithmException, InvalidKeyException { + void cstgNoHashedDiiProvided(Vertx vertx, VertxTestContext testContext) throws NoSuchAlgorithmException, InvalidKeyException { setupCstgBackend("cstg.co.uk"); Tuple.Tuple2 data = createClientSideTokenGenerateRequestWithNoPayload(Instant.now().toEpochMilli()); sendCstg(vertx, @@ -4080,7 +4080,7 @@ void cstgUserOptsOutAfterTokenGenerate(String id, DiiType diiType, Vertx vertx, testContext, response -> { verify(optOutStore, times(1)).getLatestEntry(argumentCaptor.capture()); - assertArrayEquals(TokenUtils.getFirstLevelHashFromIdentity(id, firstLevelSalt), + assertArrayEquals(TokenUtils.getFirstLevelHashFromRawDii(id, firstLevelSalt), argumentCaptor.getValue().firstLevelHash()); assertEquals("success", response.getString("status")); @@ -4156,8 +4156,8 @@ void cstgSuccessForBothOptedAndNonOptedOutTest(boolean optOutExpected, String id - byte[] expectedRawUidIdentity = getRawUidFromIdentity(diiType, id, firstLevelSalt, rotatingSalt123.getSalt()); - byte[] expectedFirstLevelHashIdentity = TokenUtils.getFirstLevelHashFromIdentity(id, firstLevelSalt); + byte[] expectedRawUidIdentity = getRawUidFromRawDii(diiType, id, firstLevelSalt, rotatingSalt123.getSalt()); + byte[] expectedFirstLevelHashIdentity = TokenUtils.getFirstLevelHashFromRawDii(id, firstLevelSalt); PrivacyBits expectedPrivacyBits = new PrivacyBits(); expectedPrivacyBits.setLegacyBit(); @@ -4309,12 +4309,12 @@ private void assertAreClientSideGeneratedTokens(AdvertisingTokenRequest advertis final PrivacyBits advertisingTokenPrivacyBits = advertisingTokenRequest.privacyBits; final PrivacyBits refreshTokenPrivacyBits = tokenRefreshRequest.privacyBits; - final byte[] rawUid = getRawUidFromIdentity(diiType, + final byte[] rawUid = getRawUidFromRawDii(diiType, identity, firstLevelSalt, rotatingSalt123.getSalt()); - final byte[] firstLevelHash = TokenUtils.getFirstLevelHashFromIdentity(identity, firstLevelSalt); + final byte[] firstLevelHash = TokenUtils.getFirstLevelHashFromRawDii(identity, firstLevelSalt); assertAll( () -> assertTrue(advertisingTokenPrivacyBits.isClientSideTokenGenerated(), "Advertising token privacy bits CSTG flag is incorrect"), @@ -4501,7 +4501,7 @@ void getActiveKeyTest() { @ValueSource(strings = {"MultiKeysets", "AddKey", "RotateKey", "DisableActiveKey", "DisableDefaultKeyset"}) void tokenGenerateRotatingKeysets_GENERATOR(String testRun, Vertx vertx, VertxTestContext testContext) { final int clientSiteId = 101; - final String emailHash = TokenUtils.getDiiHashString("test@uid2.com"); + final String emailHash = TokenUtils.getHashedDiiString("test@uid2.com"); fakeAuth(clientSiteId, Role.GENERATOR); MultipleKeysetsTests test = new MultipleKeysetsTests(); //To read these tests, open the MultipleKeysetsTests() constructor in another window so you can see the keyset contents and validate expectations From e5dea0e019116f772235449e5fd286c3d48321c2 Mon Sep 17 00:00:00 2001 From: Vishal Egbert Date: Tue, 10 Dec 2024 15:15:06 +1100 Subject: [PATCH 156/431] Increase max form buffer --- src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index 20472f9ec..617074b9b 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -40,6 +40,7 @@ import io.vertx.core.Promise; import io.vertx.core.buffer.Buffer; import io.vertx.core.http.HttpHeaders; +import io.vertx.core.http.HttpServerOptions; import io.vertx.core.http.HttpServerResponse; import io.vertx.core.json.DecodeException; import io.vertx.core.json.JsonArray; @@ -201,7 +202,7 @@ public void start(Promise startPromise) throws Exception { final Router router = createRoutesSetup(); final int port = Const.Port.ServicePortForOperator + Utils.getPortOffset(); - vertx.createHttpServer() + vertx.createHttpServer(new HttpServerOptions().setMaxFormBufferedBytes((int) MAX_REQUEST_BODY_SIZE)) .requestHandler(router) .listen(port, result -> { if (result.succeeded()) { From 1b72e558f3c3bb9debaa2392ae98d6a99301d8ff Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Tue, 10 Dec 2024 04:36:08 +0000 Subject: [PATCH 157/431] [CI Pipeline] Released Patch version: 5.43.4 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d817ec3dd..69634018d 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.43.0 + 5.43.4 UTF-8 From b7922328eb244933037e0d3aa97fe5592232dbdb Mon Sep 17 00:00:00 2001 From: Sunny Wu Date: Tue, 10 Dec 2024 17:16:26 +1100 Subject: [PATCH 158/431] Use correct raw UID version for UID2 UIDOperatorVerticleTest and rename identityV3Enabled to rawUidV3Enabled (#1190) * Use correct raw UID version for UID2's UIDOperatorVerticleTest * renamed the variable/method useIdentityV3/identityV3Enabled to useRawUidV3/rawUidV3Enabled to be up to date with latest terminlogies * Improved TokenEncodingTest#testAdvertisingTokenEncodings to tests all combo's of raw UID and ad token versions --- .../operator/service/UIDOperatorService.java | 7 +++--- .../operator/EUIDOperatorVerticleTest.java | 2 ++ .../com/uid2/operator/TokenEncodingTest.java | 23 ++++++++++++------ .../operator/UIDOperatorVerticleTest.java | 24 +++++++++---------- 4 files changed, 34 insertions(+), 22 deletions(-) diff --git a/src/main/java/com/uid2/operator/service/UIDOperatorService.java b/src/main/java/com/uid2/operator/service/UIDOperatorService.java index 6d4ff86d0..5e66dd70c 100644 --- a/src/main/java/com/uid2/operator/service/UIDOperatorService.java +++ b/src/main/java/com/uid2/operator/service/UIDOperatorService.java @@ -47,7 +47,8 @@ public class UIDOperatorService implements IUIDOperatorService { private final OperatorIdentity operatorIdentity; private final TokenVersion refreshTokenVersion; - private final boolean identityV3Enabled; + // if we use Raw UID v3 format for the raw UID2/EUIDs generated in this operator + private final boolean rawUidV3Enabled; private final Handler saltRetrievalResponseHandler; @@ -90,7 +91,7 @@ public UIDOperatorService(JsonObject config, IOptOutStore optOutStore, ISaltProv } this.refreshTokenVersion = TokenVersion.V3; - this.identityV3Enabled = config.getBoolean("identity_v3", false); + this.rawUidV3Enabled = config.getBoolean("identity_v3", false); } @Override @@ -230,7 +231,7 @@ private MappedIdentity getAdvertisingId(UserIdentity firstLevelHashIdentity, Ins final SaltEntry rotatingSalt = getSaltProviderSnapshot(asOf).getRotatingSalt(firstLevelHashIdentity.id); return new MappedIdentity( - this.identityV3Enabled + this.rawUidV3Enabled ? TokenUtils.getAdvertisingIdV3(firstLevelHashIdentity.identityScope, firstLevelHashIdentity.identityType, firstLevelHashIdentity.id, rotatingSalt.getSalt()) : TokenUtils.getAdvertisingIdV2(firstLevelHashIdentity.id, rotatingSalt.getSalt()), rotatingSalt.getHashedId()); diff --git a/src/test/java/com/uid2/operator/EUIDOperatorVerticleTest.java b/src/test/java/com/uid2/operator/EUIDOperatorVerticleTest.java index 138e17777..7c894fba6 100644 --- a/src/test/java/com/uid2/operator/EUIDOperatorVerticleTest.java +++ b/src/test/java/com/uid2/operator/EUIDOperatorVerticleTest.java @@ -21,6 +21,8 @@ public EUIDOperatorVerticleTest() throws IOException { @Override protected IdentityScope getIdentityScope() { return IdentityScope.EUID; } @Override + protected boolean useRawUidV3() { return true; } + @Override protected void addAdditionalTokenGenerateParams(JsonObject payload) { if (payload != null && !payload.containsKey("tcf_consent_string")) { payload.put("tcf_consent_string", "CPehNtWPehNtWABAMBFRACBoALAAAEJAAIYgAKwAQAKgArABAAqAAA"); diff --git a/src/test/java/com/uid2/operator/TokenEncodingTest.java b/src/test/java/com/uid2/operator/TokenEncodingTest.java index c77c81b78..73e11309c 100644 --- a/src/test/java/com/uid2/operator/TokenEncodingTest.java +++ b/src/test/java/com/uid2/operator/TokenEncodingTest.java @@ -16,6 +16,7 @@ import io.vertx.core.json.JsonObject; import org.junit.Assert; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.EnumSource; import java.time.Instant; @@ -86,15 +87,23 @@ public void testRefreshTokenEncoding(TokenVersion tokenVersion) { } @ParameterizedTest - @EnumSource(TokenVersion.class) - public void testAdvertisingTokenEncodings(TokenVersion tokenVersion) { + @CsvSource({"false, V4", //same as current UID2 prod (as at 2024-12-10) + "true, V4", //same as current EUID prod (as at 2024-12-10) + //the following combinations aren't used in any UID2/EUID environments but just testing them regardless + "false, V3", + "true, V3", + "false, V2", + "true, V2", + } + ) + public void testAdvertisingTokenEncodings(boolean useRawUIDv3, TokenVersion adTokenVersion) { final EncryptedTokenEncoder encoder = new EncryptedTokenEncoder(this.keyManager); final Instant now = EncodingUtils.NowUTCMillis(); - final byte[] rawUid = UIDOperatorVerticleTest.getRawUid(IdentityType.Email, "test@example.com", IdentityScope.UID2, tokenVersion != TokenVersion.V2); + final byte[] rawUid = UIDOperatorVerticleTest.getRawUid(IdentityType.Email, "test@example.com", IdentityScope.UID2, useRawUIDv3); final AdvertisingToken token = new AdvertisingToken( - tokenVersion, + adTokenVersion, now, now.plusSeconds(60), new OperatorIdentity(101, OperatorType.Service, 102, 103), @@ -103,9 +112,9 @@ public void testAdvertisingTokenEncodings(TokenVersion tokenVersion) { ); final byte[] encodedBytes = encoder.encode(token, now); - final AdvertisingToken decoded = encoder.decodeAdvertisingToken(EncryptedTokenEncoder.bytesToBase64Token(encodedBytes, tokenVersion)); + final AdvertisingToken decoded = encoder.decodeAdvertisingToken(EncryptedTokenEncoder.bytesToBase64Token(encodedBytes, adTokenVersion)); - assertEquals(tokenVersion, decoded.version); + assertEquals(adTokenVersion, decoded.version); assertEquals(token.createdAt, decoded.createdAt); assertEquals(token.expiresAt, decoded.expiresAt); assertTrue(token.userIdentity.matches(decoded.userIdentity)); @@ -114,7 +123,7 @@ public void testAdvertisingTokenEncodings(TokenVersion tokenVersion) { assertEquals(token.publisherIdentity.siteId, decoded.publisherIdentity.siteId); Buffer b = Buffer.buffer(encodedBytes); - int keyId = b.getInt(tokenVersion == TokenVersion.V2 ? 1 : 2); //TODO - extract master key from token should be a helper function + int keyId = b.getInt(adTokenVersion == TokenVersion.V2 ? 1 : 2); //TODO - extract master key from token should be a helper function assertEquals(Data.MasterKeySiteId, keyManager.getSiteIdFromKeyId(keyId)); } } diff --git a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java index d9a91ae01..82ab057d0 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java @@ -13,7 +13,6 @@ import com.uid2.operator.util.Tuple; import com.uid2.operator.vertx.OperatorShutdownHandler; import com.uid2.operator.vertx.UIDOperatorVerticle; -import com.uid2.operator.vertx.ClientInputValidationException; import com.uid2.shared.Utils; import com.uid2.shared.auth.ClientKey; import com.uid2.shared.auth.Keyset; @@ -27,9 +26,7 @@ import com.uid2.shared.secret.KeyHasher; import com.uid2.shared.store.*; import com.uid2.shared.store.reader.RotatingKeysetProvider; -import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.Metrics; -import io.micrometer.core.instrument.search.MeterNotFoundException; import io.micrometer.core.instrument.simple.SimpleMeterRegistry; import io.vertx.core.AsyncResult; import io.vertx.core.Future; @@ -158,7 +155,7 @@ private void setupConfig(JsonObject config) { config.put(Const.Config.SharingTokenExpiryProp, 60 * 60 * 24 * 30); config.put("identity_scope", getIdentityScope().toString()); - config.put("identity_v3", useIdentityV3()); + config.put("identity_v3", useRawUidV3()); config.put("client_side_token_generate", true); config.put("key_sharing_endpoint_provide_app_names", true); config.put("client_side_token_generate_log_invalid_http_origins", true); @@ -622,23 +619,23 @@ private void assertTokenStatusMetrics(Integer siteId, TokenResponseStatsCollecto } private byte[] getAdvertisingIdFromIdentity(IdentityType identityType, String identityString, String firstLevelSalt, String rotatingSalt) { - return getRawUid(identityType, identityString, firstLevelSalt, rotatingSalt, getIdentityScope(), useIdentityV3()); + return getRawUid(identityType, identityString, firstLevelSalt, rotatingSalt, getIdentityScope(), useRawUidV3()); } - private static byte[] getRawUid(IdentityType identityType, String identityString, String firstLevelSalt, String rotatingSalt, IdentityScope identityScope, boolean useIdentityV3) { - return !useIdentityV3 + private static byte[] getRawUid(IdentityType identityType, String identityString, String firstLevelSalt, String rotatingSalt, IdentityScope identityScope, boolean useRawUidV3) { + return !useRawUidV3 ? TokenUtils.getAdvertisingIdV2FromIdentity(identityString, firstLevelSalt, rotatingSalt) : TokenUtils.getAdvertisingIdV3FromIdentity(identityScope, identityType, identityString, firstLevelSalt, rotatingSalt); } - public static byte[] getRawUid(IdentityType identityType, String identityString, IdentityScope identityScope, boolean useIdentityV3) { - return !useIdentityV3 + public static byte[] getRawUid(IdentityType identityType, String identityString, IdentityScope identityScope, boolean useRawUidV3) { + return !useRawUidV3 ? TokenUtils.getAdvertisingIdV2FromIdentity(identityString, firstLevelSalt, rotatingSalt123.getSalt()) : TokenUtils.getAdvertisingIdV3FromIdentity(identityScope, identityType, identityString, firstLevelSalt, rotatingSalt123.getSalt()); } private byte[] getAdvertisingIdFromIdentityHash(IdentityType identityType, String identityString, String firstLevelSalt, String rotatingSalt) { - return !useIdentityV3() + return !useRawUidV3() ? TokenUtils.getAdvertisingIdV2FromIdentityHash(identityString, firstLevelSalt, rotatingSalt) : TokenUtils.getAdvertisingIdV3FromIdentityHash(getIdentityScope(), identityType, identityString, firstLevelSalt, rotatingSalt); } @@ -665,7 +662,7 @@ private JsonObject setupIdentityMapServiceLinkTest() { protected TokenVersion getTokenVersion() {return TokenVersion.V4;} - final boolean useIdentityV3() { return getTokenVersion() != TokenVersion.V2; } + protected boolean useRawUidV3() { return false; } protected IdentityScope getIdentityScope() { return IdentityScope.UID2; } protected void addAdditionalTokenGenerateParams(JsonObject payload) {} @@ -816,7 +813,10 @@ private AdvertisingToken validateAndGetToken(EncryptedTokenEncoder encoder, Json final String advertisingTokenString = body.getString("advertising_token"); validateAdvertisingToken(advertisingTokenString, getTokenVersion(), getIdentityScope(), identityType); AdvertisingToken advertisingToken = encoder.decodeAdvertisingToken(advertisingTokenString); - if (getTokenVersion() == TokenVersion.V4) { + + // without useIdentityV3() the assert will be trigger as there's no IdentityType in v4 token generated with + // a raw UID v2 as old raw UID format doesn't store the identity type (and scope) + if (useRawUidV3() && getTokenVersion() == TokenVersion.V4) { assertEquals(identityType, advertisingToken.userIdentity.identityType); } return advertisingToken; From 5383c0b5043ba0806b56ae2a1f6640ba9a54bc42 Mon Sep 17 00:00:00 2001 From: Cody Constine Date: Wed, 11 Dec 2024 17:01:40 -0700 Subject: [PATCH 159/431] Adding new readers for salts and client side keypairs --- pom.xml | 2 +- src/main/java/com/uid2/operator/Main.java | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 13a9c3676..99c48a083 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ 2.1.0 2.1.0 2.1.0 - 8.0.6 + 8.0.13-alpha-175-SNAPSHOT ${project.version} 21 21 diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index 8a5c8d5cb..19cf7bd1f 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -23,6 +23,7 @@ import com.uid2.shared.jmx.AdminApi; import com.uid2.shared.optout.OptOutCloudSync; import com.uid2.shared.store.CloudPath; +import com.uid2.shared.store.EncryptedRotatingSaltProvider; import com.uid2.shared.store.RotatingSaltProvider; import com.uid2.shared.store.reader.*; import com.uid2.shared.store.scope.GlobalScope; @@ -138,7 +139,7 @@ public Main(Vertx vertx, JsonObject config) throws Exception { this.cloudEncryptionKeyProvider = new RotatingCloudEncryptionKeyApiProvider(fsStores, new GlobalScope(new CloudPath(cloudEncryptionKeyMdPath))); String sitesMdPath = this.config.getString(Const.Config.SitesMetadataPathProp); String keypairMdPath = this.config.getString(Const.Config.ClientSideKeypairsMetadataPathProp); - this.clientSideKeypairProvider = new RotatingClientSideKeypairStore(fsStores, new GlobalScope(new CloudPath(keypairMdPath))); + this.clientSideKeypairProvider = new RotatingClientSideKeypairStore(fsStores, new GlobalScope(new CloudPath(keypairMdPath)), cloudEncryptionKeyProvider); String clientsMdPath = this.config.getString(Const.Config.ClientsMetadataPathProp); this.clientKeyProvider = new RotatingClientKeyProvider(fsStores, new GlobalScope(new CloudPath(clientsMdPath)), cloudEncryptionKeyProvider); String keysetKeysMdPath = this.config.getString(Const.Config.KeysetKeysMetadataPathProp); @@ -146,7 +147,7 @@ public Main(Vertx vertx, JsonObject config) throws Exception { String keysetMdPath = this.config.getString(Const.Config.KeysetsMetadataPathProp); this.keysetProvider = new RotatingKeysetProvider(fsStores, new GlobalScope(new CloudPath(keysetMdPath)), cloudEncryptionKeyProvider); String saltsMdPath = this.config.getString(Const.Config.SaltsMetadataPathProp); - this.saltProvider = new RotatingSaltProvider(fsStores, saltsMdPath); + this.saltProvider = new EncryptedRotatingSaltProvider(fsStores, saltsMdPath, cloudEncryptionKeyProvider); this.optOutStore = new CloudSyncOptOutStore(vertx, fsLocal, this.config, operatorKey, Clock.systemUTC()); if (this.validateServiceLinks) { From bfe7849b38bec99057e769ab8350cd5a18fdeb66 Mon Sep 17 00:00:00 2001 From: Cody Constine Date: Fri, 13 Dec 2024 11:55:22 -0700 Subject: [PATCH 160/431] Finishing salts and clientside keys --- pom.xml | 4 +-- src/main/java/com/uid2/operator/Main.java | 2 +- .../uid2/operator/reader/ApiStoreReader.java | 15 +++++++--- ...RotatingCloudEncryptionKeyApiProvider.java | 30 +++---------------- ...tingCloudEncryptionKeyApiProviderTest.java | 11 ++++--- 5 files changed, 23 insertions(+), 39 deletions(-) diff --git a/pom.xml b/pom.xml index 99c48a083..ff7f61849 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.42.7-alpha-139-SNAPSHOT + 6.0.0 UTF-8 @@ -22,7 +22,7 @@ 2.1.0 2.1.0 2.1.0 - 8.0.13-alpha-175-SNAPSHOT + 8.0.14-alpha-176-SNAPSHOT ${project.version} 21 21 diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index 19cf7bd1f..04643d21e 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -147,7 +147,7 @@ public Main(Vertx vertx, JsonObject config) throws Exception { String keysetMdPath = this.config.getString(Const.Config.KeysetsMetadataPathProp); this.keysetProvider = new RotatingKeysetProvider(fsStores, new GlobalScope(new CloudPath(keysetMdPath)), cloudEncryptionKeyProvider); String saltsMdPath = this.config.getString(Const.Config.SaltsMetadataPathProp); - this.saltProvider = new EncryptedRotatingSaltProvider(fsStores, saltsMdPath, cloudEncryptionKeyProvider); + this.saltProvider = new EncryptedRotatingSaltProvider(fsStores, cloudEncryptionKeyProvider, new GlobalScope(new CloudPath(saltsMdPath))); this.optOutStore = new CloudSyncOptOutStore(vertx, fsLocal, this.config, operatorKey, Clock.systemUTC()); if (this.validateServiceLinks) { diff --git a/src/main/java/com/uid2/operator/reader/ApiStoreReader.java b/src/main/java/com/uid2/operator/reader/ApiStoreReader.java index 78937e16c..fda4c4e6a 100644 --- a/src/main/java/com/uid2/operator/reader/ApiStoreReader.java +++ b/src/main/java/com/uid2/operator/reader/ApiStoreReader.java @@ -11,6 +11,7 @@ import org.slf4j.LoggerFactory; import java.io.ByteArrayInputStream; +import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; @@ -21,13 +22,19 @@ public ApiStoreReader(DownloadCloudStorage fileStreamProvider, StoreScope scope, super(fileStreamProvider, scope, parser, dataTypeName); } + public long loadContent(JsonObject contents) throws Exception { + return loadContent(contents, dataTypeName); + } + + @Override + public long loadContent(JsonObject contents, String dataType) throws IOException { if (contents == null) { - throw new IllegalArgumentException(String.format("No contents provided for loading data type %s, cannot load content", dataTypeName)); + throw new IllegalArgumentException(String.format("No contents provided for loading data type %s, cannot load content", dataType)); } try { - JsonArray dataArray = contents.getJsonArray(dataTypeName); + JsonArray dataArray = contents.getJsonArray(dataType); if (dataArray == null) { throw new IllegalArgumentException("No array found in the contents"); } @@ -40,10 +47,10 @@ public long loadContent(JsonObject contents) throws Exception { final int count = parsed.getCount(); latestEntryCount.set(count); - LOGGER.info(String.format("Loaded %d %s", count, dataTypeName)); + LOGGER.info(String.format("Loaded %d %s", count, dataType)); return count; } catch (Exception e) { - LOGGER.error(String.format("Unable to load %s", dataTypeName)); + LOGGER.error(String.format("Unable to load %s", dataType)); throw e; } } diff --git a/src/main/java/com/uid2/operator/reader/RotatingCloudEncryptionKeyApiProvider.java b/src/main/java/com/uid2/operator/reader/RotatingCloudEncryptionKeyApiProvider.java index 4c91ea9dc..45aff8e9a 100644 --- a/src/main/java/com/uid2/operator/reader/RotatingCloudEncryptionKeyApiProvider.java +++ b/src/main/java/com/uid2/operator/reader/RotatingCloudEncryptionKeyApiProvider.java @@ -16,41 +16,19 @@ public class RotatingCloudEncryptionKeyApiProvider extends RotatingCloudEncryptionKeyProvider { private static final Logger LOGGER = LoggerFactory.getLogger(RotatingCloudEncryptionKeyApiProvider.class); - public ApiStoreReader> apiStoreReader; - public RotatingCloudEncryptionKeyApiProvider(DownloadCloudStorage fileStreamProvider, StoreScope scope) { super(fileStreamProvider, scope); - this.apiStoreReader = new ApiStoreReader<>(fileStreamProvider, scope, new CloudEncryptionKeyParser(), "cloud_encryption_keys"); - } - - @Override - public JsonObject getMetadata() throws Exception { - return apiStoreReader.getMetadata(); + this.reader = new ApiStoreReader<>(fileStreamProvider, scope, new CloudEncryptionKeyParser(), "cloud_encryption_keys"); } - @Override - public CloudPath getMetadataPath() { - return apiStoreReader.getMetadataPath(); + public RotatingCloudEncryptionKeyApiProvider(DownloadCloudStorage fileStreamProvider, StoreScope scope, ApiStoreReader> reader) { + super(fileStreamProvider, scope); + this.reader = reader; } - @Override - public long loadContent(JsonObject metadata) throws Exception { - return apiStoreReader.loadContent(metadata); - } @Override public long getVersion(JsonObject metadata) { return Instant.now().getEpochSecond(); } - - @Override - public Map getAll() { - Map keys = apiStoreReader.getSnapshot(); - return keys != null ? keys : new HashMap<>(); - } - - @Override - public void loadContent() throws Exception { - this.loadContent(this.getMetadata()); - } } diff --git a/src/test/java/com/uid2/operator/RotatingCloudEncryptionKeyApiProviderTest.java b/src/test/java/com/uid2/operator/RotatingCloudEncryptionKeyApiProviderTest.java index 04f1ad0aa..254276658 100644 --- a/src/test/java/com/uid2/operator/RotatingCloudEncryptionKeyApiProviderTest.java +++ b/src/test/java/com/uid2/operator/RotatingCloudEncryptionKeyApiProviderTest.java @@ -34,8 +34,7 @@ class RotatingCloudEncryptionKeyApiProviderTest { @BeforeEach void setUp() { MockitoAnnotations.openMocks(this); - rotatingCloudEncryptionKeyApiProvider = new RotatingCloudEncryptionKeyApiProvider(mockFileStreamProvider, mockScope); - rotatingCloudEncryptionKeyApiProvider.apiStoreReader = mockApiStoreReader; + rotatingCloudEncryptionKeyApiProvider = new RotatingCloudEncryptionKeyApiProvider(mockFileStreamProvider, mockScope, mockApiStoreReader); } @Test @@ -61,11 +60,11 @@ void testGetMetadataPath() { @Test void testLoadContentWithMetadata() throws Exception { JsonObject metadata = new JsonObject(); - when(mockApiStoreReader.loadContent(metadata)).thenReturn(1L); + when(mockApiStoreReader.loadContent(metadata, "cloud_encryption_keys")).thenReturn(1L); long version = rotatingCloudEncryptionKeyApiProvider.loadContent(metadata); assertEquals(1L, version); - verify(mockApiStoreReader).loadContent(metadata); + verify(mockApiStoreReader).loadContent(metadata, "cloud_encryption_keys"); } @Test @@ -94,10 +93,10 @@ void testGetAllWithNullSnapshot() { void testLoadContent() throws Exception { JsonObject metadata = new JsonObject().put("version", 1L); when(mockApiStoreReader.getMetadata()).thenReturn(metadata); - when(mockApiStoreReader.loadContent(metadata)).thenReturn(1L); + when(mockApiStoreReader.loadContent(metadata, "cloud_encryption_keys")).thenReturn(1L); rotatingCloudEncryptionKeyApiProvider.loadContent(); verify(mockApiStoreReader).getMetadata(); - verify(mockApiStoreReader).loadContent(metadata); + verify(mockApiStoreReader).loadContent(metadata, "cloud_encryption_keys"); } } \ No newline at end of file From 9c9b1b9f8aadcca6e77c7d54f9a0a3f2c87e638b Mon Sep 17 00:00:00 2001 From: Cody Constine Date: Fri, 13 Dec 2024 12:31:41 -0700 Subject: [PATCH 161/431] Adding new constructor to rotatingCloudEncryptionProvider --- .../reader/RotatingCloudEncryptionKeyApiProvider.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/uid2/operator/reader/RotatingCloudEncryptionKeyApiProvider.java b/src/main/java/com/uid2/operator/reader/RotatingCloudEncryptionKeyApiProvider.java index 45aff8e9a..2b4d1b10d 100644 --- a/src/main/java/com/uid2/operator/reader/RotatingCloudEncryptionKeyApiProvider.java +++ b/src/main/java/com/uid2/operator/reader/RotatingCloudEncryptionKeyApiProvider.java @@ -17,13 +17,11 @@ public class RotatingCloudEncryptionKeyApiProvider extends RotatingCloudEncrypti private static final Logger LOGGER = LoggerFactory.getLogger(RotatingCloudEncryptionKeyApiProvider.class); public RotatingCloudEncryptionKeyApiProvider(DownloadCloudStorage fileStreamProvider, StoreScope scope) { - super(fileStreamProvider, scope); - this.reader = new ApiStoreReader<>(fileStreamProvider, scope, new CloudEncryptionKeyParser(), "cloud_encryption_keys"); + super(fileStreamProvider, scope, new ApiStoreReader<>(fileStreamProvider, scope, new CloudEncryptionKeyParser(), "cloud_encryption_keys")); } public RotatingCloudEncryptionKeyApiProvider(DownloadCloudStorage fileStreamProvider, StoreScope scope, ApiStoreReader> reader) { - super(fileStreamProvider, scope); - this.reader = reader; + super(fileStreamProvider, scope, reader); } From 3452457a9bc2a86e497c892b5e4b750c3ff9c73f Mon Sep 17 00:00:00 2001 From: Cody Constine Date: Fri, 13 Dec 2024 12:40:24 -0700 Subject: [PATCH 162/431] Updating shared --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ff7f61849..a44c49d3e 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ 2.1.0 2.1.0 2.1.0 - 8.0.14-alpha-176-SNAPSHOT + 8.0.15-alpha-177-SNAPSHOT ${project.version} 21 21 From fd2c0c538887dfb81d0a322ed193fbe5717183db Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Fri, 13 Dec 2024 19:44:13 +0000 Subject: [PATCH 163/431] [CI Pipeline] Released Snapshot version: 5.42.1-alpha-144-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a44c49d3e..9f47920d4 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 6.0.0 + 5.42.1-alpha-144-SNAPSHOT UTF-8 From d0753628b4d9ef910cca50771b94ca0d118a51be Mon Sep 17 00:00:00 2001 From: Cody Constine Date: Sun, 15 Dec 2024 10:08:37 -0700 Subject: [PATCH 164/431] Adding new operator version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9f47920d4..a44c49d3e 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.42.1-alpha-144-SNAPSHOT + 6.0.0 UTF-8 From 1a8635ff88e5bb1133c125e72c97252759e381f5 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Sun, 15 Dec 2024 17:12:49 +0000 Subject: [PATCH 165/431] [CI Pipeline] Released Snapshot version: 5.42.1-alpha-145-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a44c49d3e..7bd3b1e9b 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 6.0.0 + 5.42.1-alpha-145-SNAPSHOT UTF-8 From d2e25afd2829a18ed9c1b3591e33a32b90ba357f Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Sun, 15 Dec 2024 17:30:28 +0000 Subject: [PATCH 166/431] [CI Pipeline] Released Snapshot version: 5.43.5-alpha-146-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ea2d37285..6313aff2f 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.43.4 + 5.43.5-alpha-146-SNAPSHOT UTF-8 From eb9577c58c97d3725e75c0784cc9f8602fda93c7 Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Mon, 16 Dec 2024 14:35:54 +1100 Subject: [PATCH 167/431] Added ConfigService interface and concrete class. Added required constants and config. --- conf/default-config.json | 5 +- src/main/java/com/uid2/operator/Const.java | 3 + .../uid2/operator/service/ConfigService.java | 73 +++++++++++++++++++ .../uid2/operator/service/IConfigService.java | 7 ++ 4 files changed, 86 insertions(+), 2 deletions(-) create mode 100644 src/main/java/com/uid2/operator/service/ConfigService.java create mode 100644 src/main/java/com/uid2/operator/service/IConfigService.java diff --git a/conf/default-config.json b/conf/default-config.json index 224df8906..849501309 100644 --- a/conf/default-config.json +++ b/conf/default-config.json @@ -35,6 +35,7 @@ "enclave_platform": null, "failure_shutdown_wait_hours": 120, "sharing_token_expiry_seconds": 2592000, - "operator_type": "public" - + "operator_type": "public", + "core_config_url": "http://localhost:8088/config", + "config_scan_period_ms": 300000 } diff --git a/src/main/java/com/uid2/operator/Const.java b/src/main/java/com/uid2/operator/Const.java index 4d32b9034..45ecd4c92 100644 --- a/src/main/java/com/uid2/operator/Const.java +++ b/src/main/java/com/uid2/operator/Const.java @@ -29,5 +29,8 @@ public class Config extends com.uid2.shared.Const.Config { public static final String OptOutStatusMaxRequestSize = "optout_status_max_request_size"; public static final String MaxInvalidPaths = "logging_limit_max_invalid_paths_per_interval"; public static final String MaxVersionBucketsPerSite = "logging_limit_max_version_buckets_per_site"; + + public static final String CoreConfigUrl = "core_config_url"; //TODO: update when endpoint name finalised + public static final String ConfigScanPeriodMs = "config_scan_period_ms"; } } diff --git a/src/main/java/com/uid2/operator/service/ConfigService.java b/src/main/java/com/uid2/operator/service/ConfigService.java new file mode 100644 index 000000000..47ccbf1d7 --- /dev/null +++ b/src/main/java/com/uid2/operator/service/ConfigService.java @@ -0,0 +1,73 @@ +package com.uid2.operator.service; + +import io.vertx.config.ConfigRetriever; +import io.vertx.config.ConfigRetrieverOptions; +import io.vertx.config.ConfigStoreOptions; +import io.vertx.core.Vertx; +import io.vertx.core.json.JsonObject; + +import static com.uid2.operator.Const.Config.*; + +public class ConfigService implements IConfigService { + + private static volatile ConfigService instance; + private JsonObject config; + private ConfigRetriever configRetriever; + + + private ConfigService(Vertx vertx, JsonObject bootstrapConfig) { + this.initialiseConfigRetriever(vertx, bootstrapConfig); + } + + public static ConfigService getInstance(Vertx vertx, JsonObject bootstrapConfig) { + ConfigService configService = instance; + + if (configService == null) { + synchronized (ConfigService.class) { + configService = instance; + if (configService == null) { + instance = configService = new ConfigService(vertx, bootstrapConfig); + } + } + } + + return configService; + } + + @Override + public JsonObject getConfig() { + return config; + } + + private void initialiseConfigRetriever(Vertx vertx, JsonObject bootstrapConfig) { + String configUrl = bootstrapConfig.getString(CoreConfigUrl); + + ConfigStoreOptions httpStore = new ConfigStoreOptions() + .setType("http") + .setConfig(new JsonObject() + .put("url", configUrl) + .put("method", "GET")); + + ConfigStoreOptions bootstrapStore = new ConfigStoreOptions() + .setType("json") + .setConfig(bootstrapConfig); + + ConfigRetrieverOptions retrieverOptions = new ConfigRetrieverOptions() + .setScanPeriod(bootstrapConfig.getLong(ConfigScanPeriodMs)) + .addStore(bootstrapStore) + .addStore(httpStore); + + this.configRetriever = ConfigRetriever.create(vertx, retrieverOptions); + + this.configRetriever.getConfig(ar -> { + if (ar.succeeded()) { + this.config = ar.result(); + } else { + System.err.println("Failed to load config: " + ar.cause().getMessage()); + } + }); + + this.configRetriever.listen(change -> this.config = change.getNewConfiguration()); + + } +} diff --git a/src/main/java/com/uid2/operator/service/IConfigService.java b/src/main/java/com/uid2/operator/service/IConfigService.java new file mode 100644 index 000000000..0fb863242 --- /dev/null +++ b/src/main/java/com/uid2/operator/service/IConfigService.java @@ -0,0 +1,7 @@ +package com.uid2.operator.service; + +import io.vertx.core.json.JsonObject; + +public interface IConfigService { + JsonObject getConfig(); +} From c3fafd175d7835040c72865027039285c81d757b Mon Sep 17 00:00:00 2001 From: Sophia Chen Date: Mon, 16 Dec 2024 16:24:29 +1100 Subject: [PATCH 168/431] sch-UID2-4560 added gauge for vertx service instances --- src/main/java/com/uid2/operator/Main.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index 6b88d715b..51c7bef6c 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -265,6 +265,7 @@ private ICloudStorage wrapCloudStorageForOptOut(ICloudStorage cloudStorage) { } private void run() throws Exception { + this.createServiceInstancesMetric(); Supplier operatorVerticleSupplier = () -> { UIDOperatorVerticle verticle = new UIDOperatorVerticle(config, this.clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, getKeyManager(), saltProvider, optOutStore, Clock.systemUTC(), _statsCollectorQueue, new SecureLinkValidatorService(this.serviceLinkProvider, this.serviceProvider), this.shutdownHandler::handleSaltRetrievalResponse); return verticle; @@ -467,6 +468,12 @@ public DistributionStatisticConfig configure(Meter.Id id, DistributionStatisticC .register(globalRegistry); } + private void createServiceInstancesMetric() { + Gauge.builder("uid2.operator.vertx_service_instances", () -> config.getInteger("service_instances")) + .description("gauge for number of request processing threads") + .register(Metrics.globalRegistry); + } + private Map.Entry createUidClients(Vertx vertx, String attestationUrl, String clientApiToken, Handler> responseWatcher) throws Exception { AttestationResponseHandler attestationResponseHandler = getAttestationTokenRetriever(vertx, attestationUrl, clientApiToken, responseWatcher); UidCoreClient coreClient = new UidCoreClient(clientApiToken, CloudUtils.defaultProxy, attestationResponseHandler); From c98584191592b5f79b9055070144274e10c322bb Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Mon, 16 Dec 2024 17:07:53 +1100 Subject: [PATCH 169/431] Add config to RoutingContext and implement ConfigService injection --- src/main/java/com/uid2/operator/Main.java | 5 ++++- .../com/uid2/operator/vertx/UIDOperatorVerticle.java | 9 ++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index 6b88d715b..6e25029bd 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -8,6 +8,7 @@ import com.uid2.operator.monitoring.IStatsCollectorQueue; import com.uid2.operator.monitoring.OperatorMetrics; import com.uid2.operator.monitoring.StatsCollectorVerticle; +import com.uid2.operator.service.ConfigService; import com.uid2.operator.service.SecureLinkValidatorService; import com.uid2.operator.service.ShutdownService; import com.uid2.operator.vertx.Endpoints; @@ -265,8 +266,10 @@ private ICloudStorage wrapCloudStorageForOptOut(ICloudStorage cloudStorage) { } private void run() throws Exception { + ConfigService configService = ConfigService.getInstance(vertx, config); + Supplier operatorVerticleSupplier = () -> { - UIDOperatorVerticle verticle = new UIDOperatorVerticle(config, this.clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, getKeyManager(), saltProvider, optOutStore, Clock.systemUTC(), _statsCollectorQueue, new SecureLinkValidatorService(this.serviceLinkProvider, this.serviceProvider), this.shutdownHandler::handleSaltRetrievalResponse); + UIDOperatorVerticle verticle = new UIDOperatorVerticle(configService, config, this.clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, getKeyManager(), saltProvider, optOutStore, Clock.systemUTC(), _statsCollectorQueue, new SecureLinkValidatorService(this.serviceLinkProvider, this.serviceProvider), this.shutdownHandler::handleSaltRetrievalResponse); return verticle; }; diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index c3784ae38..6ecc96dc6 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -84,6 +84,7 @@ public class UIDOperatorVerticle extends AbstractVerticle { private static final String REQUEST = "request"; private final HealthComponent healthComponent = HealthManager.instance.registerComponent("http-server"); private final Cipher aesGcm; + private final IConfigService configService; private final JsonObject config; private final boolean clientSideTokenGenerate; private final AuthMiddleware auth; @@ -135,7 +136,7 @@ public class UIDOperatorVerticle extends AbstractVerticle { private static final String ERROR_INVALID_INPUT_EMAIL_TWICE = "Only one of email or email_hash can be specified"; public final static String ORIGIN_HEADER = "Origin"; - public UIDOperatorVerticle(JsonObject config, + public UIDOperatorVerticle(IConfigService configService, JsonObject config, boolean clientSideTokenGenerate, ISiteStore siteProvider, IClientKeyProvider clientKeyProvider, @@ -154,6 +155,7 @@ public UIDOperatorVerticle(JsonObject config, } catch (NoSuchAlgorithmException | NoSuchPaddingException e) { throw new RuntimeException(e); } + this.configService = configService; this.config = config; this.clientSideTokenGenerate = clientSideTokenGenerate; this.healthComponent.setHealthStatus(false, "not started"); @@ -236,6 +238,11 @@ private Router createRoutesSetup() throws IOException { .allowedHeader("Content-Type")); router.route().handler(new StatsCollectorHandler(_statsCollectorQueue, vertx)); router.route("/static/*").handler(StaticHandler.create("static")); + router.route().handler(ctx -> { + JsonObject curConfig = configService.getConfig(); + ctx.put("config", curConfig); + ctx.next(); + }); router.route().failureHandler(new GenericFailureHandler()); final BodyHandler bodyHandler = BodyHandler.create().setHandleFileUploads(false).setBodyLimit(MAX_REQUEST_BODY_SIZE); From bdcaf8bc8960469e7d84d7dc1082c736107a90dd Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Mon, 16 Dec 2024 19:10:02 +0000 Subject: [PATCH 170/431] [CI Pipeline] Released Snapshot version: 5.43.6-alpha-147-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 6313aff2f..c11f47781 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.43.5-alpha-146-SNAPSHOT + 5.43.6-alpha-147-SNAPSHOT UTF-8 From 33763cc6700b77fb203a6de35a1796faace7b0c0 Mon Sep 17 00:00:00 2001 From: Cody Constine Date: Mon, 16 Dec 2024 12:54:46 -0700 Subject: [PATCH 171/431] Adding all the configs for private operators --- scripts/aws/conf/integ-euid-config.json | 1 + scripts/aws/conf/integ-uid2-config.json | 1 + scripts/aws/conf/prod-euid-config.json | 1 + scripts/aws/conf/prod-uid2-config.json | 1 + scripts/azure-cc/conf/integ-uid2-config.json | 1 + scripts/azure-cc/conf/prod-uid2-config.json | 1 + scripts/gcp/conf/integ-config.json | 1 + scripts/gcp/conf/prod-config.json | 1 + 8 files changed, 8 insertions(+) diff --git a/scripts/aws/conf/integ-euid-config.json b/scripts/aws/conf/integ-euid-config.json index 45d3dbe94..22f8a15e7 100644 --- a/scripts/aws/conf/integ-euid-config.json +++ b/scripts/aws/conf/integ-euid-config.json @@ -9,6 +9,7 @@ "service_links_metadata_path": "https://core.integ.euid.eu/service_links/refresh", "optout_metadata_path": "https://optout.integ.euid.eu/optout/refresh", "core_attest_url": "https://core.integ.euid.eu/attest", + "cloud_encryption_keys_metadata_path": "https://core.integ.euid.eu/cloud_encryption_keys/retrieve", "optout_api_uri": "https://optout.integ.euid.eu/optout/replicate", "optout_s3_folder": "optout/", "allow_legacy_api": false diff --git a/scripts/aws/conf/integ-uid2-config.json b/scripts/aws/conf/integ-uid2-config.json index a7272a26a..11b0d6048 100644 --- a/scripts/aws/conf/integ-uid2-config.json +++ b/scripts/aws/conf/integ-uid2-config.json @@ -8,6 +8,7 @@ "services_metadata_path": "https://core-integ.uidapi.com/services/refresh", "service_links_metadata_path": "https://core-integ.uidapi.com/service_links/refresh", "optout_metadata_path": "https://optout-integ.uidapi.com/optout/refresh", + "cloud_encryption_keys_metadata_path": "https://core-integ.uidapi.com/cloud_encryption_keys/retrieve", "core_attest_url": "https://core-integ.uidapi.com/attest", "optout_api_uri": "https://optout-integ.uidapi.com/optout/replicate", "optout_s3_folder": "uid-optout-integ/", diff --git a/scripts/aws/conf/prod-euid-config.json b/scripts/aws/conf/prod-euid-config.json index 0fbf5d69c..7f7ecdef4 100644 --- a/scripts/aws/conf/prod-euid-config.json +++ b/scripts/aws/conf/prod-euid-config.json @@ -9,6 +9,7 @@ "services_metadata_path": "https://core.prod.euid.eu/services/refresh", "service_links_metadata_path": "https://core.prod.euid.eu/service_links/refresh", "optout_metadata_path": "https://optout.prod.euid.eu/optout/refresh", + "cloud_encryption_keys_metadata_path": "https://core.prod.euid.eu/cloud_encryption_keys/retrieve", "core_attest_url": "https://core.prod.euid.eu/attest", "core_api_token": "your-api-token", "optout_s3_path_compat": false, diff --git a/scripts/aws/conf/prod-uid2-config.json b/scripts/aws/conf/prod-uid2-config.json index 5da450033..8dd9b63d5 100644 --- a/scripts/aws/conf/prod-uid2-config.json +++ b/scripts/aws/conf/prod-uid2-config.json @@ -8,6 +8,7 @@ "salts_metadata_path": "https://core-prod.uidapi.com/salt/refresh", "services_metadata_path": "https://core-prod.uidapi.com/services/refresh", "service_links_metadata_path": "https://core-prod.uidapi.com/service_links/refresh", + "cloud_encryption_keys_metadata_path": "https://core-prod.uidapi.com/cloud_encryption_keys/retrieve", "optout_metadata_path": "https://optout-prod.uidapi.com/optout/refresh", "core_attest_url": "https://core-prod.uidapi.com/attest", "core_api_token": "your-api-token", diff --git a/scripts/azure-cc/conf/integ-uid2-config.json b/scripts/azure-cc/conf/integ-uid2-config.json index 2cd4be5c3..f75f3717d 100644 --- a/scripts/azure-cc/conf/integ-uid2-config.json +++ b/scripts/azure-cc/conf/integ-uid2-config.json @@ -7,6 +7,7 @@ "salts_metadata_path": "https://core-integ.uidapi.com/salt/refresh", "services_metadata_path": "https://core-integ.uidapi.com/services/refresh", "service_links_metadata_path": "https://core-integ.uidapi.com/service_links/refresh", + "cloud_encryption_keys_metadata_path": "https://core-integ.uidapi.com/cloud_encryption_keys/retrieve", "optout_metadata_path": "https://optout-integ.uidapi.com/optout/refresh", "core_attest_url": "https://core-integ.uidapi.com/attest", "optout_api_uri": "https://optout-integ.uidapi.com/optout/replicate", diff --git a/scripts/azure-cc/conf/prod-uid2-config.json b/scripts/azure-cc/conf/prod-uid2-config.json index 02e2cde20..3703337a4 100644 --- a/scripts/azure-cc/conf/prod-uid2-config.json +++ b/scripts/azure-cc/conf/prod-uid2-config.json @@ -7,6 +7,7 @@ "salts_metadata_path": "https://core-prod.uidapi.com/salt/refresh", "services_metadata_path": "https://core-prod.uidapi.com/services/refresh", "service_links_metadata_path": "https://core-prod.uidapi.com/service_links/refresh", + "cloud_encryption_keys_metadata_path": "https://core-prod.uidapi.com/cloud_encryption_keys/retrieve", "optout_metadata_path": "https://optout-prod.uidapi.com/optout/refresh", "core_attest_url": "https://core-prod.uidapi.com/attest", "optout_api_uri": "https://optout-prod.uidapi.com/optout/replicate", diff --git a/scripts/gcp/conf/integ-config.json b/scripts/gcp/conf/integ-config.json index d3fb9e9ff..09e93dfcc 100644 --- a/scripts/gcp/conf/integ-config.json +++ b/scripts/gcp/conf/integ-config.json @@ -5,6 +5,7 @@ "salts_metadata_path": "https://core-integ.uidapi.com/salt/refresh", "core_attest_url": "https://core-integ.uidapi.com/attest", "optout_metadata_path": "https://optout-integ.uidapi.com/optout/refresh", + "cloud_encryption_keys_metadata_path": "https://core-integ.uidapi.com/cloud_encryption_keys/retrieve", "optout_api_uri": "https://optout-integ.uidapi.com/optout/replicate", "optout_s3_folder": "optout-v2/", "optout_inmem_cache": true, diff --git a/scripts/gcp/conf/prod-config.json b/scripts/gcp/conf/prod-config.json index 836349c19..32bc0085c 100644 --- a/scripts/gcp/conf/prod-config.json +++ b/scripts/gcp/conf/prod-config.json @@ -6,6 +6,7 @@ "core_attest_url": "https://core-prod.uidapi.com/attest", "optout_metadata_path": "https://optout-prod.uidapi.com/optout/refresh", "optout_api_uri": "https://optout-prod.uidapi.com/optout/replicate", + "cloud_encryption_keys_metadata_path": "https://core-prod.uidapi.com/cloud_encryption_keys/retrieve", "optout_s3_folder": "optout-v2/", "optout_inmem_cache": true, "identity_token_expires_after_seconds": 14400, From 5163db8bfb982f09eda43d59548df52e5aeac091 Mon Sep 17 00:00:00 2001 From: Cody Constine Date: Mon, 16 Dec 2024 13:09:32 -0700 Subject: [PATCH 172/431] Upping the default cloud encryption rotation interval --- conf/default-config.json | 1 + 1 file changed, 1 insertion(+) diff --git a/conf/default-config.json b/conf/default-config.json index 372f7ccb2..d4408bc34 100644 --- a/conf/default-config.json +++ b/conf/default-config.json @@ -31,6 +31,7 @@ "services_metadata_path": "services/metadata.json", "service_links_metadata_path": "service_links/metadata.json", "cloud_encryption_keys_metadata_path": "cloud_encryption_keys/metadata.json", + "cloud_encryption_keys_refresh_ms": 300000, "optout_metadata_path": null, "optout_inmem_cache": false, "enclave_platform": null, From 83fe8d7ade3118e7c968f6dd673300d599e99efa Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Tue, 17 Dec 2024 14:10:20 +1100 Subject: [PATCH 173/431] Updated ConfigService getConfig method to use ConfigRetriever getCachedConfig method rather than storing config --- src/main/java/com/uid2/operator/service/ConfigService.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/uid2/operator/service/ConfigService.java b/src/main/java/com/uid2/operator/service/ConfigService.java index 47ccbf1d7..fcdfa7598 100644 --- a/src/main/java/com/uid2/operator/service/ConfigService.java +++ b/src/main/java/com/uid2/operator/service/ConfigService.java @@ -11,7 +11,6 @@ public class ConfigService implements IConfigService { private static volatile ConfigService instance; - private JsonObject config; private ConfigRetriever configRetriever; @@ -36,7 +35,7 @@ public static ConfigService getInstance(Vertx vertx, JsonObject bootstrapConfig) @Override public JsonObject getConfig() { - return config; + return configRetriever.getCachedConfig(); } private void initialiseConfigRetriever(Vertx vertx, JsonObject bootstrapConfig) { @@ -61,13 +60,11 @@ private void initialiseConfigRetriever(Vertx vertx, JsonObject bootstrapConfig) this.configRetriever.getConfig(ar -> { if (ar.succeeded()) { - this.config = ar.result(); + System.out.println("Successfully loaded config"); } else { System.err.println("Failed to load config: " + ar.cause().getMessage()); } }); - this.configRetriever.listen(change -> this.config = change.getNewConfiguration()); - } } From 6c8aefb6a2383d39f300f8aaa4a4b4fd79f7acbc Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Tue, 17 Dec 2024 17:05:11 +1100 Subject: [PATCH 174/431] Updated UIDOperatorVerticle to use ConfigService, Mocked ConfigService in UIDOperatorVerticleTest "identity_token_expires_after_seconds", "max_bidstream_lifetime_seconds", "max_sharing_lifetime_seconds" and "sharing_token_expiry_seconds" now are retrieved from RoutingContext in handlers. --- src/main/java/com/uid2/operator/Const.java | 1 + src/main/java/com/uid2/operator/Main.java | 2 +- .../operator/vertx/UIDOperatorVerticle.java | 51 +++++++++++-------- .../operator/ExtendedUIDOperatorVerticle.java | 6 +-- .../operator/UIDOperatorVerticleTest.java | 4 +- 5 files changed, 37 insertions(+), 27 deletions(-) diff --git a/src/main/java/com/uid2/operator/Const.java b/src/main/java/com/uid2/operator/Const.java index 45ecd4c92..8326db6ce 100644 --- a/src/main/java/com/uid2/operator/Const.java +++ b/src/main/java/com/uid2/operator/Const.java @@ -32,5 +32,6 @@ public class Config extends com.uid2.shared.Const.Config { public static final String CoreConfigUrl = "core_config_url"; //TODO: update when endpoint name finalised public static final String ConfigScanPeriodMs = "config_scan_period_ms"; + public static final String Config = "config"; } } diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index 6e25029bd..7dbf64153 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -269,7 +269,7 @@ private void run() throws Exception { ConfigService configService = ConfigService.getInstance(vertx, config); Supplier operatorVerticleSupplier = () -> { - UIDOperatorVerticle verticle = new UIDOperatorVerticle(configService, config, this.clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, getKeyManager(), saltProvider, optOutStore, Clock.systemUTC(), _statsCollectorQueue, new SecureLinkValidatorService(this.serviceLinkProvider, this.serviceProvider), this.shutdownHandler::handleSaltRetrievalResponse); + UIDOperatorVerticle verticle = new UIDOperatorVerticle(configService, this.clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, getKeyManager(), saltProvider, optOutStore, Clock.systemUTC(), _statsCollectorQueue, new SecureLinkValidatorService(this.serviceLinkProvider, this.serviceProvider), this.shutdownHandler::handleSaltRetrievalResponse); return verticle; }; diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index 6ecc96dc6..cf8fffaa8 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -67,6 +67,7 @@ import java.util.function.Supplier; import java.util.stream.Collectors; +import static com.uid2.operator.Const.Config.*; import static com.uid2.operator.IdentityConst.*; import static com.uid2.operator.service.ResponseUtil.*; import static com.uid2.operator.vertx.Endpoints.*; @@ -85,7 +86,6 @@ public class UIDOperatorVerticle extends AbstractVerticle { private final HealthComponent healthComponent = HealthManager.instance.registerComponent("http-server"); private final Cipher aesGcm; private final IConfigService configService; - private final JsonObject config; private final boolean clientSideTokenGenerate; private final AuthMiddleware auth; private final ISiteStore siteProvider; @@ -117,7 +117,6 @@ public class UIDOperatorVerticle extends AbstractVerticle { public final static int MASTER_KEYSET_ID_FOR_SDKS = 9999999; //this is because SDKs have an issue where they assume keyset ids are always positive; that will be fixed. public final static long OPT_OUT_CHECK_CUTOFF_DATE = Instant.parse("2023-09-01T00:00:00.00Z").getEpochSecond(); private final Handler saltRetrievalResponseHandler; - private final int maxBidstreamLifetimeSeconds; private final int allowClockSkewSeconds; protected int maxSharingLifetimeSeconds; protected Map> siteIdToInvalidOriginsAndAppNames = new HashMap<>(); @@ -136,7 +135,7 @@ public class UIDOperatorVerticle extends AbstractVerticle { private static final String ERROR_INVALID_INPUT_EMAIL_TWICE = "Only one of email or email_hash can be specified"; public final static String ORIGIN_HEADER = "Origin"; - public UIDOperatorVerticle(IConfigService configService, JsonObject config, + public UIDOperatorVerticle(IConfigService configService, boolean clientSideTokenGenerate, ISiteStore siteProvider, IClientKeyProvider clientKeyProvider, @@ -156,7 +155,6 @@ public UIDOperatorVerticle(IConfigService configService, JsonObject config, throw new RuntimeException(e); } this.configService = configService; - this.config = config; this.clientSideTokenGenerate = clientSideTokenGenerate; this.healthComponent.setHealthStatus(false, "not started"); this.auth = new AuthMiddleware(clientKeyProvider); @@ -166,6 +164,7 @@ public UIDOperatorVerticle(IConfigService configService, JsonObject config, this.saltProvider = saltProvider; this.optOutStore = optOutStore; this.clock = clock; + JsonObject config = configService.getConfig(); this.identityScope = IdentityScope.fromString(config.getString("identity_scope", "uid2")); this.v2PayloadHandler = new V2PayloadHandler(keyManager, config.getBoolean("enable_v2_encryption", true), this.identityScope, siteProvider); this.phoneSupport = config.getBoolean("enable_phone_support", true); @@ -175,14 +174,7 @@ public UIDOperatorVerticle(IConfigService configService, JsonObject config, this._statsCollectorQueue = statsCollectorQueue; this.clientKeyProvider = clientKeyProvider; this.clientSideTokenGenerateLogInvalidHttpOrigin = config.getBoolean("client_side_token_generate_log_invalid_http_origins", false); - final Integer identityTokenExpiresAfterSeconds = config.getInteger(UIDOperatorService.IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS); - this.maxBidstreamLifetimeSeconds = config.getInteger(Const.Config.MaxBidstreamLifetimeSecondsProp, identityTokenExpiresAfterSeconds); - if (this.maxBidstreamLifetimeSeconds < identityTokenExpiresAfterSeconds) { - LOGGER.error("Max bidstream lifetime seconds ({} seconds) is less than identity token lifetime ({} seconds)", maxBidstreamLifetimeSeconds, identityTokenExpiresAfterSeconds); - throw new RuntimeException("Max bidstream lifetime seconds is less than identity token lifetime seconds"); - } this.allowClockSkewSeconds = config.getInteger(Const.Config.AllowClockSkewSecondsProp, 1800); - this.maxSharingLifetimeSeconds = config.getInteger(Const.Config.MaxSharingLifetimeProp, config.getInteger(Const.Config.SharingTokenExpiryProp)); this.saltRetrievalResponseHandler = saltRetrievalResponseHandler; this.optOutStatusApiEnabled = config.getBoolean(Const.Config.OptOutStatusApiEnabled, true); this.optOutStatusMaxRequestSize = config.getInteger(Const.Config.OptOutStatusMaxRequestSize, 5000); @@ -192,7 +184,7 @@ public UIDOperatorVerticle(IConfigService configService, JsonObject config, public void start(Promise startPromise) throws Exception { this.healthComponent.setHealthStatus(false, "still starting"); this.idService = new UIDOperatorService( - this.config, + this.configService.getConfig(), //TODO this.optOutStore, this.saltProvider, this.encoder, @@ -240,7 +232,7 @@ private Router createRoutesSetup() throws IOException { router.route("/static/*").handler(StaticHandler.create("static")); router.route().handler(ctx -> { JsonObject curConfig = configService.getConfig(); - ctx.put("config", curConfig); + ctx.put(Config, curConfig); ctx.next(); }); router.route().failureHandler(new GenericFailureHandler()); @@ -251,7 +243,7 @@ private Router createRoutesSetup() throws IOException { // Static and health check router.get(OPS_HEALTHCHECK.toString()).handler(this::handleHealthCheck); - if (this.config.getBoolean(Const.Config.AllowLegacyAPIProp, true)) { + if (this.configService.getConfig().getBoolean(Const.Config.AllowLegacyAPIProp, true)) { // V1 APIs router.get(V1_TOKEN_GENERATE.toString()).handler(auth.handleV1(this::handleTokenGenerateV1, Role.GENERATOR)); router.get(V1_TOKEN_VALIDATE.toString()).handler(this::handleTokenValidateV1); @@ -320,6 +312,9 @@ private void handleClientSideTokenGenerate(RoutingContext rc) { } } + private JsonObject getConfigFromRc(RoutingContext rc) { + return rc.get(Config); + } private Set getDomainNameListForClientSideTokenGenerate(ClientSideKeypair keypair) { Site s = siteProvider.getSite(keypair.getSiteId()); @@ -611,11 +606,14 @@ public void handleKeysRequest(RoutingContext rc) { } } - private String getSharingTokenExpirySeconds() { - return config.getString(Const.Config.SharingTokenExpiryProp); - } +// private String getSharingTokenExpirySeconds() { +// return config.getString(Const.Config.SharingTokenExpiryProp); +// } public void handleKeysSharing(RoutingContext rc) { + JsonObject config = this.getConfigFromRc(rc); + Integer maxSharingLifetimeSeconds = config.getInteger(Const.Config.MaxSharingLifetimeProp, config.getInteger(Const.Config.SharingTokenExpiryProp)); + String sharingTokenExpirySeconds = config.getString(Const.Config.SharingTokenExpiryProp); try { final ClientKey clientKey = AuthMiddleware.getAuthClient(ClientKey.class, rc); @@ -624,7 +622,7 @@ public void handleKeysSharing(RoutingContext rc) { Map keysetMap = keyManagerSnapshot.getAllKeysets(); final JsonObject resp = new JsonObject(); - addSharingHeaderFields(resp, keyManagerSnapshot, clientKey); + addSharingHeaderFields(resp, keyManagerSnapshot, clientKey, maxSharingLifetimeSeconds, sharingTokenExpirySeconds); final List accessibleKeys = getAccessibleKeys(keysetKeyStore, keyManagerSnapshot, clientKey); @@ -669,14 +667,23 @@ public void handleKeysBidstream(RoutingContext rc) { .collect(Collectors.toList()); final JsonObject resp = new JsonObject(); - addBidstreamHeaderFields(resp); + + JsonObject config = this.getConfigFromRc(rc); + Integer identityTokenExpiresAfterSeconds = config.getInteger(UIDOperatorService.IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS); + Integer maxBidstreamLifetimeSeconds = config.getInteger(Const.Config.MaxBidstreamLifetimeSecondsProp, identityTokenExpiresAfterSeconds); + if (maxBidstreamLifetimeSeconds < identityTokenExpiresAfterSeconds) { + LOGGER.error("Max bidstream lifetime seconds ({} seconds) is less than identity token lifetime ({} seconds)", maxBidstreamLifetimeSeconds, identityTokenExpiresAfterSeconds); + throw new RuntimeException("Max bidstream lifetime seconds is less than identity token lifetime seconds"); + } + + addBidstreamHeaderFields(resp, maxBidstreamLifetimeSeconds); resp.put("keys", keysJson); addSites(resp, accessibleKeys, keysetMap); ResponseUtil.SuccessV2(rc, resp); } - private void addBidstreamHeaderFields(JsonObject resp) { + private void addBidstreamHeaderFields(JsonObject resp, Integer maxBidstreamLifetimeSeconds) { resp.put("max_bidstream_lifetime_seconds", maxBidstreamLifetimeSeconds + TOKEN_LIFETIME_TOLERANCE.toSeconds()); addIdentityScopeField(resp); addAllowClockSkewSecondsField(resp); @@ -714,7 +721,7 @@ private void addSites(JsonObject resp, List keys, Map saltRetrievalResponseHandler) { - super(config, clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, keyManager, saltProvider, optOutStore, clock, statsCollectorQueue, secureLinkValidationService, saltRetrievalResponseHandler); + super(configService, clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, keyManager, saltProvider, optOutStore, clock, statsCollectorQueue, secureLinkValidationService, saltRetrievalResponseHandler); } public IUIDOperatorService getIdService() { diff --git a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java index d9a91ae01..8a2888d61 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java @@ -117,6 +117,7 @@ public class UIDOperatorVerticleTest { @Mock private Clock clock; @Mock private IStatsCollectorQueue statsCollectorQueue; @Mock private OperatorShutdownHandler shutdownHandler; + @Mock private IConfigService configService; private SimpleMeterRegistry registry; private ExtendedUIDOperatorVerticle uidOperatorVerticle; @@ -134,8 +135,9 @@ public void deployVerticle(Vertx vertx, VertxTestContext testContext, TestInfo t if(testInfo.getDisplayName().equals("cstgNoPhoneSupport(Vertx, VertxTestContext)")) { config.put("enable_phone_support", false); } + when(configService.getConfig()).thenReturn(config); - this.uidOperatorVerticle = new ExtendedUIDOperatorVerticle(config, config.getBoolean("client_side_token_generate"), siteProvider, clientKeyProvider, clientSideKeypairProvider, new KeyManager(keysetKeyStore, keysetProvider), saltProvider, optOutStore, clock, statsCollectorQueue, secureLinkValidatorService, shutdownHandler::handleSaltRetrievalResponse); + this.uidOperatorVerticle = new ExtendedUIDOperatorVerticle(configService, config.getBoolean("client_side_token_generate"), siteProvider, clientKeyProvider, clientSideKeypairProvider, new KeyManager(keysetKeyStore, keysetProvider), saltProvider, optOutStore, clock, statsCollectorQueue, secureLinkValidatorService, shutdownHandler::handleSaltRetrievalResponse); vertx.deployVerticle(uidOperatorVerticle, testContext.succeeding(id -> testContext.completeNow())); From c7a788ffcca4373a0ad4b882a56b3e077e8d4dc3 Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Wed, 18 Dec 2024 09:51:43 +1100 Subject: [PATCH 175/431] Updated UIDOperatorService to get "identity_v3" config from UIDOperatorVerticle --- .../java/com/uid2/operator/service/UIDOperatorService.java | 4 ++-- .../java/com/uid2/operator/vertx/UIDOperatorVerticle.java | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/uid2/operator/service/UIDOperatorService.java b/src/main/java/com/uid2/operator/service/UIDOperatorService.java index 6d4ff86d0..80a1a5735 100644 --- a/src/main/java/com/uid2/operator/service/UIDOperatorService.java +++ b/src/main/java/com/uid2/operator/service/UIDOperatorService.java @@ -52,7 +52,7 @@ public class UIDOperatorService implements IUIDOperatorService { private final Handler saltRetrievalResponseHandler; public UIDOperatorService(JsonObject config, IOptOutStore optOutStore, ISaltProvider saltProvider, ITokenEncoder encoder, Clock clock, - IdentityScope identityScope, Handler saltRetrievalResponseHandler) { + IdentityScope identityScope, Handler saltRetrievalResponseHandler, Boolean identityV3Enabled) { this.saltProvider = saltProvider; this.encoder = encoder; this.optOutStore = optOutStore; @@ -90,7 +90,7 @@ public UIDOperatorService(JsonObject config, IOptOutStore optOutStore, ISaltProv } this.refreshTokenVersion = TokenVersion.V3; - this.identityV3Enabled = config.getBoolean("identity_v3", false); + this.identityV3Enabled = identityV3Enabled; } @Override diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index cf8fffaa8..4cf97d9d5 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -182,6 +182,7 @@ public UIDOperatorVerticle(IConfigService configService, @Override public void start(Promise startPromise) throws Exception { + Boolean identityV3Enabled = this.configService.getConfig().getBoolean("identity_v3", false); this.healthComponent.setHealthStatus(false, "still starting"); this.idService = new UIDOperatorService( this.configService.getConfig(), //TODO @@ -190,7 +191,8 @@ public void start(Promise startPromise) throws Exception { this.encoder, this.clock, this.identityScope, - this.saltRetrievalResponseHandler + this.saltRetrievalResponseHandler, + identityV3Enabled ); final Router router = createRoutesSetup(); From cfd6a6a21b2a3fbd25d24aa2f1c4935c9096690d Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Wed, 18 Dec 2024 11:57:33 +1100 Subject: [PATCH 176/431] Added ConfigValidatorUtil, Added configValidationHandler method to ConfigService Moved validation logic from UIDOperatorVerticle and UIDOperatorService to ConfigValidatorUtil configValidationHandler is set as a config processor for configRetriever in ConfigService --- .../uid2/operator/service/ConfigService.java | 32 ++++++++++++++++- .../operator/service/ConfigValidatorUtil.java | 34 +++++++++++++++++++ .../operator/service/UIDOperatorService.java | 10 ------ .../operator/vertx/UIDOperatorVerticle.java | 5 +-- 4 files changed, 66 insertions(+), 15 deletions(-) create mode 100644 src/main/java/com/uid2/operator/service/ConfigValidatorUtil.java diff --git a/src/main/java/com/uid2/operator/service/ConfigService.java b/src/main/java/com/uid2/operator/service/ConfigService.java index fcdfa7598..4f95b8696 100644 --- a/src/main/java/com/uid2/operator/service/ConfigService.java +++ b/src/main/java/com/uid2/operator/service/ConfigService.java @@ -1,18 +1,23 @@ package com.uid2.operator.service; +import com.uid2.operator.Const; import io.vertx.config.ConfigRetriever; import io.vertx.config.ConfigRetrieverOptions; import io.vertx.config.ConfigStoreOptions; import io.vertx.core.Vertx; import io.vertx.core.json.JsonObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import static com.uid2.operator.Const.Config.*; +import static com.uid2.operator.service.ConfigValidatorUtil.*; +import static com.uid2.operator.service.UIDOperatorService.*; public class ConfigService implements IConfigService { private static volatile ConfigService instance; private ConfigRetriever configRetriever; - + private static final Logger logger = LoggerFactory.getLogger(ConfigService.class); private ConfigService(Vertx vertx, JsonObject bootstrapConfig) { this.initialiseConfigRetriever(vertx, bootstrapConfig); @@ -58,6 +63,8 @@ private void initialiseConfigRetriever(Vertx vertx, JsonObject bootstrapConfig) this.configRetriever = ConfigRetriever.create(vertx, retrieverOptions); + this.configRetriever.setConfigurationProcessor(this::configValidationHandler); + this.configRetriever.getConfig(ar -> { if (ar.succeeded()) { System.out.println("Successfully loaded config"); @@ -67,4 +74,27 @@ private void initialiseConfigRetriever(Vertx vertx, JsonObject bootstrapConfig) }); } + + private JsonObject configValidationHandler(JsonObject config) { + boolean isValid = true; + Integer identityExpiresAfter = config.getInteger(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS); + Integer refreshExpiresAfter = config.getInteger(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS); + Integer refreshIdentityAfter = config.getInteger(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS); + Integer maxBidstreamLifetimeSeconds = config.getInteger(Const.Config.MaxBidstreamLifetimeSecondsProp); + + isValid &= validateIdentityRefreshTokens(identityExpiresAfter, refreshExpiresAfter, refreshIdentityAfter); + + isValid &= validateBidstreamLifetime(maxBidstreamLifetimeSeconds, identityExpiresAfter); + + if (!isValid) { + logger.error("Failed to update config"); + JsonObject lastConfig = this.getConfig(); + if (lastConfig == null || lastConfig.isEmpty()) { + throw new RuntimeException("Invalid config retrieved and no previous config to revert to"); + } + return lastConfig; + } + + return config; + } } diff --git a/src/main/java/com/uid2/operator/service/ConfigValidatorUtil.java b/src/main/java/com/uid2/operator/service/ConfigValidatorUtil.java new file mode 100644 index 000000000..4f1c0999c --- /dev/null +++ b/src/main/java/com/uid2/operator/service/ConfigValidatorUtil.java @@ -0,0 +1,34 @@ +package com.uid2.operator.service; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static com.uid2.operator.service.UIDOperatorService.*; + +public class ConfigValidatorUtil { + private static final Logger logger = LoggerFactory.getLogger(ConfigValidatorUtil.class); + + public static Boolean validateIdentityRefreshTokens(Integer identityExpiresAfter, Integer refreshExpiresAfter, Integer refreshIdentityAfter) { + boolean isValid = true; + if (identityExpiresAfter > refreshExpiresAfter) { + logger.error(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS + " must be >= " + IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS); + isValid = false; + } + if (refreshIdentityAfter > identityExpiresAfter) { + logger.error(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS + " must be >= " + REFRESH_IDENTITY_TOKEN_AFTER_SECONDS); + isValid = false; + } + if (refreshIdentityAfter > refreshExpiresAfter) { + logger.error(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS + " must be >= " + REFRESH_IDENTITY_TOKEN_AFTER_SECONDS); + } + return isValid; + } + + public static Boolean validateBidstreamLifetime(Integer maxBidstreamLifetimeSeconds, Integer identityTokenExpiresAfterSeconds) { + if (maxBidstreamLifetimeSeconds < identityTokenExpiresAfterSeconds) { + logger.error("Max bidstream lifetime seconds ({} seconds) is less than identity token lifetime ({} seconds)", maxBidstreamLifetimeSeconds, identityTokenExpiresAfterSeconds); + return false; + } + return true; + } +} diff --git a/src/main/java/com/uid2/operator/service/UIDOperatorService.java b/src/main/java/com/uid2/operator/service/UIDOperatorService.java index 80a1a5735..846e4a475 100644 --- a/src/main/java/com/uid2/operator/service/UIDOperatorService.java +++ b/src/main/java/com/uid2/operator/service/UIDOperatorService.java @@ -79,16 +79,6 @@ public UIDOperatorService(JsonObject config, IOptOutStore optOutStore, ISaltProv this.refreshExpiresAfter = Duration.ofSeconds(config.getInteger(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS)); this.refreshIdentityAfter = Duration.ofSeconds(config.getInteger(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS)); - if (this.identityExpiresAfter.compareTo(this.refreshExpiresAfter) > 0) { - throw new IllegalStateException(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS + " must be >= " + IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS); - } - if (this.refreshIdentityAfter.compareTo(this.identityExpiresAfter) > 0) { - throw new IllegalStateException(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS + " must be >= " + REFRESH_IDENTITY_TOKEN_AFTER_SECONDS); - } - if (this.refreshIdentityAfter.compareTo(this.refreshExpiresAfter) > 0) { - throw new IllegalStateException(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS + " must be >= " + REFRESH_IDENTITY_TOKEN_AFTER_SECONDS); - } - this.refreshTokenVersion = TokenVersion.V3; this.identityV3Enabled = identityV3Enabled; } diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index 4cf97d9d5..ed67f71a7 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -673,10 +673,7 @@ public void handleKeysBidstream(RoutingContext rc) { JsonObject config = this.getConfigFromRc(rc); Integer identityTokenExpiresAfterSeconds = config.getInteger(UIDOperatorService.IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS); Integer maxBidstreamLifetimeSeconds = config.getInteger(Const.Config.MaxBidstreamLifetimeSecondsProp, identityTokenExpiresAfterSeconds); - if (maxBidstreamLifetimeSeconds < identityTokenExpiresAfterSeconds) { - LOGGER.error("Max bidstream lifetime seconds ({} seconds) is less than identity token lifetime ({} seconds)", maxBidstreamLifetimeSeconds, identityTokenExpiresAfterSeconds); - throw new RuntimeException("Max bidstream lifetime seconds is less than identity token lifetime seconds"); - } + addBidstreamHeaderFields(resp, maxBidstreamLifetimeSeconds); resp.put("keys", keysJson); From 975f6ab89e6813bc446226b22efe84056ab582d4 Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Wed, 18 Dec 2024 14:54:42 +1100 Subject: [PATCH 177/431] Updated UIDOperatorService to take config as method parameters, Update UIDOperatorServiceTest --- src/main/java/com/uid2/operator/Const.java | 1 + .../operator/service/IUIDOperatorService.java | 6 +- .../operator/service/UIDOperatorService.java | 33 +-- .../operator/vertx/UIDOperatorVerticle.java | 82 ++++++-- .../uid2/operator/UIDOperatorServiceTest.java | 189 ++++++++++++++---- .../operator/benchmark/BenchmarkCommon.java | 29 ++- .../benchmark/TokenEndecBenchmark.java | 28 ++- 7 files changed, 272 insertions(+), 96 deletions(-) diff --git a/src/main/java/com/uid2/operator/Const.java b/src/main/java/com/uid2/operator/Const.java index 8326db6ce..084e8f3bb 100644 --- a/src/main/java/com/uid2/operator/Const.java +++ b/src/main/java/com/uid2/operator/Const.java @@ -33,5 +33,6 @@ public class Config extends com.uid2.shared.Const.Config { public static final String CoreConfigUrl = "core_config_url"; //TODO: update when endpoint name finalised public static final String ConfigScanPeriodMs = "config_scan_period_ms"; public static final String Config = "config"; + public static final String identityV3 = "identity_v3"; } } diff --git a/src/main/java/com/uid2/operator/service/IUIDOperatorService.java b/src/main/java/com/uid2/operator/service/IUIDOperatorService.java index c64e499fe..567f17eb4 100644 --- a/src/main/java/com/uid2/operator/service/IUIDOperatorService.java +++ b/src/main/java/com/uid2/operator/service/IUIDOperatorService.java @@ -11,9 +11,9 @@ public interface IUIDOperatorService { - IdentityTokens generateIdentity(IdentityRequest request); + IdentityTokens generateIdentity(IdentityRequest request, Duration refreshIdentityAfter, Duration refreshExpiresAfter, Duration identityExpiresAfter); - RefreshResponse refreshIdentity(RefreshToken refreshToken); + RefreshResponse refreshIdentity(RefreshToken token, Duration refreshIdentityAfter, Duration refreshExpiresAfter, Duration identityExpiresAfter); MappedIdentity mapIdentity(MapRequest request); @@ -27,6 +27,4 @@ public interface IUIDOperatorService { boolean advertisingTokenMatches(String advertisingToken, UserIdentity userIdentity, Instant asOf); Instant getLatestOptoutEntry(UserIdentity userIdentity, Instant asOf); - - Duration getIdentityExpiryDuration(); } diff --git a/src/main/java/com/uid2/operator/service/UIDOperatorService.java b/src/main/java/com/uid2/operator/service/UIDOperatorService.java index 846e4a475..2c20782fe 100644 --- a/src/main/java/com/uid2/operator/service/UIDOperatorService.java +++ b/src/main/java/com/uid2/operator/service/UIDOperatorService.java @@ -9,7 +9,6 @@ import io.vertx.core.AsyncResult; import io.vertx.core.Future; import io.vertx.core.Handler; -import io.vertx.core.json.JsonObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,9 +40,6 @@ public class UIDOperatorService implements IUIDOperatorService { private final UserIdentity testValidateIdentityForPhone; private final UserIdentity testRefreshOptOutIdentityForEmail; private final UserIdentity testRefreshOptOutIdentityForPhone; - private final Duration identityExpiresAfter; - private final Duration refreshExpiresAfter; - private final Duration refreshIdentityAfter; private final OperatorIdentity operatorIdentity; private final TokenVersion refreshTokenVersion; @@ -51,7 +47,7 @@ public class UIDOperatorService implements IUIDOperatorService { private final Handler saltRetrievalResponseHandler; - public UIDOperatorService(JsonObject config, IOptOutStore optOutStore, ISaltProvider saltProvider, ITokenEncoder encoder, Clock clock, + public UIDOperatorService(IOptOutStore optOutStore, ISaltProvider saltProvider, ITokenEncoder encoder, Clock clock, IdentityScope identityScope, Handler saltRetrievalResponseHandler, Boolean identityV3Enabled) { this.saltProvider = saltProvider; this.encoder = encoder; @@ -75,16 +71,12 @@ public UIDOperatorService(JsonObject config, IOptOutStore optOutStore, ISaltProv this.operatorIdentity = new OperatorIdentity(0, OperatorType.Service, 0, 0); - this.identityExpiresAfter = Duration.ofSeconds(config.getInteger(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); - this.refreshExpiresAfter = Duration.ofSeconds(config.getInteger(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS)); - this.refreshIdentityAfter = Duration.ofSeconds(config.getInteger(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS)); - this.refreshTokenVersion = TokenVersion.V3; this.identityV3Enabled = identityV3Enabled; } @Override - public IdentityTokens generateIdentity(IdentityRequest request) { + public IdentityTokens generateIdentity(IdentityRequest request, Duration refreshIdentityAfter, Duration refreshExpiresAfter, Duration identityExpiresAfter) { final Instant now = EncodingUtils.NowUTCMillis(this.clock); final byte[] firstLevelHash = getFirstLevelHash(request.userIdentity.id, now); final UserIdentity firstLevelHashIdentity = new UserIdentity( @@ -94,12 +86,12 @@ public IdentityTokens generateIdentity(IdentityRequest request) { if (request.shouldCheckOptOut() && getGlobalOptOutResult(firstLevelHashIdentity, false).isOptedOut()) { return IdentityTokens.LogoutToken; } else { - return generateIdentity(request.publisherIdentity, firstLevelHashIdentity); + return this.generateIdentity(request.publisherIdentity, firstLevelHashIdentity, refreshIdentityAfter, refreshExpiresAfter, identityExpiresAfter); } } @Override - public RefreshResponse refreshIdentity(RefreshToken token) { + public RefreshResponse refreshIdentity(RefreshToken token, Duration refreshIdentityAfter, Duration refreshExpiresAfter, Duration identityExpiresAfter) { // should not be possible as different scopes should be using different keys, but just in case if (token.userIdentity.identityScope != this.identityScope) { return RefreshResponse.Invalid; @@ -125,7 +117,7 @@ public RefreshResponse refreshIdentity(RefreshToken token) { final Duration durationSinceLastRefresh = Duration.between(token.createdAt, now); if (!optedOut) { - IdentityTokens identityTokens = this.generateIdentity(token.publisherIdentity, token.userIdentity); + IdentityTokens identityTokens = this.generateIdentity(token.publisherIdentity, token.userIdentity, refreshIdentityAfter, refreshExpiresAfter, identityExpiresAfter); return RefreshResponse.createRefreshedResponse(identityTokens, durationSinceLastRefresh, isCstg); } else { @@ -198,11 +190,6 @@ public Instant getLatestOptoutEntry(UserIdentity userIdentity, Instant asOf) { return this.optOutStore.getLatestEntry(firstLevelHashIdentity); } - @Override - public Duration getIdentityExpiryDuration() { - return this.identityExpiresAfter; - } - private UserIdentity getFirstLevelHashIdentity(UserIdentity userIdentity, Instant asOf) { return getFirstLevelHashIdentity(userIdentity.identityScope, userIdentity.identityType, userIdentity.id, asOf); } @@ -226,7 +213,7 @@ private MappedIdentity getAdvertisingId(UserIdentity firstLevelHashIdentity, Ins rotatingSalt.getHashedId()); } - private IdentityTokens generateIdentity(PublisherIdentity publisherIdentity, UserIdentity firstLevelHashIdentity) { + private IdentityTokens generateIdentity(PublisherIdentity publisherIdentity, UserIdentity firstLevelHashIdentity, Duration refreshIdentityAfter, Duration refreshExpiresAfter, Duration identityExpiresAfter) { final Instant nowUtc = EncodingUtils.NowUTCMillis(this.clock); final MappedIdentity mappedIdentity = getAdvertisingId(firstLevelHashIdentity, nowUtc); @@ -234,14 +221,14 @@ private IdentityTokens generateIdentity(PublisherIdentity publisherIdentity, Use mappedIdentity.advertisingId, firstLevelHashIdentity.privacyBits, firstLevelHashIdentity.establishedAt, nowUtc); return this.encoder.encode( - this.createAdvertisingToken(publisherIdentity, advertisingIdentity, nowUtc), - this.createRefreshToken(publisherIdentity, firstLevelHashIdentity, nowUtc), + this.createAdvertisingToken(publisherIdentity, advertisingIdentity, nowUtc, identityExpiresAfter), + this.createRefreshToken(publisherIdentity, firstLevelHashIdentity, nowUtc, refreshExpiresAfter), nowUtc.plusMillis(refreshIdentityAfter.toMillis()), nowUtc ); } - private RefreshToken createRefreshToken(PublisherIdentity publisherIdentity, UserIdentity userIdentity, Instant now) { + private RefreshToken createRefreshToken(PublisherIdentity publisherIdentity, UserIdentity userIdentity, Instant now, Duration refreshExpiresAfter) { return new RefreshToken( this.refreshTokenVersion, now, @@ -251,7 +238,7 @@ private RefreshToken createRefreshToken(PublisherIdentity publisherIdentity, Use userIdentity); } - private AdvertisingToken createAdvertisingToken(PublisherIdentity publisherIdentity, UserIdentity userIdentity, Instant now) { + private AdvertisingToken createAdvertisingToken(PublisherIdentity publisherIdentity, UserIdentity userIdentity, Instant now, Duration identityExpiresAfter) { return new AdvertisingToken(TokenVersion.V4, now, now.plusMillis(identityExpiresAfter.toMillis()), this.operatorIdentity, publisherIdentity, userIdentity); } diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index ed67f71a7..2c2c275ff 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -182,10 +182,9 @@ public UIDOperatorVerticle(IConfigService configService, @Override public void start(Promise startPromise) throws Exception { - Boolean identityV3Enabled = this.configService.getConfig().getBoolean("identity_v3", false); + Boolean identityV3Enabled = this.configService.getConfig().getBoolean(identityV3, false); this.healthComponent.setHealthStatus(false, "still starting"); this.idService = new UIDOperatorService( - this.configService.getConfig(), //TODO this.optOutStore, this.saltProvider, this.encoder, @@ -337,6 +336,13 @@ private Set getAppNames(ClientSideKeypair keypair) { private void handleClientSideTokenGenerateImpl(RoutingContext rc) throws NoSuchAlgorithmException, InvalidKeyException { final JsonObject body; + + JsonObject config = this.getConfigFromRc(rc); + + Duration refreshIdentityAfter = Duration.ofSeconds(config.getInteger(UIDOperatorService.REFRESH_IDENTITY_TOKEN_AFTER_SECONDS)); + Duration refreshExpiresAfter = Duration.ofSeconds(config.getInteger(UIDOperatorService.REFRESH_TOKEN_EXPIRES_AFTER_SECONDS)); + Duration identityExpiresAfter = Duration.ofSeconds(config.getInteger(UIDOperatorService.IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); + TokenResponseStatsCollector.PlatformType platformType = TokenResponseStatsCollector.PlatformType.Other; try { body = rc.body().asJsonObject(); @@ -474,7 +480,10 @@ else if(emailHash != null) { new IdentityRequest( new PublisherIdentity(clientSideKeypair.getSiteId(), 0, 0), input.toUserIdentity(this.identityScope, privacyBits.getAsInt(), Instant.now()), - OptoutCheckPolicy.RespectOptOut)); + OptoutCheckPolicy.RespectOptOut), + refreshIdentityAfter, + refreshExpiresAfter, + identityExpiresAfter); } catch (KeyManager.NoActiveKeyException e){ SendServerErrorResponseAndRecordStats(rc, "No active encryption key available", clientSideKeypair.getSiteId(), TokenResponseStatsCollector.Endpoint.ClientSideTokenGenerateV2, TokenResponseStatsCollector.ResponseStatus.NoActiveKey, siteProvider, e, platformType); return; @@ -825,6 +834,10 @@ private void handleTokenRefreshV1(RoutingContext rc) { } } + JsonObject config = this.getConfigFromRc(rc); + + Duration identityExpiresAfter = Duration.ofSeconds(config.getInteger(UIDOperatorService.IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); + try { final RefreshResponse r = this.refreshIdentity(rc, refreshToken); siteId = rc.get(Const.RoutingContextData.SiteId); @@ -843,7 +856,7 @@ private void handleTokenRefreshV1(RoutingContext rc) { } } else { ResponseUtil.Success(rc, toJsonV1(r.getTokens())); - this.recordRefreshDurationStats(siteId, getApiContact(rc), r.getDurationSinceLastRefresh(), rc.request().headers().contains(ORIGIN_HEADER)); + this.recordRefreshDurationStats(siteId, getApiContact(rc), r.getDurationSinceLastRefresh(), rc.request().headers().contains(ORIGIN_HEADER), identityExpiresAfter); } TokenResponseStatsCollector.recordRefresh(siteProvider, siteId, TokenResponseStatsCollector.Endpoint.RefreshV1, r, platformType); @@ -855,6 +868,9 @@ private void handleTokenRefreshV1(RoutingContext rc) { private void handleTokenRefreshV2(RoutingContext rc) { Integer siteId = null; TokenResponseStatsCollector.PlatformType platformType = TokenResponseStatsCollector.PlatformType.Other; + + JsonObject config = this.getConfigFromRc(rc); + Duration identityExpiresAfter = Duration.ofSeconds(config.getInteger(UIDOperatorService.IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); try { platformType = getPlatformType(rc); String tokenStr = (String) rc.data().get("request"); @@ -877,7 +893,7 @@ private void handleTokenRefreshV2(RoutingContext rc) { } } else { ResponseUtil.SuccessV2(rc, toJsonV1(r.getTokens())); - this.recordRefreshDurationStats(siteId, getApiContact(rc), r.getDurationSinceLastRefresh(), rc.request().headers().contains(ORIGIN_HEADER)); + this.recordRefreshDurationStats(siteId, getApiContact(rc), r.getDurationSinceLastRefresh(), rc.request().headers().contains(ORIGIN_HEADER), identityExpiresAfter); } TokenResponseStatsCollector.recordRefresh(siteProvider, siteId, TokenResponseStatsCollector.Endpoint.RefreshV2, r, platformType); } catch (Exception e) { @@ -948,6 +964,12 @@ private void handleTokenValidateV2(RoutingContext rc) { private void handleTokenGenerateV1(RoutingContext rc) { final int siteId = AuthMiddleware.getAuthClient(rc).getSiteId(); TokenResponseStatsCollector.PlatformType platformType = TokenResponseStatsCollector.PlatformType.Other; + + JsonObject config = this.getConfigFromRc(rc); + Duration refreshIdentityAfter = Duration.ofSeconds(config.getInteger(UIDOperatorService.REFRESH_IDENTITY_TOKEN_AFTER_SECONDS)); + Duration refreshExpiresAfter = Duration.ofSeconds(config.getInteger(UIDOperatorService.REFRESH_TOKEN_EXPIRES_AFTER_SECONDS)); + Duration identityExpiresAfter = Duration.ofSeconds(config.getInteger(UIDOperatorService.IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); + try { final InputUtil.InputVal input = this.phoneSupport ? this.getTokenInputV1(rc) : this.getTokenInput(rc); platformType = getPlatformType(rc); @@ -956,7 +978,10 @@ private void handleTokenGenerateV1(RoutingContext rc) { new IdentityRequest( new PublisherIdentity(siteId, 0, 0), input.toUserIdentity(this.identityScope, 1, Instant.now()), - OptoutCheckPolicy.defaultPolicy())); + OptoutCheckPolicy.defaultPolicy()), + refreshIdentityAfter, + refreshExpiresAfter, + identityExpiresAfter); ResponseUtil.Success(rc, toJsonV1(t)); recordTokenResponseStats(siteId, TokenResponseStatsCollector.Endpoint.GenerateV1, TokenResponseStatsCollector.ResponseStatus.Success, siteProvider, t.getAdvertisingTokenVersion(), platformType); @@ -969,6 +994,12 @@ private void handleTokenGenerateV1(RoutingContext rc) { private void handleTokenGenerateV2(RoutingContext rc) { final Integer siteId = AuthMiddleware.getAuthClient(rc).getSiteId(); TokenResponseStatsCollector.PlatformType platformType = TokenResponseStatsCollector.PlatformType.Other; + + JsonObject config = this.getConfigFromRc(rc); + Duration refreshIdentityAfter = Duration.ofSeconds(config.getInteger(UIDOperatorService.REFRESH_IDENTITY_TOKEN_AFTER_SECONDS)); + Duration refreshExpiresAfter = Duration.ofSeconds(config.getInteger(UIDOperatorService.REFRESH_TOKEN_EXPIRES_AFTER_SECONDS)); + Duration identityExpiresAfter = Duration.ofSeconds(config.getInteger(UIDOperatorService.IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); + try { JsonObject req = (JsonObject) rc.data().get("request"); platformType = getPlatformType(rc); @@ -1009,7 +1040,10 @@ private void handleTokenGenerateV2(RoutingContext rc) { new IdentityRequest( new PublisherIdentity(siteId, 0, 0), input.toUserIdentity(this.identityScope, 1, Instant.now()), - OptoutCheckPolicy.respectOptOut())); + OptoutCheckPolicy.respectOptOut()), + refreshIdentityAfter, + refreshExpiresAfter, + identityExpiresAfter); if (t.isEmptyToken()) { if (optoutCheckPolicy.getItem1() == OptoutCheckPolicy.DoNotRespect) { // only legacy can use this policy @@ -1025,7 +1059,10 @@ private void handleTokenGenerateV2(RoutingContext rc) { new IdentityRequest( new PublisherIdentity(siteId, 0, 0), optOutTokenInput.toUserIdentity(this.identityScope, pb.getAsInt(), Instant.now()), - OptoutCheckPolicy.DoNotRespect)); + OptoutCheckPolicy.DoNotRespect), + refreshIdentityAfter, + refreshExpiresAfter, + identityExpiresAfter); ResponseUtil.SuccessV2(rc, toJsonV1(optOutTokens)); recordTokenResponseStats(siteId, TokenResponseStatsCollector.Endpoint.GenerateV2, TokenResponseStatsCollector.ResponseStatus.Success, siteProvider, optOutTokens.getAdvertisingTokenVersion(), platformType); @@ -1050,6 +1087,13 @@ private void handleTokenGenerateV2(RoutingContext rc) { private void handleTokenGenerate(RoutingContext rc) { final InputUtil.InputVal input = this.getTokenInput(rc); Integer siteId = null; + + JsonObject config = this.getConfigFromRc(rc); + Duration refreshIdentityAfter = Duration.ofSeconds(config.getInteger(UIDOperatorService.REFRESH_IDENTITY_TOKEN_AFTER_SECONDS)); + Duration refreshExpiresAfter = Duration.ofSeconds(config.getInteger(UIDOperatorService.REFRESH_TOKEN_EXPIRES_AFTER_SECONDS)); + Duration identityExpiresAfter = Duration.ofSeconds(config.getInteger(UIDOperatorService.IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); + + if (input == null) { SendClientErrorResponseAndRecordStats(ResponseStatus.ClientError, 400, rc, ERROR_INVALID_INPUT_EMAIL_MISSING, siteId, TokenResponseStatsCollector.Endpoint.GenerateV0, TokenResponseStatsCollector.ResponseStatus.BadPayload, siteProvider, TokenResponseStatsCollector.PlatformType.Other); return; @@ -1065,7 +1109,10 @@ else if (!input.isValid()) { new IdentityRequest( new PublisherIdentity(siteId, 0, 0), input.toUserIdentity(this.identityScope, 1, Instant.now()), - OptoutCheckPolicy.defaultPolicy())); + OptoutCheckPolicy.defaultPolicy()), + refreshIdentityAfter, + refreshExpiresAfter, + identityExpiresAfter); recordTokenResponseStats(siteId, TokenResponseStatsCollector.Endpoint.GenerateV0, TokenResponseStatsCollector.ResponseStatus.Success, siteProvider, t.getAdvertisingTokenVersion(), TokenResponseStatsCollector.PlatformType.Other); sendJsonResponse(rc, toJson(t)); @@ -1083,6 +1130,10 @@ private void handleTokenRefresh(RoutingContext rc) { return; } + JsonObject config = this.getConfigFromRc(rc); + + Duration identityExpiresAfter = Duration.ofSeconds(config.getInteger(UIDOperatorService.IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); + try { final RefreshResponse r = this.refreshIdentity(rc, tokenList.get(0)); @@ -1090,7 +1141,7 @@ private void handleTokenRefresh(RoutingContext rc) { siteId = rc.get(Const.RoutingContextData.SiteId); if (r.isRefreshed()) { - this.recordRefreshDurationStats(siteId, getApiContact(rc), r.getDurationSinceLastRefresh(), rc.request().headers().contains(ORIGIN_HEADER)); + this.recordRefreshDurationStats(siteId, getApiContact(rc), r.getDurationSinceLastRefresh(), rc.request().headers().contains(ORIGIN_HEADER), identityExpiresAfter); } TokenResponseStatsCollector.recordRefresh(siteProvider, siteId, TokenResponseStatsCollector.Endpoint.RefreshV0, r, TokenResponseStatsCollector.PlatformType.Other); } catch (Exception e) { @@ -1797,7 +1848,12 @@ private RefreshResponse refreshIdentity(RoutingContext rc, String tokenStr) { } recordRefreshTokenVersionCount(String.valueOf(rc.data().get(Const.RoutingContextData.SiteId)), this.getRefreshTokenVersion(tokenStr)); - return this.idService.refreshIdentity(refreshToken); + JsonObject config = this.getConfigFromRc(rc); + Duration refreshIdentityAfter = Duration.ofSeconds(config.getInteger(UIDOperatorService.REFRESH_IDENTITY_TOKEN_AFTER_SECONDS)); + Duration refreshExpiresAfter = Duration.ofSeconds(config.getInteger(UIDOperatorService.REFRESH_TOKEN_EXPIRES_AFTER_SECONDS)); + Duration identityExpiresAfter = Duration.ofSeconds(config.getInteger(UIDOperatorService.IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); + + return this.idService.refreshIdentity(refreshToken, refreshIdentityAfter, refreshExpiresAfter, identityExpiresAfter); } public static String getSiteName(ISiteStore siteStore, Integer siteId) { @@ -1823,7 +1879,7 @@ private TokenResponseStatsCollector.PlatformType getPlatformType(RoutingContext return origin != null ? TokenResponseStatsCollector.PlatformType.HasOriginHeader : TokenResponseStatsCollector.PlatformType.Other; } - private void recordRefreshDurationStats(Integer siteId, String apiContact, Duration durationSinceLastRefresh, boolean hasOriginHeader) { + private void recordRefreshDurationStats(Integer siteId, String apiContact, Duration durationSinceLastRefresh, boolean hasOriginHeader, Duration identityExpiresAfter) { DistributionSummary ds = _refreshDurationMetricSummaries.computeIfAbsent(new Tuple.Tuple2<>(apiContact, hasOriginHeader), k -> DistributionSummary .builder("uid2.token_refresh_duration_seconds") @@ -1836,7 +1892,7 @@ private void recordRefreshDurationStats(Integer siteId, String apiContact, Durat ); ds.record(durationSinceLastRefresh.getSeconds()); - boolean isExpired = durationSinceLastRefresh.compareTo(this.idService.getIdentityExpiryDuration()) > 0; + boolean isExpired = durationSinceLastRefresh.compareTo(identityExpiresAfter) > 0; Counter c = _advertisingTokenExpiryStatus.computeIfAbsent(new Tuple.Tuple3<>(String.valueOf(siteId), hasOriginHeader, isExpired), k -> Counter .builder("uid2.advertising_token_expired_on_refresh") diff --git a/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java b/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java index 37eeef36f..a818d29df 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java @@ -21,6 +21,8 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; + +import static com.uid2.operator.Const.Config.identityV3; import static org.junit.jupiter.api.Assertions.*; import org.junit.jupiter.params.ParameterizedTest; @@ -31,6 +33,7 @@ import java.nio.charset.StandardCharsets; import java.security.Security; import java.time.Clock; +import java.time.Duration; import java.time.Instant; import java.time.temporal.ChronoUnit; @@ -54,8 +57,8 @@ public class UIDOperatorServiceTest { final int REFRESH_IDENTITY_TOKEN_AFTER_SECONDS = 300; class ExtendedUIDOperatorService extends UIDOperatorService { - public ExtendedUIDOperatorService(JsonObject config, IOptOutStore optOutStore, ISaltProvider saltProvider, ITokenEncoder encoder, Clock clock, IdentityScope identityScope, Handler saltRetrievalResponseHandler) { - super(config, optOutStore, saltProvider, encoder, clock, identityScope, saltRetrievalResponseHandler); + public ExtendedUIDOperatorService(IOptOutStore optOutStore, ISaltProvider saltProvider, ITokenEncoder encoder, Clock clock, IdentityScope identityScope, Handler saltRetrievalResponseHandler, Boolean identityV3Enabled) { + super(optOutStore, saltProvider, encoder, clock, identityScope, saltRetrievalResponseHandler, identityV3Enabled); } } @@ -88,32 +91,32 @@ void setup() throws Exception { uid2Config.put(UIDOperatorService.IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS); uid2Config.put(UIDOperatorService.REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, REFRESH_TOKEN_EXPIRES_AFTER_SECONDS); uid2Config.put(UIDOperatorService.REFRESH_IDENTITY_TOKEN_AFTER_SECONDS, REFRESH_IDENTITY_TOKEN_AFTER_SECONDS); - uid2Config.put("identity_v3", false); + uid2Config.put(identityV3, false); uid2Service = new ExtendedUIDOperatorService( - uid2Config, optOutStore, saltProvider, tokenEncoder, this.clock, IdentityScope.UID2, - this.shutdownHandler::handleSaltRetrievalResponse + this.shutdownHandler::handleSaltRetrievalResponse, + uid2Config.getBoolean(identityV3) ); euidConfig = new JsonObject(); euidConfig.put(UIDOperatorService.IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS); euidConfig.put(UIDOperatorService.REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, REFRESH_TOKEN_EXPIRES_AFTER_SECONDS); euidConfig.put(UIDOperatorService.REFRESH_IDENTITY_TOKEN_AFTER_SECONDS, REFRESH_IDENTITY_TOKEN_AFTER_SECONDS); - euidConfig.put("identity_v3", true); + euidConfig.put(identityV3, true); euidService = new ExtendedUIDOperatorService( - euidConfig, optOutStore, saltProvider, tokenEncoder, this.clock, IdentityScope.EUID, - this.shutdownHandler::handleSaltRetrievalResponse + this.shutdownHandler::handleSaltRetrievalResponse, + euidConfig.getBoolean(identityV3) ); } @@ -157,7 +160,11 @@ public void testGenerateAndRefresh(int siteId, TokenVersion tokenVersion) { createUserIdentity("test-email-hash", IdentityScope.UID2, IdentityType.Email), OptoutCheckPolicy.DoNotRespect ); - final IdentityTokens tokens = uid2Service.generateIdentity(identityRequest); + final IdentityTokens tokens = uid2Service.generateIdentity( + identityRequest, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); assertNotNull(tokens); @@ -176,7 +183,11 @@ public void testGenerateAndRefresh(int siteId, TokenVersion tokenVersion) { setNow(Instant.now().plusSeconds(200)); reset(shutdownHandler); - final RefreshResponse refreshResponse = uid2Service.refreshIdentity(refreshToken); + final RefreshResponse refreshResponse = uid2Service.refreshIdentity( + refreshToken, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); assertNotNull(refreshResponse); @@ -207,14 +218,22 @@ public void testTestOptOutKey_DoNotRespectOptout() { inputVal.toUserIdentity(IdentityScope.UID2, 0, this.now), OptoutCheckPolicy.DoNotRespect ); - final IdentityTokens tokens = uid2Service.generateIdentity(identityRequest); + final IdentityTokens tokens = uid2Service.generateIdentity( + identityRequest, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); assertNotNull(tokens); assertFalse(tokens.isEmptyToken()); final RefreshToken refreshToken = this.tokenEncoder.decodeRefreshToken(tokens.getRefreshToken()); - assertEquals(RefreshResponse.Optout, uid2Service.refreshIdentity(refreshToken)); + assertEquals(RefreshResponse.Optout, uid2Service.refreshIdentity( + refreshToken, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS))); } @Test @@ -226,7 +245,11 @@ public void testTestOptOutKey_RespectOptout() { inputVal.toUserIdentity(IdentityScope.UID2, 0, this.now), OptoutCheckPolicy.RespectOptOut ); - final IdentityTokens tokens = uid2Service.generateIdentity(identityRequest); + final IdentityTokens tokens = uid2Service.generateIdentity( + identityRequest, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); assertTrue(tokens.isEmptyToken()); verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); @@ -242,14 +265,22 @@ public void testTestOptOutKeyIdentityScopeMismatch() { inputVal.toUserIdentity(IdentityScope.EUID, 0, this.now), OptoutCheckPolicy.DoNotRespect ); - final IdentityTokens tokens = euidService.generateIdentity(identityRequest); + final IdentityTokens tokens = euidService.generateIdentity( + identityRequest, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); assertNotNull(tokens); final RefreshToken refreshToken = this.tokenEncoder.decodeRefreshToken(tokens.getRefreshToken()); reset(shutdownHandler); - assertEquals(RefreshResponse.Invalid, uid2Service.refreshIdentity(refreshToken)); + assertEquals(RefreshResponse.Invalid, uid2Service.refreshIdentity( + refreshToken, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS))); verify(shutdownHandler, never()).handleSaltRetrievalResponse(anyBoolean()); } @@ -279,20 +310,36 @@ public void testGenerateTokenForOptOutUser(IdentityType type, String identity, I final AdvertisingToken advertisingToken; final IdentityTokens tokensAfterOptOut; if (scope == IdentityScope.UID2) { - tokens = uid2Service.generateIdentity(identityRequestForceGenerate); + tokens = uid2Service.generateIdentity( + identityRequestForceGenerate, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); advertisingToken = validateAndGetToken(tokenEncoder, tokens.getAdvertisingToken(), IdentityScope.UID2, userIdentity.identityType, identityRequestRespectOptOut.publisherIdentity.siteId); reset(shutdownHandler); - tokensAfterOptOut = uid2Service.generateIdentity(identityRequestRespectOptOut); + tokensAfterOptOut = uid2Service.generateIdentity( + identityRequestRespectOptOut, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); } else { - tokens = euidService.generateIdentity(identityRequestForceGenerate); + tokens = euidService.generateIdentity( + identityRequestForceGenerate, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); advertisingToken = validateAndGetToken(tokenEncoder, tokens.getAdvertisingToken(), IdentityScope.EUID, userIdentity.identityType, identityRequestRespectOptOut.publisherIdentity.siteId); reset(shutdownHandler); - tokensAfterOptOut = euidService.generateIdentity(identityRequestRespectOptOut); + tokensAfterOptOut = euidService.generateIdentity( + identityRequestRespectOptOut, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); } verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); @@ -403,10 +450,18 @@ void testSpecialIdentityOptOutTokenGenerate(TestIdentityInputType type, String i IdentityTokens tokens; if(scope == IdentityScope.EUID) { - tokens = euidService.generateIdentity(identityRequest); + tokens = euidService.generateIdentity( + identityRequest, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); } else { - tokens = uid2Service.generateIdentity(identityRequest); + tokens = uid2Service.generateIdentity( + identityRequest, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); } verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); @@ -466,10 +521,18 @@ void testSpecialIdentityOptOutTokenRefresh(TestIdentityInputType type, String id IdentityTokens tokens; if(scope == IdentityScope.EUID) { - tokens = euidService.generateIdentity(identityRequest); + tokens = euidService.generateIdentity( + identityRequest, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); } else { - tokens = uid2Service.generateIdentity(identityRequest); + tokens = uid2Service.generateIdentity( + identityRequest, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); } verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); @@ -481,7 +544,11 @@ void testSpecialIdentityOptOutTokenRefresh(TestIdentityInputType type, String id final RefreshToken refreshToken = this.tokenEncoder.decodeRefreshToken(tokens.getRefreshToken()); reset(shutdownHandler); - assertEquals(RefreshResponse.Optout, (scope == IdentityScope.EUID? euidService: uid2Service).refreshIdentity(refreshToken)); + assertEquals(RefreshResponse.Optout, (scope == IdentityScope.EUID? euidService: uid2Service).refreshIdentity( + refreshToken, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS))); verify(shutdownHandler, never()).handleSaltRetrievalResponse(anyBoolean()); } @@ -508,10 +575,18 @@ void testSpecialIdentityRefreshOptOutGenerate(TestIdentityInputType type, String IdentityTokens tokens; if(scope == IdentityScope.EUID) { - tokens = euidService.generateIdentity(identityRequest); + tokens = euidService.generateIdentity( + identityRequest, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); } else { - tokens = uid2Service.generateIdentity(identityRequest); + tokens = uid2Service.generateIdentity( + identityRequest, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); } verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); @@ -523,7 +598,11 @@ void testSpecialIdentityRefreshOptOutGenerate(TestIdentityInputType type, String final RefreshToken refreshToken = this.tokenEncoder.decodeRefreshToken(tokens.getRefreshToken()); reset(shutdownHandler); - assertEquals(RefreshResponse.Optout, (scope == IdentityScope.EUID? euidService: uid2Service).refreshIdentity(refreshToken)); + assertEquals(RefreshResponse.Optout, (scope == IdentityScope.EUID? euidService: uid2Service).refreshIdentity( + refreshToken, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS))); verify(shutdownHandler, never()).handleSaltRetrievalResponse(anyBoolean()); } @@ -584,10 +663,18 @@ void testSpecialIdentityValidateGenerate(TestIdentityInputType type, String id, IdentityTokens tokens; AdvertisingToken advertisingToken; if (scope == IdentityScope.EUID) { - tokens = euidService.generateIdentity(identityRequest); + tokens = euidService.generateIdentity( + identityRequest, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); } else { - tokens = uid2Service.generateIdentity(identityRequest); + tokens = uid2Service.generateIdentity( + identityRequest, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); } advertisingToken = validateAndGetToken(tokenEncoder, tokens.getAdvertisingToken(), scope, identityRequest.userIdentity.identityType, identityRequest.publisherIdentity.siteId); verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); @@ -647,10 +734,18 @@ void testNormalIdentityOptIn(TestIdentityInputType type, String id, IdentityScop ); IdentityTokens tokens; if(scope == IdentityScope.EUID) { - tokens = euidService.generateIdentity(identityRequest); + tokens = euidService.generateIdentity( + identityRequest, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); } else { - tokens = uid2Service.generateIdentity(identityRequest); + tokens = uid2Service.generateIdentity( + identityRequest, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); } verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); @@ -658,7 +753,11 @@ void testNormalIdentityOptIn(TestIdentityInputType type, String id, IdentityScop assertNotNull(tokens); final RefreshToken refreshToken = this.tokenEncoder.decodeRefreshToken(tokens.getRefreshToken()); - RefreshResponse refreshResponse = (scope == IdentityScope.EUID? euidService: uid2Service).refreshIdentity(refreshToken); + RefreshResponse refreshResponse = (scope == IdentityScope.EUID? euidService: uid2Service).refreshIdentity( + refreshToken, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); assertTrue(refreshResponse.isRefreshed()); assertNotNull(refreshResponse.getTokens()); assertNotEquals(RefreshResponse.Optout, refreshResponse); @@ -678,23 +777,23 @@ void testExpiredSaltsNotifiesShutdownHandler(TestIdentityInputType type, String saltProvider.loadContent(); UIDOperatorService uid2Service = new UIDOperatorService( - uid2Config, optOutStore, saltProvider, tokenEncoder, this.clock, IdentityScope.UID2, - this.shutdownHandler::handleSaltRetrievalResponse + this.shutdownHandler::handleSaltRetrievalResponse, + uid2Config.getBoolean(identityV3) ); UIDOperatorService euidService = new UIDOperatorService( - euidConfig, optOutStore, saltProvider, tokenEncoder, this.clock, IdentityScope.EUID, - this.shutdownHandler::handleSaltRetrievalResponse + this.shutdownHandler::handleSaltRetrievalResponse, + euidConfig.getBoolean(identityV3) ); when(this.optOutStore.getLatestEntry(any())).thenReturn(null); @@ -710,11 +809,19 @@ void testExpiredSaltsNotifiesShutdownHandler(TestIdentityInputType type, String AdvertisingToken advertisingToken; reset(shutdownHandler); if(scope == IdentityScope.EUID) { - tokens = euidService.generateIdentity(identityRequest); + tokens = euidService.generateIdentity( + identityRequest, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); advertisingToken = validateAndGetToken(tokenEncoder, tokens.getAdvertisingToken(), IdentityScope.EUID, identityRequest.userIdentity.identityType, identityRequest.publisherIdentity.siteId); } else { - tokens = uid2Service.generateIdentity(identityRequest); + tokens = uid2Service.generateIdentity( + identityRequest, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); advertisingToken = validateAndGetToken(tokenEncoder, tokens.getAdvertisingToken(), IdentityScope.UID2, identityRequest.userIdentity.identityType, identityRequest.publisherIdentity.siteId); } verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(true); @@ -725,7 +832,11 @@ void testExpiredSaltsNotifiesShutdownHandler(TestIdentityInputType type, String final RefreshToken refreshToken = this.tokenEncoder.decodeRefreshToken(tokens.getRefreshToken()); reset(shutdownHandler); - RefreshResponse refreshResponse = (scope == IdentityScope.EUID? euidService: uid2Service).refreshIdentity(refreshToken); + RefreshResponse refreshResponse = (scope == IdentityScope.EUID? euidService: uid2Service).refreshIdentity( + refreshToken, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(true); verify(shutdownHandler, never()).handleSaltRetrievalResponse(false); assertTrue(refreshResponse.isRefreshed()); diff --git a/src/test/java/com/uid2/operator/benchmark/BenchmarkCommon.java b/src/test/java/com/uid2/operator/benchmark/BenchmarkCommon.java index 4cc327e9f..11b0f4a3b 100644 --- a/src/test/java/com/uid2/operator/benchmark/BenchmarkCommon.java +++ b/src/test/java/com/uid2/operator/benchmark/BenchmarkCommon.java @@ -41,6 +41,8 @@ import java.util.List; import java.util.Random; +import static com.uid2.operator.Const.Config.identityV3; + public class BenchmarkCommon { static IUIDOperatorService createUidOperatorService() throws Exception { @@ -59,16 +61,9 @@ static IUIDOperatorService createUidOperatorService() throws Exception { "/com.uid2.core/test/salts/metadata.json"); saltProvider.loadContent(); - final int IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS = 600; - final int REFRESH_TOKEN_EXPIRES_AFTER_SECONDS = 900; - final int REFRESH_IDENTITY_TOKEN_AFTER_SECONDS = 300; - - final JsonObject config = new JsonObject(); - config.put(UIDOperatorService.IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS); - config.put(UIDOperatorService.REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, REFRESH_TOKEN_EXPIRES_AFTER_SECONDS); - config.put(UIDOperatorService.REFRESH_IDENTITY_TOKEN_AFTER_SECONDS, REFRESH_IDENTITY_TOKEN_AFTER_SECONDS); + final JsonObject config = getConfig(); - final EncryptedTokenEncoder tokenEncoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); + final EncryptedTokenEncoder tokenEncoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); final List optOutPartitionFiles = new ArrayList<>(); final ICloudStorage optOutLocalStorage = make1mOptOutEntryStorage( saltProvider.getSnapshot(Instant.now()).getFirstLevelSalt(), @@ -76,16 +71,28 @@ static IUIDOperatorService createUidOperatorService() throws Exception { final IOptOutStore optOutStore = new StaticOptOutStore(optOutLocalStorage, make1mOptOutEntryConfig(), optOutPartitionFiles); return new UIDOperatorService( - config, optOutStore, saltProvider, tokenEncoder, Clock.systemUTC(), IdentityScope.UID2, - null + null, + config.getBoolean(identityV3) ); } + public static JsonObject getConfig() { + final int IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS = 600; + final int REFRESH_TOKEN_EXPIRES_AFTER_SECONDS = 900; + final int REFRESH_IDENTITY_TOKEN_AFTER_SECONDS = 300; + + final JsonObject config = new JsonObject(); + config.put(UIDOperatorService.IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS); + config.put(UIDOperatorService.REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, REFRESH_TOKEN_EXPIRES_AFTER_SECONDS); + config.put(UIDOperatorService.REFRESH_IDENTITY_TOKEN_AFTER_SECONDS, REFRESH_IDENTITY_TOKEN_AFTER_SECONDS); + return config; + } + static EncryptedTokenEncoder createTokenEncoder() throws Exception { RotatingKeysetKeyStore keysetKeyStore = new RotatingKeysetKeyStore( new EmbeddedResourceStorage(Main.class), diff --git a/src/test/java/com/uid2/operator/benchmark/TokenEndecBenchmark.java b/src/test/java/com/uid2/operator/benchmark/TokenEndecBenchmark.java index aaa821db9..d1fba74be 100644 --- a/src/test/java/com/uid2/operator/benchmark/TokenEndecBenchmark.java +++ b/src/test/java/com/uid2/operator/benchmark/TokenEndecBenchmark.java @@ -3,13 +3,17 @@ import com.uid2.operator.model.*; import com.uid2.operator.service.EncryptedTokenEncoder; import com.uid2.operator.service.IUIDOperatorService; +import io.vertx.core.json.JsonObject; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; import org.openjdk.jmh.annotations.Mode; +import java.time.Duration; import java.util.ArrayList; import java.util.List; +import static com.uid2.operator.service.UIDOperatorService.*; + public class TokenEndecBenchmark { private static final IUIDOperatorService uidService; @@ -18,6 +22,7 @@ public class TokenEndecBenchmark { private static final EncryptedTokenEncoder encoder; private static final IdentityTokens[] generatedTokens; private static int idx = 0; + private static final JsonObject config; static { try { @@ -29,6 +34,7 @@ public class TokenEndecBenchmark { if (generatedTokens.length < 65536 || userIdentities.length < 65536) { throw new IllegalStateException("must create more than 65535 test candidates."); } + config = BenchmarkCommon.getConfig(); } catch (Exception e) { throw new RuntimeException(e); } @@ -38,10 +44,14 @@ static IdentityTokens[] createAdvertisingTokens() { List tokens = new ArrayList<>(); for (int i = 0; i < userIdentities.length; i++) { tokens.add( - uidService.generateIdentity(new IdentityRequest( - publisher, - userIdentities[i], - OptoutCheckPolicy.DoNotRespect))); + uidService.generateIdentity( + new IdentityRequest( + publisher, + userIdentities[i], + OptoutCheckPolicy.DoNotRespect), + Duration.ofSeconds(config.getInteger(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS)), + Duration.ofSeconds(config.getInteger(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS)), + Duration.ofSeconds(config.getInteger(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)))); } return tokens.toArray(new IdentityTokens[tokens.size()]); } @@ -52,7 +62,10 @@ public IdentityTokens TokenGenerationBenchmark() { return uidService.generateIdentity(new IdentityRequest( publisher, userIdentities[(idx++) & 65535], - OptoutCheckPolicy.DoNotRespect)); + OptoutCheckPolicy.DoNotRespect), + Duration.ofSeconds(config.getInteger(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS)), + Duration.ofSeconds(config.getInteger(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS)), + Duration.ofSeconds(config.getInteger(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS))); } @Benchmark @@ -60,6 +73,9 @@ public IdentityTokens TokenGenerationBenchmark() { public RefreshResponse TokenRefreshBenchmark() { return uidService.refreshIdentity( encoder.decodeRefreshToken( - generatedTokens[(idx++) & 65535].getRefreshToken())); + generatedTokens[(idx++) & 65535].getRefreshToken()), + Duration.ofSeconds(config.getInteger(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS)), + Duration.ofSeconds(config.getInteger(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS)), + Duration.ofSeconds(config.getInteger(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS))); } } From 491025bc54f3f53da3aebb2ae6012e19e85f964f Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Wed, 18 Dec 2024 15:15:17 +1100 Subject: [PATCH 178/431] Removed setMaxSharingLifetimeSeconds metho from ExtendedUIDOperatorVerticle, Updated UIDOperatorVerticleTest --- .../java/com/uid2/operator/vertx/UIDOperatorVerticle.java | 1 - .../java/com/uid2/operator/ExtendedUIDOperatorVerticle.java | 4 ---- src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java | 2 +- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index 2c2c275ff..a8a4cbdcf 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -118,7 +118,6 @@ public class UIDOperatorVerticle extends AbstractVerticle { public final static long OPT_OUT_CHECK_CUTOFF_DATE = Instant.parse("2023-09-01T00:00:00.00Z").getEpochSecond(); private final Handler saltRetrievalResponseHandler; private final int allowClockSkewSeconds; - protected int maxSharingLifetimeSeconds; protected Map> siteIdToInvalidOriginsAndAppNames = new HashMap<>(); protected boolean keySharingEndpointProvideAppNames; protected Instant lastInvalidOriginProcessTime = Instant.now(); diff --git a/src/test/java/com/uid2/operator/ExtendedUIDOperatorVerticle.java b/src/test/java/com/uid2/operator/ExtendedUIDOperatorVerticle.java index cfb90630b..379d39719 100644 --- a/src/test/java/com/uid2/operator/ExtendedUIDOperatorVerticle.java +++ b/src/test/java/com/uid2/operator/ExtendedUIDOperatorVerticle.java @@ -40,10 +40,6 @@ public void setKeySharingEndpointProvideAppNames(boolean enable) { this.keySharingEndpointProvideAppNames = enable; } - public void setMaxSharingLifetimeSeconds(int maxSharingLifetimeSeconds) { - this.maxSharingLifetimeSeconds = maxSharingLifetimeSeconds; - } - public void setLastInvalidOriginProcessTime(Instant lastInvalidOriginProcessTime) { this.lastInvalidOriginProcessTime = lastInvalidOriginProcessTime; } diff --git a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java index 8a2888d61..830b7521b 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java @@ -4745,7 +4745,7 @@ void keyDownloadEndpointKeysets_IDREADER(boolean provideAppNames, KeyDownloadEnd @Test void keySharingKeysets_SHARER_CustomMaxSharingLifetimeSeconds(Vertx vertx, VertxTestContext testContext) { - this.uidOperatorVerticle.setMaxSharingLifetimeSeconds(999999); + this.config.put(Const.Config.MaxSharingLifetimeProp, 999999); keySharingKeysets_SHARER(true, true, vertx, testContext, 999999); } From 8018196a3e5f2e3432105fb7432eb8ed740ed25d Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Wed, 18 Dec 2024 15:47:15 +1100 Subject: [PATCH 179/431] Added ConfigValidatorUtilTest, Removed unused imports in UIDOperatorVerticleTest --- .../operator/UIDOperatorVerticleTest.java | 3 -- .../service/ConfigValidatorUtilTest.java | 32 +++++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) create mode 100644 src/test/java/com/uid2/operator/service/ConfigValidatorUtilTest.java diff --git a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java index 830b7521b..bfcaa4451 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java @@ -13,7 +13,6 @@ import com.uid2.operator.util.Tuple; import com.uid2.operator.vertx.OperatorShutdownHandler; import com.uid2.operator.vertx.UIDOperatorVerticle; -import com.uid2.operator.vertx.ClientInputValidationException; import com.uid2.shared.Utils; import com.uid2.shared.auth.ClientKey; import com.uid2.shared.auth.Keyset; @@ -27,9 +26,7 @@ import com.uid2.shared.secret.KeyHasher; import com.uid2.shared.store.*; import com.uid2.shared.store.reader.RotatingKeysetProvider; -import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.Metrics; -import io.micrometer.core.instrument.search.MeterNotFoundException; import io.micrometer.core.instrument.simple.SimpleMeterRegistry; import io.vertx.core.AsyncResult; import io.vertx.core.Future; diff --git a/src/test/java/com/uid2/operator/service/ConfigValidatorUtilTest.java b/src/test/java/com/uid2/operator/service/ConfigValidatorUtilTest.java new file mode 100644 index 000000000..9d71bb376 --- /dev/null +++ b/src/test/java/com/uid2/operator/service/ConfigValidatorUtilTest.java @@ -0,0 +1,32 @@ +package com.uid2.operator.service; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +public class ConfigValidatorUtilTest { + @Test + void testValidateIdentityRefreshTokens() { + // identityExpiresAfter is greater than refreshExpiresAfter + assertFalse(ConfigValidatorUtil.validateIdentityRefreshTokens(10, 5, 3)); + + // refreshIdentityAfter is greater than identityExpiresAfter + assertFalse(ConfigValidatorUtil.validateIdentityRefreshTokens(5, 10, 6)); + + // refreshIdentityAfter is greater than refreshExpiresAfter + assertFalse(ConfigValidatorUtil.validateIdentityRefreshTokens(5, 10, 11)); + + // all conditions are valid + assertTrue(ConfigValidatorUtil.validateIdentityRefreshTokens(5, 10, 3)); + } + + @Test + void testValidateBidstreamLifetime() { + // maxBidstreamLifetimeSeconds is less than identityTokenExpiresAfterSeconds + assertFalse(ConfigValidatorUtil.validateBidstreamLifetime(5, 10)); + + // maxBidstreamLifetimeSeconds is greater than or equal to identityTokenExpiresAfterSeconds + assertTrue(ConfigValidatorUtil.validateBidstreamLifetime(10, 5)); + assertTrue(ConfigValidatorUtil.validateBidstreamLifetime(10, 10)); + } +} From a40e3821df227a28c04177fe1578a80f0eddd6d5 Mon Sep 17 00:00:00 2001 From: Sophia Chen Date: Wed, 18 Dec 2024 16:02:10 +1100 Subject: [PATCH 180/431] sch-UID2-4560-adding-gauge-for-number-of-request-processing-threads added gauge for event loop threads --- src/main/java/com/uid2/operator/Main.java | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index 51c7bef6c..9ca0c27a7 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -265,7 +265,8 @@ private ICloudStorage wrapCloudStorageForOptOut(ICloudStorage cloudStorage) { } private void run() throws Exception { - this.createServiceInstancesMetric(); + this.createVertxInstancesMetric(); + this.createVertxEventLoopsMetric(); Supplier operatorVerticleSupplier = () -> { UIDOperatorVerticle verticle = new UIDOperatorVerticle(config, this.clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, getKeyManager(), saltProvider, optOutStore, Clock.systemUTC(), _statsCollectorQueue, new SecureLinkValidatorService(this.serviceLinkProvider, this.serviceProvider), this.shutdownHandler::handleSaltRetrievalResponse); return verticle; @@ -468,9 +469,15 @@ public DistributionStatisticConfig configure(Meter.Id id, DistributionStatisticC .register(globalRegistry); } - private void createServiceInstancesMetric() { + private void createVertxInstancesMetric() { Gauge.builder("uid2.operator.vertx_service_instances", () -> config.getInteger("service_instances")) - .description("gauge for number of request processing threads") + .description("gauge for number of vertx service instances requested") + .register(Metrics.globalRegistry); + } + + private void createVertxEventLoopsMetric() { + Gauge.builder("uid2.operator.vertx_event_loop_threads", () -> VertxOptions.DEFAULT_EVENT_LOOP_POOL_SIZE) + .description("gauge for number of vertx event loop threads") .register(Metrics.globalRegistry); } From 4f094c80548116c733933a6b185b7bfbf0af61b9 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Thu, 19 Dec 2024 00:15:33 +0000 Subject: [PATCH 181/431] [CI Pipeline] Released Patch version: 5.43.9 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 69634018d..7450255cd 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.43.4 + 5.43.9 UTF-8 From cd23998333ec6c9c9168464025d9fc85567914d7 Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Thu, 19 Dec 2024 15:34:56 +1100 Subject: [PATCH 182/431] Updated ConfigValidatorUtil to handle null values, Updated ConfigValidatorUtilTest to test handling null values --- .../operator/service/ConfigValidatorUtil.java | 13 +++++++++ .../service/ConfigValidatorUtilTest.java | 27 +++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/src/main/java/com/uid2/operator/service/ConfigValidatorUtil.java b/src/main/java/com/uid2/operator/service/ConfigValidatorUtil.java index 4f1c0999c..c7643d707 100644 --- a/src/main/java/com/uid2/operator/service/ConfigValidatorUtil.java +++ b/src/main/java/com/uid2/operator/service/ConfigValidatorUtil.java @@ -7,9 +7,17 @@ public class ConfigValidatorUtil { private static final Logger logger = LoggerFactory.getLogger(ConfigValidatorUtil.class); + public static final String VALUES_ARE_NULL = "Required config values are null"; public static Boolean validateIdentityRefreshTokens(Integer identityExpiresAfter, Integer refreshExpiresAfter, Integer refreshIdentityAfter) { boolean isValid = true; + + if (identityExpiresAfter == null || refreshExpiresAfter == null || refreshIdentityAfter == null) { + logger.error(VALUES_ARE_NULL); + return false; + } + + if (identityExpiresAfter > refreshExpiresAfter) { logger.error(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS + " must be >= " + IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS); isValid = false; @@ -25,6 +33,11 @@ public static Boolean validateIdentityRefreshTokens(Integer identityExpiresAfter } public static Boolean validateBidstreamLifetime(Integer maxBidstreamLifetimeSeconds, Integer identityTokenExpiresAfterSeconds) { + if (maxBidstreamLifetimeSeconds == null || identityTokenExpiresAfterSeconds == null) { + logger.error(VALUES_ARE_NULL); + return false; + } + if (maxBidstreamLifetimeSeconds < identityTokenExpiresAfterSeconds) { logger.error("Max bidstream lifetime seconds ({} seconds) is less than identity token lifetime ({} seconds)", maxBidstreamLifetimeSeconds, identityTokenExpiresAfterSeconds); return false; diff --git a/src/test/java/com/uid2/operator/service/ConfigValidatorUtilTest.java b/src/test/java/com/uid2/operator/service/ConfigValidatorUtilTest.java index 9d71bb376..dde9b6080 100644 --- a/src/test/java/com/uid2/operator/service/ConfigValidatorUtilTest.java +++ b/src/test/java/com/uid2/operator/service/ConfigValidatorUtilTest.java @@ -29,4 +29,31 @@ void testValidateBidstreamLifetime() { assertTrue(ConfigValidatorUtil.validateBidstreamLifetime(10, 5)); assertTrue(ConfigValidatorUtil.validateBidstreamLifetime(10, 10)); } + + @Test + void testValidateIdentityRefreshTokensWithNullValues() { + // identityExpiresAfter is null + assertFalse(ConfigValidatorUtil.validateIdentityRefreshTokens(null, 10, 5)); + + // refreshExpiresAfter is null + assertFalse(ConfigValidatorUtil.validateIdentityRefreshTokens(10, null, 5)); + + // refreshIdentityAfter is null + assertFalse(ConfigValidatorUtil.validateIdentityRefreshTokens(10, 5, null)); + + // all values are null + assertFalse(ConfigValidatorUtil.validateIdentityRefreshTokens(null, null, null)); + } + + @Test + void testValidateBidstreamLifetimeWithNullValues() { + // maxBidstreamLifetimeSeconds is null + assertFalse(ConfigValidatorUtil.validateBidstreamLifetime(null, 10)); + + // identityTokenExpiresAfterSeconds is null + assertFalse(ConfigValidatorUtil.validateBidstreamLifetime(10, null)); + + // both values are null + assertFalse(ConfigValidatorUtil.validateBidstreamLifetime(null, null)); + } } From bde83a5362e2eaee5dc1633c57bc0ff4d84be77f Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Thu, 19 Dec 2024 15:52:35 +1100 Subject: [PATCH 183/431] Fixed httpStore in ConfigService --- conf/default-config.json | 2 +- src/main/java/com/uid2/operator/Const.java | 2 +- .../java/com/uid2/operator/service/ConfigService.java | 9 ++++++--- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/conf/default-config.json b/conf/default-config.json index 849501309..d160858de 100644 --- a/conf/default-config.json +++ b/conf/default-config.json @@ -36,6 +36,6 @@ "failure_shutdown_wait_hours": 120, "sharing_token_expiry_seconds": 2592000, "operator_type": "public", - "core_config_url": "http://localhost:8088/config", + "core_config_path": "/config", "config_scan_period_ms": 300000 } diff --git a/src/main/java/com/uid2/operator/Const.java b/src/main/java/com/uid2/operator/Const.java index 084e8f3bb..2b70c1cb0 100644 --- a/src/main/java/com/uid2/operator/Const.java +++ b/src/main/java/com/uid2/operator/Const.java @@ -30,7 +30,7 @@ public class Config extends com.uid2.shared.Const.Config { public static final String MaxInvalidPaths = "logging_limit_max_invalid_paths_per_interval"; public static final String MaxVersionBucketsPerSite = "logging_limit_max_version_buckets_per_site"; - public static final String CoreConfigUrl = "core_config_url"; //TODO: update when endpoint name finalised + public static final String CoreConfigPath = "core_config_path"; //TODO: update when endpoint name finalised public static final String ConfigScanPeriodMs = "config_scan_period_ms"; public static final String Config = "config"; public static final String identityV3 = "identity_v3"; diff --git a/src/main/java/com/uid2/operator/service/ConfigService.java b/src/main/java/com/uid2/operator/service/ConfigService.java index 4f95b8696..44de433d3 100644 --- a/src/main/java/com/uid2/operator/service/ConfigService.java +++ b/src/main/java/com/uid2/operator/service/ConfigService.java @@ -44,13 +44,15 @@ public JsonObject getConfig() { } private void initialiseConfigRetriever(Vertx vertx, JsonObject bootstrapConfig) { - String configUrl = bootstrapConfig.getString(CoreConfigUrl); + String configPath = bootstrapConfig.getString(CoreConfigPath); + ConfigStoreOptions httpStore = new ConfigStoreOptions() .setType("http") .setConfig(new JsonObject() - .put("url", configUrl) - .put("method", "GET")); + .put("host", "127.0.0.1") + .put("port", Const.Port.ServicePortForCore) + .put("path", configPath)); ConfigStoreOptions bootstrapStore = new ConfigStoreOptions() .setType("json") @@ -70,6 +72,7 @@ private void initialiseConfigRetriever(Vertx vertx, JsonObject bootstrapConfig) System.out.println("Successfully loaded config"); } else { System.err.println("Failed to load config: " + ar.cause().getMessage()); + logger.error("Failed to load config"); } }); From 1c254fe632f3fe13de21608b4cc64382d808ed01 Mon Sep 17 00:00:00 2001 From: Sophia Chen Date: Thu, 19 Dec 2024 17:22:40 +1100 Subject: [PATCH 184/431] sch-UID2-4560-adding-gauge-for-number-of-request-processing-threads renamed gauge --- src/main/java/com/uid2/operator/Main.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index 9ca0c27a7..a307e3695 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -470,13 +470,13 @@ public DistributionStatisticConfig configure(Meter.Id id, DistributionStatisticC } private void createVertxInstancesMetric() { - Gauge.builder("uid2.operator.vertx_service_instances", () -> config.getInteger("service_instances")) + Gauge.builder("uid2.vertx_service_instances", () -> config.getInteger("service_instances")) .description("gauge for number of vertx service instances requested") .register(Metrics.globalRegistry); } private void createVertxEventLoopsMetric() { - Gauge.builder("uid2.operator.vertx_event_loop_threads", () -> VertxOptions.DEFAULT_EVENT_LOOP_POOL_SIZE) + Gauge.builder("uid2.vertx_event_loop_threads", () -> VertxOptions.DEFAULT_EVENT_LOOP_POOL_SIZE) .description("gauge for number of vertx event loop threads") .register(Metrics.globalRegistry); } From d840b91d2e4580ff61a5dc73e171e3bb4225b05e Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Thu, 19 Dec 2024 22:17:28 +0000 Subject: [PATCH 185/431] [CI Pipeline] Released Patch version: 5.43.11 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 7450255cd..7b231d3ce 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.43.9 + 5.43.11 UTF-8 From 8478fd208d9668137f35c6e691f8b3cc5566f1e5 Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Fri, 20 Dec 2024 10:31:27 +1100 Subject: [PATCH 186/431] Added ConfigServiceTest --- .../com/uid2/operator/ConfigServiceTest.java | 161 ++++++++++++++++++ 1 file changed, 161 insertions(+) create mode 100644 src/test/java/com/uid2/operator/ConfigServiceTest.java diff --git a/src/test/java/com/uid2/operator/ConfigServiceTest.java b/src/test/java/com/uid2/operator/ConfigServiceTest.java new file mode 100644 index 000000000..e41954ee5 --- /dev/null +++ b/src/test/java/com/uid2/operator/ConfigServiceTest.java @@ -0,0 +1,161 @@ +package com.uid2.operator; + +import com.uid2.operator.service.ConfigService; +import io.vertx.core.Vertx; +import io.vertx.core.json.JsonObject; +import io.vertx.config.ConfigRetriever; +import io.vertx.ext.web.Router; +import io.vertx.ext.web.handler.BodyHandler; +import org.junit.jupiter.api.*; +import io.vertx.junit5.VertxExtension; +import io.vertx.junit5.VertxTestContext; +import org.junit.jupiter.api.extension.ExtendWith; + +import java.lang.reflect.*; +import java.util.concurrent.TimeUnit; + +import static com.uid2.operator.Const.Config.*; +import static com.uid2.operator.service.UIDOperatorService.*; +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; + +@ExtendWith(VertxExtension.class) +class ConfigServiceTest { + private Vertx vertx; + private JsonObject bootstrapConfig; + + @BeforeEach + void setUp() { + vertx = Vertx.vertx(); + bootstrapConfig = new JsonObject() + .put(CoreConfigPath, "/config") + .put(ConfigScanPeriodMs, 300000) + .put(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, 3600) + .put(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, 7200) + .put(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS, 1800) + .put(MaxBidstreamLifetimeSecondsProp, 7200); + } + + @AfterEach + void tearDown() { + vertx.close(); + } + + void startMockServer(VertxTestContext testContext, JsonObject config) throws InterruptedException { + Router router = Router.router(vertx); + router.route().handler(BodyHandler.create()); + router.get("/config").handler(ctx -> ctx.response() + .putHeader("content-type", "application/json") + .end(config.encode())); + + vertx.createHttpServer() + .requestHandler(router) + .listen(Const.Port.ServicePortForCore,"127.0.0.1", http -> { + if (!http.succeeded()) { + testContext.failNow(http.cause()); + } + }); + + testContext.awaitCompletion(5, TimeUnit.SECONDS); + } + + @Test + void testSingletonBehavior() { + ConfigService instance1 = ConfigService.getInstance(vertx, bootstrapConfig); + ConfigService instance2 = ConfigService.getInstance(vertx, bootstrapConfig); + assertSame(instance1, instance2, "getInstance should return the same instance"); + } + + @Test + void testGetConfig() { + ConfigRetriever mockConfigRetriever = mock(ConfigRetriever.class); + JsonObject cachedConfig = new JsonObject().put("key", "value"); + when(mockConfigRetriever.getCachedConfig()).thenReturn(cachedConfig); + ConfigService configService = ConfigService.getInstance(vertx, bootstrapConfig); + // Reflection to inject mocked ConfigRetriever + try { + Field configRetrieverField = ConfigService.class.getDeclaredField("configRetriever"); + configRetrieverField.setAccessible(true); + configRetrieverField.set(configService, mockConfigRetriever); + } catch (Exception e) { + fail("Failed to inject mock ConfigRetriever: " + e.getMessage()); + } + JsonObject result = configService.getConfig(); + assertEquals(cachedConfig, result, "getConfig should return the cached configuration"); + } + + @Test + void testInitialiseConfigRetriever(VertxTestContext testContext) throws InterruptedException { + JsonObject httpStoreConfig = new JsonObject().put("http", "value"); + this.startMockServer(testContext, httpStoreConfig); + ConfigService configService = ConfigService.getInstance(vertx, bootstrapConfig); + // Wait for the initialisation to finish - alternative is to have getInstance return a future + testContext.awaitCompletion(1, TimeUnit.SECONDS); + JsonObject retrievedConfig = configService.getConfig(); + assertNotNull(retrievedConfig, "Config retriever should initialise without error"); + assertTrue(retrievedConfig.fieldNames().containsAll(bootstrapConfig.fieldNames()), "Retrieved config should contain all keys in bootstrap config"); + assertTrue(retrievedConfig.fieldNames().containsAll(httpStoreConfig.fieldNames()), "Retrieved config should contain all keys in http store config"); + testContext.completeNow(); + } + + @Test + void testInvalidConfigRevertsToPrevious() { + ConfigRetriever mockConfigRetriever = mock(ConfigRetriever.class); + JsonObject lastConfig = new JsonObject().put("previous", "config"); + JsonObject invalidConfig = new JsonObject() + .put(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, 1000) + .put(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, 2000); + when(mockConfigRetriever.getCachedConfig()).thenReturn(lastConfig); + ConfigService configService = ConfigService.getInstance(vertx, bootstrapConfig); + try { + Field configRetrieverField = ConfigService.class.getDeclaredField("configRetriever"); + configRetrieverField.setAccessible(true); + configRetrieverField.set(configService, mockConfigRetriever); + } catch (Exception e) { + fail("Failed to inject mock ConfigRetriever: " + e.getMessage()); + } + + try { + Method configValidationHandlerMethod = ConfigService.class.getDeclaredMethod("configValidationHandler", JsonObject.class); + configValidationHandlerMethod.setAccessible(true); + JsonObject validatedConfig = (JsonObject) configValidationHandlerMethod.invoke(configService, invalidConfig); + assertEquals(lastConfig, validatedConfig, "Invalid config not reverted to previous config"); + } catch (Exception e) { + fail("Failed to access and invoke the configValidationHandler method: " + e.getMessage()); + } + } + + @Test + void testFirstInvalidConfigThrowsRuntimeException() { + ConfigRetriever mockConfigRetriever = mock(ConfigRetriever.class); + JsonObject invalidConfig = new JsonObject() + .put(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, 1000) + .put(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, 2000); + when(mockConfigRetriever.getCachedConfig()).thenReturn(null); + ConfigService configService = ConfigService.getInstance(vertx, bootstrapConfig); + try { + Field configRetrieverField = ConfigService.class.getDeclaredField("configRetriever"); + configRetrieverField.setAccessible(true); + configRetrieverField.set(configService, mockConfigRetriever); + } catch (Exception e) { + fail("Failed to inject mock ConfigRetriever: " + e.getMessage()); + } + + + + try { + Method configValidationHandlerMethod = ConfigService.class.getDeclaredMethod("configValidationHandler", JsonObject.class); + configValidationHandlerMethod.setAccessible(true); + assertThrows(RuntimeException.class, () -> { + try { + configValidationHandlerMethod.invoke(configService, invalidConfig); + } catch (InvocationTargetException e) { + // Throw cause as InvocationTargetException wraps actual exception thrown by configValidationHandler + throw e.getCause(); + } + }); + } catch (Exception e) { + fail("Failed to access and invoke the configValidationHandler method: " + e.getMessage()); + } + } +} \ No newline at end of file From cfb31c6d5f91e1ea6202c9f6c03113500bc8c91a Mon Sep 17 00:00:00 2001 From: abuabraham-ttd <130511456+abuabraham-ttd@users.noreply.github.com> Date: Thu, 26 Dec 2024 11:06:57 -0800 Subject: [PATCH 187/431] Removing start, stop with ec2.py, adding validations (#1191) * Removing start, stop with ec2.py, adding validations --------- Co-authored-by: Release Workflow --- .github/actions/build_aws_eif/action.yaml | 5 +- scripts/aws/EUID_CloudFormation.template.yml | 28 +- scripts/aws/UID_CloudFormation.template.yml | 28 +- scripts/aws/config-server/requirements.txt | 2 +- scripts/aws/ec2.py | 259 ++++++++++++++++++ scripts/aws/entrypoint.sh | 1 + scripts/aws/requirements.txt | 4 + scripts/aws/start.sh | 124 --------- scripts/aws/stop.sh | 31 --- .../uid2-operator-ami/ansible/playbook.yml | 27 +- scripts/aws/uid2operator.service | 6 +- scripts/confidential_compute.py | 144 ++++++++++ 12 files changed, 474 insertions(+), 185 deletions(-) create mode 100644 scripts/aws/ec2.py create mode 100644 scripts/aws/requirements.txt delete mode 100644 scripts/aws/start.sh delete mode 100644 scripts/aws/stop.sh create mode 100644 scripts/confidential_compute.py diff --git a/.github/actions/build_aws_eif/action.yaml b/.github/actions/build_aws_eif/action.yaml index f17523a44..08e6d6604 100644 --- a/.github/actions/build_aws_eif/action.yaml +++ b/.github/actions/build_aws_eif/action.yaml @@ -96,8 +96,9 @@ runs: cp ${{ steps.buildFolder.outputs.BUILD_FOLDER }}/identity_scope.txt ${ARTIFACTS_OUTPUT_DIR}/ cp ${{ steps.buildFolder.outputs.BUILD_FOLDER }}/version_number.txt ${ARTIFACTS_OUTPUT_DIR}/ - cp ./scripts/aws/start.sh ${ARTIFACTS_OUTPUT_DIR}/ - cp ./scripts/aws/stop.sh ${ARTIFACTS_OUTPUT_DIR}/ + cp ./scripts/aws/ec2.py ${ARTIFACTS_OUTPUT_DIR}/ + cp ./scripts/confidential_compute.py ${ARTIFACTS_OUTPUT_DIR}/ + cp ./scripts/aws/requirements.txt ${ARTIFACTS_OUTPUT_DIR}/ cp ./scripts/aws/proxies.host.yaml ${ARTIFACTS_OUTPUT_DIR}/ cp ./scripts/aws/sockd.conf ${ARTIFACTS_OUTPUT_DIR}/ cp ./scripts/aws/uid2operator.service ${ARTIFACTS_OUTPUT_DIR}/ diff --git a/scripts/aws/EUID_CloudFormation.template.yml b/scripts/aws/EUID_CloudFormation.template.yml index 9c5982488..3bb7958ad 100644 --- a/scripts/aws/EUID_CloudFormation.template.yml +++ b/scripts/aws/EUID_CloudFormation.template.yml @@ -118,6 +118,10 @@ Mappings: AMI: ami-xxxxxxxxxxxxxxxxx eu-north-1: AMI: ami-xxxxxxxxxxxxxxxxx +Conditions: + IsIntegEnvironment: !Equals + - !Ref DeployToEnvironment + - integ Resources: KMSKey: Type: AWS::KMS::Key @@ -154,13 +158,23 @@ Resources: Description: EUID Token KmsKeyId: !GetAtt KMSKey.Arn Name: !Sub 'euid-config-stack-${AWS::StackName}' - SecretString: !Sub '{ - "api_token":"${APIToken}", - "service_instances":6, - "enclave_cpu_count":6, - "enclave_memory_mb":24576, - "environment":"${DeployToEnvironment}" - }' + SecretString: !Join + - '' + - - '{' + - '"core_base_url": ' + - !If [IsIntegEnvironment, 'https://core.integ.euid.eu', 'https://core.prod.euid.eu'] + - ', "optout_base_url": ' + - !If [IsIntegEnvironment, 'https://optout.integ.euid.eu', 'https://optout.prod.euid.eu'] + - ', "api_token": "' + - Ref: APIToken + - '"' + - ', "service_instances": 6' + - ', "enclave_cpu_count": 6' + - ', "enclave_memory_mb": 24576' + - ', "environment": "' + - Ref: DeployToEnvironment + - '"' + - '}' WorkerRole: Type: 'AWS::IAM::Role' Properties: diff --git a/scripts/aws/UID_CloudFormation.template.yml b/scripts/aws/UID_CloudFormation.template.yml index 711d1ab0e..179ec47a1 100644 --- a/scripts/aws/UID_CloudFormation.template.yml +++ b/scripts/aws/UID_CloudFormation.template.yml @@ -146,6 +146,10 @@ Mappings: AMI: ami-xxxxxxxxxxxxxxxxx af-south-1: AMI: ami-xxxxxxxxxxxxxxxxx +Conditions: + IsIntegEnvironment: !Equals + - !Ref DeployToEnvironment + - integ Resources: KMSKey: Type: AWS::KMS::Key @@ -182,13 +186,23 @@ Resources: Description: UID2 Token KmsKeyId: !GetAtt KMSKey.Arn Name: !Sub 'uid2-config-stack-${AWS::StackName}' - SecretString: !Sub '{ - "api_token":"${APIToken}", - "service_instances":6, - "enclave_cpu_count":6, - "enclave_memory_mb":24576, - "environment":"${DeployToEnvironment}" - }' + SecretString: !Join + - '' + - - '{' + - '"core_base_url": ' + - !If [IsIntegEnvironment, 'https://core-integ.uidapi.com', 'https://core.uidapi.com'] + - ', "optout_base_url": ' + - !If [IsIntegEnvironment, 'https://optout-integ.uidapi.com', 'https://optout.uidapi.com'] + - ', "api_token": "' + - Ref: APIToken + - '"' + - ', "service_instances": 6' + - ', "enclave_cpu_count": 6' + - ', "enclave_memory_mb": 24576' + - ', "environment": "' + - Ref: DeployToEnvironment + - '"' + - '}' WorkerRole: Type: 'AWS::IAM::Role' Properties: diff --git a/scripts/aws/config-server/requirements.txt b/scripts/aws/config-server/requirements.txt index 57652a258..8cdd5ef92 100644 --- a/scripts/aws/config-server/requirements.txt +++ b/scripts/aws/config-server/requirements.txt @@ -1,3 +1,3 @@ Flask==2.3.2 Werkzeug==3.0.3 -setuptools==70.0.0 +setuptools==70.0.0 \ No newline at end of file diff --git a/scripts/aws/ec2.py b/scripts/aws/ec2.py new file mode 100644 index 000000000..69fed7f91 --- /dev/null +++ b/scripts/aws/ec2.py @@ -0,0 +1,259 @@ +#!/usr/bin/env python3 + +import boto3 +import json +import os +import subprocess +import re +import multiprocessing +import requests +import signal +import argparse +from botocore.exceptions import ClientError +from typing import Dict +import sys +import time +import yaml + +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from confidential_compute import ConfidentialCompute, ConfidentialComputeConfig, SecretNotFoundException, ConfidentialComputeStartupException + +class AWSConfidentialComputeConfig(ConfidentialComputeConfig): + enclave_memory_mb: int + enclave_cpu_count: int + +class AuxiliaryConfig: + FLASK_PORT: str = "27015" + LOCALHOST: str = "127.0.0.1" + AWS_METADATA: str = "169.254.169.254" + + @classmethod + def get_socks_url(cls) -> str: + return f"socks5://{cls.LOCALHOST}:3306" + + @classmethod + def get_config_url(cls) -> str: + return f"http://{cls.LOCALHOST}:{cls.FLASK_PORT}/getConfig" + + @classmethod + def get_user_data_url(cls) -> str: + return f"http://{cls.AWS_METADATA}/latest/user-data" + + @classmethod + def get_token_url(cls) -> str: + return f"http://{cls.AWS_METADATA}/latest/api/token" + + @classmethod + def get_meta_url(cls) -> str: + return f"http://{cls.AWS_METADATA}/latest/dynamic/instance-identity/document" + + +class EC2(ConfidentialCompute): + + def __init__(self): + super().__init__() + + def __get_aws_token(self) -> str: + """Fetches a temporary AWS EC2 metadata token.""" + try: + response = requests.put( + AuxiliaryConfig.get_token_url(), headers={"X-aws-ec2-metadata-token-ttl-seconds": "3600"}, timeout=2 + ) + return response.text + except requests.RequestException as e: + raise RuntimeError(f"Failed to fetch aws token: {e}") + + def __get_current_region(self) -> str: + """Fetches the current AWS region from EC2 instance metadata.""" + token = self.__get_aws_token() + headers = {"X-aws-ec2-metadata-token": token} + try: + response = requests.get(AuxiliaryConfig.get_meta_url(), headers=headers, timeout=2) + response.raise_for_status() + return response.json()["region"] + except requests.RequestException as e: + raise RuntimeError(f"Failed to fetch region: {e}") + + def __validate_aws_specific_config(self, secret): + if "enclave_memory_mb" in secret or "enclave_cpu_count" in secret: + max_capacity = self.__get_max_capacity() + min_capacity = {"enclave_memory_mb": 11000, "enclave_cpu_count" : 2 } + for key in ["enclave_memory_mb", "enclave_cpu_count"]: + if int(secret.get(key, 0)) > max_capacity.get(key): + raise ValueError(f"{key} value ({secret.get(key, 0)}) exceeds the maximum allowed ({max_capacity.get(key)}).") + if min_capacity.get(key) > int(secret.get(key, 10**9)): + raise ValueError(f"{key} value ({secret.get(key, 0)}) needs to be higher than the minimum required ({min_capacity.get(key)}).") + + def _get_secret(self, secret_identifier: str) -> AWSConfidentialComputeConfig: + """Fetches a secret value from AWS Secrets Manager and adds defaults""" + + def add_defaults(configs: Dict[str, any]) -> AWSConfidentialComputeConfig: + """Adds default values to configuration if missing.""" + default_capacity = self.__get_max_capacity() + configs.setdefault("enclave_memory_mb", default_capacity["enclave_memory_mb"]) + configs.setdefault("enclave_cpu_count", default_capacity["enclave_cpu_count"]) + configs.setdefault("debug_mode", False) + return configs + + region = self.__get_current_region() + print(f"Running in {region}") + try: + client = boto3.client("secretsmanager", region_name=region) + except Exception as e: + raise RuntimeError("Please use IAM instance profile for your instance and make sure that has permission to access Secret Manager", e) + try: + secret = add_defaults(json.loads(client.get_secret_value(SecretId=secret_identifier)["SecretString"])) + self.__validate_aws_specific_config(secret) + return secret + except ClientError as _: + raise SecretNotFoundException(f"{secret_identifier} in {region}") + + @staticmethod + def __get_max_capacity(): + try: + with open("/etc/nitro_enclaves/allocator.yaml", "r") as file: + nitro_config = yaml.safe_load(file) + return {"enclave_memory_mb": nitro_config['memory_mib'], "enclave_cpu_count": nitro_config['cpu_count']} + except Exception as e: + raise RuntimeError("/etc/nitro_enclaves/allocator.yaml does not have CPU, memory allocated") + + def __setup_vsockproxy(self, log_level: int) -> None: + """ + Sets up the vsock proxy service. + """ + thread_count = (multiprocessing.cpu_count() + 1) // 2 + command = [ + "/usr/bin/vsockpx", "-c", "/etc/uid2operator/proxy.yaml", + "--workers", str(thread_count), "--log-level", str(log_level), "--daemon" + ] + self.run_command(command) + + def __run_config_server(self) -> None: + """ + Starts the Flask configuration server. + """ + os.makedirs("/etc/secret/secret-value", exist_ok=True) + config_path = "/etc/secret/secret-value/config" + with open(config_path, 'w') as config_file: + json.dump(self.configs, config_file) + os.chdir("/opt/uid2operator/config-server") + command = ["./bin/flask", "run", "--host", AuxiliaryConfig.LOCALHOST, "--port", AuxiliaryConfig.FLASK_PORT] + self.run_command(command, seperate_process=True) + + def __run_socks_proxy(self) -> None: + """ + Starts the SOCKS proxy service. + """ + command = ["sockd", "-D"] + self.run_command(command) + + def __get_secret_name_from_userdata(self) -> str: + """Extracts the secret name from EC2 user data.""" + token = self.__get_aws_token() + response = requests.get(AuxiliaryConfig.get_user_data_url(), headers={"X-aws-ec2-metadata-token": token}) + user_data = response.text + + with open("/opt/uid2operator/identity_scope.txt") as file: + identity_scope = file.read().strip() + + default_name = f"{identity_scope.lower()}-operator-config-key" + hardcoded_value = f"{identity_scope.upper()}_CONFIG_SECRET_KEY" + match = re.search(rf'^export {hardcoded_value}="(.+?)"$', user_data, re.MULTILINE) + return match.group(1) if match else default_name + + def _setup_auxiliaries(self) -> None: + """Sets up the vsock tunnel, socks proxy and flask server""" + log_level = 1 if self.configs["debug_mode"] else 3 + self.__setup_vsockproxy(log_level) + self.__run_config_server() + self.__run_socks_proxy() + print("Finished setting up all auxiliaries") + + def _validate_auxiliaries(self) -> None: + """Validates connection to flask server direct and through socks proxy.""" + print("Validating auxiliaries") + try: + for attempt in range(10): + try: + response = requests.get(AuxiliaryConfig.get_config_url()) + print("Config server is reachable") + break + except requests.exceptions.ConnectionError as e: + print(f"Connecting to config server, attempt {attempt + 1} failed with ConnectionError: {e}") + time.sleep(1) + else: + raise RuntimeError(f"Config server unreachable") + response.raise_for_status() + except requests.RequestException as e: + raise RuntimeError(f"Failed to get config from config server: {e}") + proxies = {"http": AuxiliaryConfig.get_socks_url(), "https": AuxiliaryConfig.get_socks_url()} + try: + response = requests.get(AuxiliaryConfig.get_config_url(), proxies=proxies) + response.raise_for_status() + except requests.RequestException as e: + raise RuntimeError(f"Cannot connect to config server via SOCKS proxy: {e}") + print("Connectivity check to config server passes") + + def __run_nitro_enclave(self): + command = [ + "nitro-cli", "run-enclave", + "--eif-path", "/opt/uid2operator/uid2operator.eif", + "--memory", str(self.configs["enclave_memory_mb"]), + "--cpu-count", str(self.configs["enclave_cpu_count"]), + "--enclave-cid", "42", + "--enclave-name", "uid2operator" + ] + if self.configs.get('debug_mode', False): + print("Running in debug_mode") + command += ["--debug-mode", "--attach-console"] + self.run_command(command, seperate_process=True) + + def run_compute(self) -> None: + """Main execution flow for confidential compute.""" + secret_manager_key = self.__get_secret_name_from_userdata() + self.configs = self._get_secret(secret_manager_key) + print(f"Fetched configs from {secret_manager_key}") + if not self.configs.get("skip_validations"): + self.validate_configuration() + self._setup_auxiliaries() + self._validate_auxiliaries() + self.__run_nitro_enclave() + + def cleanup(self) -> None: + """Terminates the Nitro Enclave and auxiliary processes.""" + try: + self.run_command(["nitro-cli", "terminate-enclave", "--all"]) + self.__kill_auxiliaries() + except subprocess.SubprocessError as e: + raise (f"Error during cleanup: {e}") + + def __kill_auxiliaries(self) -> None: + """Kills all auxiliary processes spawned.""" + for process_name in ["vsockpx", "sockd", "flask"]: + try: + result = subprocess.run(["pgrep", "-f", process_name], stdout=subprocess.PIPE, text=True, check=False) + if result.stdout.strip(): + for pid in result.stdout.strip().split("\n"): + os.kill(int(pid), signal.SIGKILL) + print(f"Killed process '{process_name}'.") + else: + print(f"No process named '{process_name}' found.") + except Exception as e: + print(f"Error killing process '{process_name}': {e}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Manage EC2-based confidential compute workflows.") + parser.add_argument("-o", "--operation", choices=["stop", "start"], default="start", help="Operation to perform.") + args = parser.parse_args() + try: + ec2 = EC2() + if args.operation == "stop": + ec2.cleanup() + else: + ec2.run_compute() + except ConfidentialComputeStartupException as e: + print("Failed starting up Confidential Compute. Please checks the logs for errors and retry \n", e) + except Exception as e: + print("Unknown failure while starting up Confidential Compute. Please contact UID support team with this log \n ", e) + \ No newline at end of file diff --git a/scripts/aws/entrypoint.sh b/scripts/aws/entrypoint.sh index 37214388b..c7f3676f9 100755 --- a/scripts/aws/entrypoint.sh +++ b/scripts/aws/entrypoint.sh @@ -101,6 +101,7 @@ cd /app # -- start operator echo "Starting Java application..." + java \ -XX:MaxRAMPercentage=95 -XX:-UseCompressedOops -XX:+PrintFlagsFinal \ -Djava.security.egd=file:/dev/./urandom \ diff --git a/scripts/aws/requirements.txt b/scripts/aws/requirements.txt new file mode 100644 index 000000000..421faba98 --- /dev/null +++ b/scripts/aws/requirements.txt @@ -0,0 +1,4 @@ +requests[socks]==2.32.3 +boto3==1.35.59 +urllib3==1.26.20 +PyYAML===6.0.2 \ No newline at end of file diff --git a/scripts/aws/start.sh b/scripts/aws/start.sh deleted file mode 100644 index 429826928..000000000 --- a/scripts/aws/start.sh +++ /dev/null @@ -1,124 +0,0 @@ -#!/bin/bash - -echo "$HOSTNAME" > /etc/uid2operator/HOSTNAME -EIF_PATH=${EIF_PATH:-/opt/uid2operator/uid2operator.eif} -IDENTITY_SCOPE=${IDENTITY_SCOPE:-$(cat /opt/uid2operator/identity_scope.txt)} -CID=${CID:-42} -TOKEN=$(curl --request PUT "http://169.254.169.254/latest/api/token" --header "X-aws-ec2-metadata-token-ttl-seconds: 3600") -USER_DATA=$(curl -s http://169.254.169.254/latest/user-data --header "X-aws-ec2-metadata-token: $TOKEN") -AWS_REGION_NAME=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document/ --header "X-aws-ec2-metadata-token: $TOKEN" | jq -r '.region') -if [ "$IDENTITY_SCOPE" = 'UID2' ]; then - UID2_CONFIG_SECRET_KEY=$([[ "$(echo "${USER_DATA}" | grep UID2_CONFIG_SECRET_KEY=)" =~ ^export\ UID2_CONFIG_SECRET_KEY=\"(.*)\"$ ]] && echo "${BASH_REMATCH[1]}" || echo "uid2-operator-config-key") -elif [ "$IDENTITY_SCOPE" = 'EUID' ]; then - UID2_CONFIG_SECRET_KEY=$([[ "$(echo "${USER_DATA}" | grep EUID_CONFIG_SECRET_KEY=)" =~ ^export\ EUID_CONFIG_SECRET_KEY=\"(.*)\"$ ]] && echo "${BASH_REMATCH[1]}" || echo "euid-operator-config-key") -else - echo "Unrecognized IDENTITY_SCOPE $IDENTITY_SCOPE" - exit 1 -fi -CORE_BASE_URL=$([[ "$(echo "${USER_DATA}" | grep CORE_BASE_URL=)" =~ ^export\ CORE_BASE_URL=\"(.*)\"$ ]] && echo "${BASH_REMATCH[1]}" || echo "") -OPTOUT_BASE_URL=$([[ "$(echo "${USER_DATA}" | grep OPTOUT_BASE_URL=)" =~ ^export\ OPTOUT_BASE_URL=\"(.*)\"$ ]] && echo "${BASH_REMATCH[1]}" || echo "") - -echo "UID2_CONFIG_SECRET_KEY=${UID2_CONFIG_SECRET_KEY}" -echo "CORE_BASE_URL=${CORE_BASE_URL}" -echo "OPTOUT_BASE_URL=${OPTOUT_BASE_URL}" -echo "AWS_REGION_NAME=${AWS_REGION_NAME}" - -function terminate_old_enclave() { - ENCLAVE_ID=$(nitro-cli describe-enclaves | jq -r ".[0].EnclaveID") - [ "$ENCLAVE_ID" != "null" ] && nitro-cli terminate-enclave --enclave-id ${ENCLAVE_ID} -} - -function config_aws() { - aws configure set default.region $AWS_REGION_NAME -} - -function default_cpu() { - target=$(( $(nproc) * 3 / 4 )) - if [ $target -lt 2 ]; then - target="2" - fi - echo $target -} - -function default_mem() { - target=$(( $(grep MemTotal /proc/meminfo | awk '{print $2}') * 3 / 4000 )) - if [ $target -lt 24576 ]; then - target="24576" - fi - echo $target -} - -function read_allocation() { - USER_CUSTOMIZED=$(aws secretsmanager get-secret-value --secret-id "$UID2_CONFIG_SECRET_KEY" | jq -r '.SecretString' | jq -r '.customize_enclave') - shopt -s nocasematch - if [ "$USER_CUSTOMIZED" = "true" ]; then - echo "Applying user customized CPU/Mem allocation..." - CPU_COUNT=${CPU_COUNT:-$(aws secretsmanager get-secret-value --secret-id "$UID2_CONFIG_SECRET_KEY" | jq -r '.SecretString' | jq -r '.enclave_cpu_count')} - MEMORY_MB=${MEMORY_MB:-$(aws secretsmanager get-secret-value --secret-id "$UID2_CONFIG_SECRET_KEY" | jq -r '.SecretString' | jq -r '.enclave_memory_mb')} - else - echo "Applying default CPU/Mem allocation..." - CPU_COUNT=6 - MEMORY_MB=24576 - fi - shopt -u nocasematch -} - - -function update_allocation() { - ALLOCATOR_YAML=/etc/nitro_enclaves/allocator.yaml - if [ -z "$CPU_COUNT" ] || [ -z "$MEMORY_MB" ]; then - echo 'No CPU_COUNT or MEMORY_MB set, cannot start enclave' - exit 1 - fi - echo "updating allocator: CPU_COUNT=$CPU_COUNT, MEMORY_MB=$MEMORY_MB..." - systemctl stop nitro-enclaves-allocator.service - sed -r "s/^(\s*memory_mib\s*:\s*).*/\1$MEMORY_MB/" -i $ALLOCATOR_YAML - sed -r "s/^(\s*cpu_count\s*:\s*).*/\1$CPU_COUNT/" -i $ALLOCATOR_YAML - systemctl start nitro-enclaves-allocator.service && systemctl enable nitro-enclaves-allocator.service - echo "nitro-enclaves-allocator restarted" -} - -function setup_vsockproxy() { - VSOCK_PROXY=${VSOCK_PROXY:-/usr/bin/vsockpx} - VSOCK_CONFIG=${VSOCK_CONFIG:-/etc/uid2operator/proxy.yaml} - VSOCK_THREADS=${VSOCK_THREADS:-$(( ( $(nproc) + 1 ) / 2 )) } - VSOCK_LOG_LEVEL=${VSOCK_LOG_LEVEL:-3} - echo "starting vsock proxy at $VSOCK_PROXY with $VSOCK_THREADS worker threads..." - $VSOCK_PROXY -c $VSOCK_CONFIG --workers $VSOCK_THREADS --log-level $VSOCK_LOG_LEVEL --daemon - echo "vsock proxy now running in background." -} - -function setup_dante() { - sockd -D -} - -function run_config_server() { - mkdir -p /etc/secret/secret-value - { - set +x; # Disable tracing within this block - 2>/dev/null; - SECRET_JSON=$(aws secretsmanager get-secret-value --secret-id "$UID2_CONFIG_SECRET_KEY" | jq -r '.SecretString') - echo "${SECRET_JSON}" > /etc/secret/secret-value/config; - } - echo $(jq ".core_base_url = \"$CORE_BASE_URL\"" /etc/secret/secret-value/config) > /etc/secret/secret-value/config - echo $(jq ".optout_base_url = \"$OPTOUT_BASE_URL\"" /etc/secret/secret-value/config) > /etc/secret/secret-value/config - echo "run_config_server" - cd /opt/uid2operator/config-server - ./bin/flask run --host 127.0.0.1 --port 27015 & -} - -function run_enclave() { - echo "starting enclave..." - nitro-cli run-enclave --eif-path $EIF_PATH --memory $MEMORY_MB --cpu-count $CPU_COUNT --enclave-cid $CID --enclave-name uid2operator -} - -terminate_old_enclave -config_aws -read_allocation -# update_allocation -setup_vsockproxy -setup_dante -run_config_server -run_enclave - -echo "Done!" diff --git a/scripts/aws/stop.sh b/scripts/aws/stop.sh deleted file mode 100644 index c37bdc729..000000000 --- a/scripts/aws/stop.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -function terminate_old_enclave() { - echo "Terminating Enclave..." - ENCLAVE_ID=$(nitro-cli describe-enclaves | jq -r ".[0].EnclaveID") - if [ "$ENCLAVE_ID" != "null" ]; then - nitro-cli terminate-enclave --enclave-id $ENCLAVE_ID - else - echo "no running enclaves to terminate" - fi -} - -function kill_process() { - echo "Shutting down $1..." - pid=$(pidof $1) - if [ -z "$pid" ]; then - echo "process $1 not found" - else - kill -9 $pid - echo "$1 exited" - fi -} - -terminate_old_enclave -kill_process vsockpx -kill_process sockd -# we start aws vsock-proxy via nohup -kill_process vsock-proxy -kill_process nohup - -echo "Done!" diff --git a/scripts/aws/uid2-operator-ami/ansible/playbook.yml b/scripts/aws/uid2-operator-ami/ansible/playbook.yml index 84c6c6f14..85f20e3ac 100644 --- a/scripts/aws/uid2-operator-ami/ansible/playbook.yml +++ b/scripts/aws/uid2-operator-ami/ansible/playbook.yml @@ -70,27 +70,34 @@ requirements: /opt/uid2operator/config-server/requirements.txt virtualenv_command: 'python3 -m venv' + - name: Install requirements.txt for enclave init + ansible.builtin.copy: + src: /tmp/artifacts/requirements.txt + dest: /opt/uid2operator/requirements.txt + remote_src: yes + - name: Install starter script ansible.builtin.copy: - src: /tmp/artifacts/start.sh - dest: /opt/uid2operator/start.sh + src: /tmp/artifacts/ec2.py + dest: /opt/uid2operator/ec2.py remote_src: yes - name: Make starter script executable ansible.builtin.file: - path: /opt/uid2operator/start.sh + path: /opt/uid2operator/ec2.py mode: '0755' - - name: Install stopper script + - name: Copy confidential_compute script ansible.builtin.copy: - src: /tmp/artifacts/stop.sh - dest: /opt/uid2operator/stop.sh + src: /tmp/artifacts/confidential_compute.py + dest: /opt/uid2operator/confidential_compute.py remote_src: yes - - name: Make starter script executable - ansible.builtin.file: - path: /opt/uid2operator/stop.sh - mode: '0755' + - name: Create virtualenv for eif init + ansible.builtin.pip: + virtualenv: /opt/uid2operator/init + requirements: /opt/uid2operator/requirements.txt + virtualenv_command: 'python3.11 -m venv' - name: Install Operator EIF ansible.builtin.copy: diff --git a/scripts/aws/uid2operator.service b/scripts/aws/uid2operator.service index 1d36b7a91..56559e3c2 100644 --- a/scripts/aws/uid2operator.service +++ b/scripts/aws/uid2operator.service @@ -8,8 +8,8 @@ RemainAfterExit=true StandardOutput=journal StandardError=journal SyslogIdentifier=uid2operator -ExecStart=/opt/uid2operator/start.sh -ExecStop=/opt/uid2operator/stop.sh +ExecStart=/opt/uid2operator/init/bin/python /opt/uid2operator/ec2.py +ExecStop=/opt/uid2operator/init/bin/python /opt/uid2operator/ec2.py -o stop [Install] -WantedBy=multi-user.target \ No newline at end of file +WantedBy=multi-user.target diff --git a/scripts/confidential_compute.py b/scripts/confidential_compute.py new file mode 100644 index 000000000..306474e9c --- /dev/null +++ b/scripts/confidential_compute.py @@ -0,0 +1,144 @@ +import requests +import re +import socket +from urllib.parse import urlparse +from abc import ABC, abstractmethod +from typing import TypedDict, NotRequired, get_type_hints +import subprocess + +class ConfidentialComputeConfig(TypedDict): + api_token: str + core_base_url: str + optout_base_url: str + environment: str + skip_validations: NotRequired[bool] + debug_mode: NotRequired[bool] + +class ConfidentialCompute(ABC): + + def __init__(self): + self.configs: ConfidentialComputeConfig = {} + + def validate_configuration(self): + """ Validates the paramters specified through configs/secret manager .""" + print("Validating configurations provided") + def validate_operator_key(): + """ Validates the operator key format and its environment alignment.""" + operator_key = self.configs.get("api_token") + if not operator_key: + raise ValueError("API token is missing from the configuration.") + pattern = r"^(UID2|EUID)-.\-(I|P|L)-\d+-.*$" + if re.match(pattern, operator_key): + env = self.configs.get("environment", "").lower() + debug_mode = self.configs.get("debug_mode", False) + expected_env = "I" if debug_mode or env == "integ" else "P" + + if operator_key.split("-")[2] != expected_env: + raise ValueError( + f"Operator key does not match the expected environment ({expected_env})." + ) + print("Validated operator key matches environment") + else: + print("Skipping operator key validation") + + def validate_url(url_key, environment): + """URL should include environment except in prod""" + if environment != "prod" and environment not in self.configs[url_key]: + raise ValueError( + f"{url_key} must match the environment. Ensure the URL includes '{environment}'." + ) + parsed_url = urlparse(self.configs[url_key]) + if parsed_url.scheme != 'https' and parsed_url.path: + raise ValueError( + f"{url_key} is invalid. Ensure {self.configs[url_key]} follows HTTPS, and doesn't have any path specified." + ) + print(f"Validated {self.configs[url_key]} matches other config parameters") + + + def validate_connectivity() -> None: + """ Validates that the core URL is accessible.""" + try: + core_url = self.configs["core_base_url"] + core_ip = socket.gethostbyname(urlparse(core_url).netloc) + requests.get(core_url, timeout=5) + print(f"Validated connectivity to {core_url}") + except (requests.ConnectionError, requests.Timeout) as e: + raise RuntimeError( + f"Failed to reach required URLs. Consider enabling {core_ip} in the egress firewall." + ) + except Exception as e: + raise Exception("Failed to reach the URLs.") from e + type_hints = get_type_hints(ConfidentialComputeConfig, include_extras=True) + required_keys = [field for field, hint in type_hints.items() if "NotRequired" not in str(hint)] + missing_keys = [key for key in required_keys if key not in self.configs] + if missing_keys: + raise MissingConfigError(missing_keys) + + environment = self.configs["environment"] + + if environment not in ["integ", "prod"]: + raise ValueError("Environment must be either prod/integ. It is currently set to", environment) + + if self.configs.get("debug_mode") and environment == "prod": + raise ValueError("Debug mode cannot be enabled in the production environment.") + + validate_url("core_base_url", environment) + validate_url("optout_base_url", environment) + validate_operator_key() + validate_connectivity() + print("Completed static validation of confidential compute config values") + + + @abstractmethod + def _get_secret(self, secret_identifier: str) -> ConfidentialComputeConfig: + """ + Fetches the secret from a secret store. + + Raises: + SecretNotFoundException: If the secret is not found. + """ + pass + + @abstractmethod + def _setup_auxiliaries(self) -> None: + """ Sets up auxiliary processes required for confidential computing. """ + pass + + @abstractmethod + def _validate_auxiliaries(self) -> None: + """ Validates auxiliary services are running.""" + pass + + @abstractmethod + def run_compute(self) -> None: + """ Runs confidential computing.""" + pass + + @staticmethod + def run_command(command, seperate_process=False): + print(f"Running command: {' '.join(command)}") + try: + if seperate_process: + subprocess.Popen(command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + else: + subprocess.run(command,check=True) + except Exception as e: + print(f"Failed to run command: {str(e)}") + raise RuntimeError (f"Failed to start {' '.join(command)} ") + +class ConfidentialComputeStartupException(Exception): + def __init__(self, message): + super().__init__(message) + +class MissingConfigError(ConfidentialComputeStartupException): + """Custom exception to handle missing config keys.""" + def __init__(self, missing_keys): + self.missing_keys = missing_keys + self.message = f"\n Missing configuration keys: {', '.join(missing_keys)} \n" + super().__init__(self.message) + +class SecretNotFoundException(ConfidentialComputeStartupException): + """Custom exception if secret manager is not found""" + def __init__(self, name): + self.message = f"Secret manager not found - {name}. Please check if secret exist and the Instance Profile has permission to read it" + super().__init__(self.message) From b1d093de6d42af04869b85da61325fdb47334e86 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Thu, 26 Dec 2024 22:35:30 +0000 Subject: [PATCH 188/431] [CI Pipeline] Released Minor version: 5.44.0 --- pom.xml | 2 +- version.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 7b231d3ce..ef9b19c29 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.43.11 + 5.44.0 UTF-8 diff --git a/version.json b/version.json index 61e9e46d1..0e3febcce 100644 --- a/version.json +++ b/version.json @@ -1 +1 @@ -{ "$schema": "https://raw.githubusercontent.com/dotnet/Nerdbank.GitVersioning/master/src/NerdBank.GitVersioning/version.schema.json", "version": "5.43", "publicReleaseRefSpec": [ "^refs/heads/master$", "^refs/heads/v\\d+(?:\\.\\d+)?$" ], "cloudBuild": { "setVersionVariables": true, "buildNumber": { "enabled": true, "includeCommitId": { "when": "always" } } } } +{ "$schema": "https://raw.githubusercontent.com/dotnet/Nerdbank.GitVersioning/master/src/NerdBank.GitVersioning/version.schema.json", "version": "5.44", "publicReleaseRefSpec": [ "^refs/heads/master$", "^refs/heads/v\\d+(?:\\.\\d+)?$" ], "cloudBuild": { "setVersionVariables": true, "buildNumber": { "enabled": true, "includeCommitId": { "when": "always" } } } } From 8aee5a37ba79698199bfd73b8c8e940962ab28ea Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Tue, 31 Dec 2024 16:30:29 +1100 Subject: [PATCH 189/431] Added ConfigRetrieverFactory to create vert.x ConfigRetriever --- .../service/ConfigRetrieverFactory.java | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java diff --git a/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java b/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java new file mode 100644 index 000000000..5b18cf3b1 --- /dev/null +++ b/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java @@ -0,0 +1,36 @@ +package com.uid2.operator.service; + +import com.uid2.operator.Const; +import io.vertx.config.ConfigRetriever; +import io.vertx.config.ConfigRetrieverOptions; +import io.vertx.config.ConfigStoreOptions; +import io.vertx.core.Vertx; +import io.vertx.core.json.JsonObject; + +import static com.uid2.operator.Const.Config.ConfigScanPeriodMs; +import static com.uid2.operator.Const.Config.CoreConfigPath; + +public class ConfigRetrieverFactory { + public ConfigRetriever create(Vertx vertx, JsonObject bootstrapConfig) { + String configPath = bootstrapConfig.getString(CoreConfigPath); + + + ConfigStoreOptions httpStore = new ConfigStoreOptions() + .setType("http") + .setConfig(new JsonObject() + .put("host", "127.0.0.1") + .put("port", Const.Port.ServicePortForCore) + .put("path", configPath)); + + ConfigStoreOptions bootstrapStore = new ConfigStoreOptions() + .setType("json") + .setConfig(bootstrapConfig); + + ConfigRetrieverOptions retrieverOptions = new ConfigRetrieverOptions() + .setScanPeriod(bootstrapConfig.getLong(ConfigScanPeriodMs)) + .addStore(bootstrapStore) + .addStore(httpStore); + + return ConfigRetriever.create(vertx, retrieverOptions); + } +} From 8f40ba2a9b2e88733a2d2e92d5a6c75ccba37646 Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Tue, 31 Dec 2024 16:33:29 +1100 Subject: [PATCH 190/431] Updated ConfigService, Main ConfigService returns a Future when created and takes ConfigRetriever in "create" method "run()" method in Main waits for ConfigService Future to complete before creating UIDOperatorVerticle --- src/main/java/com/uid2/operator/Main.java | 71 ++++++++++--------- .../uid2/operator/service/ConfigService.java | 69 +++++------------- 2 files changed, 57 insertions(+), 83 deletions(-) diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index 7dbf64153..589f64bd3 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -8,9 +8,7 @@ import com.uid2.operator.monitoring.IStatsCollectorQueue; import com.uid2.operator.monitoring.OperatorMetrics; import com.uid2.operator.monitoring.StatsCollectorVerticle; -import com.uid2.operator.service.ConfigService; -import com.uid2.operator.service.SecureLinkValidatorService; -import com.uid2.operator.service.ShutdownService; +import com.uid2.operator.service.*; import com.uid2.operator.vertx.Endpoints; import com.uid2.operator.vertx.OperatorShutdownHandler; import com.uid2.operator.store.CloudSyncOptOutStore; @@ -38,6 +36,7 @@ import io.micrometer.core.instrument.distribution.DistributionStatisticConfig; import io.micrometer.prometheus.PrometheusMeterRegistry; import io.micrometer.prometheus.PrometheusRenameFilter; +import io.vertx.config.ConfigRetriever; import io.vertx.core.*; import io.vertx.core.http.HttpServerOptions; import io.vertx.core.http.impl.HttpUtils; @@ -266,41 +265,49 @@ private ICloudStorage wrapCloudStorageForOptOut(ICloudStorage cloudStorage) { } private void run() throws Exception { - ConfigService configService = ConfigService.getInstance(vertx, config); + ConfigRetrieverFactory configRetrieverFactory = new ConfigRetrieverFactory(); + ConfigRetriever configRetriever = configRetrieverFactory.create(vertx, config); - Supplier operatorVerticleSupplier = () -> { - UIDOperatorVerticle verticle = new UIDOperatorVerticle(configService, this.clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, getKeyManager(), saltProvider, optOutStore, Clock.systemUTC(), _statsCollectorQueue, new SecureLinkValidatorService(this.serviceLinkProvider, this.serviceProvider), this.shutdownHandler::handleSaltRetrievalResponse); - return verticle; - }; + ConfigService.create(configRetriever).compose(configService -> { - DeploymentOptions options = new DeploymentOptions(); - int svcInstances = this.config.getInteger(Const.Config.ServiceInstancesProp); - options.setInstances(svcInstances); + Supplier operatorVerticleSupplier = () -> { + UIDOperatorVerticle verticle = new UIDOperatorVerticle(configService, this.clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, getKeyManager(), saltProvider, optOutStore, Clock.systemUTC(), _statsCollectorQueue, new SecureLinkValidatorService(this.serviceLinkProvider, this.serviceProvider), this.shutdownHandler::handleSaltRetrievalResponse); + return verticle; + }; - Promise compositePromise = Promise.promise(); - List fs = new ArrayList<>(); - fs.add(createAndDeployStatsCollector()); - fs.add(createStoreVerticles()); + DeploymentOptions options = new DeploymentOptions(); + int svcInstances = this.config.getInteger(Const.Config.ServiceInstancesProp); + options.setInstances(svcInstances); - CompositeFuture.all(fs).onComplete(ar -> { - if (ar.failed()) compositePromise.fail(new Exception(ar.cause())); - else compositePromise.complete(); - }); + Promise compositePromise = Promise.promise(); + List fs = new ArrayList<>(); + fs.add(createAndDeployStatsCollector()); + try { + fs.add(createStoreVerticles()); + } catch (Exception e) { + throw new RuntimeException(e); + } - compositePromise.future() - .compose(v -> { - metrics.setup(); - vertx.setPeriodic(60000, id -> metrics.update()); - - Promise promise = Promise.promise(); - vertx.deployVerticle(operatorVerticleSupplier, options, promise); - return promise.future(); - }) - .onFailure(t -> { - LOGGER.error("Failed to bootstrap operator: " + t.getMessage(), new Exception(t)); - vertx.close(); - System.exit(1); + CompositeFuture.all(fs).onComplete(ar -> { + if (ar.failed()) compositePromise.fail(new Exception(ar.cause())); + else compositePromise.complete(); }); + + return compositePromise.future() + .compose(v -> { + metrics.setup(); + vertx.setPeriodic(60000, id -> metrics.update()); + + Promise promise = Promise.promise(); + vertx.deployVerticle(operatorVerticleSupplier, options, promise); + return promise.future(); + }) + .onFailure(t -> { + LOGGER.error("Failed to bootstrap operator: " + t.getMessage(), new Exception(t)); + vertx.close(); + System.exit(1); + }); + }); } private Future createStoreVerticles() throws Exception { diff --git a/src/main/java/com/uid2/operator/service/ConfigService.java b/src/main/java/com/uid2/operator/service/ConfigService.java index 44de433d3..c58e59fc7 100644 --- a/src/main/java/com/uid2/operator/service/ConfigService.java +++ b/src/main/java/com/uid2/operator/service/ConfigService.java @@ -2,80 +2,47 @@ import com.uid2.operator.Const; import io.vertx.config.ConfigRetriever; -import io.vertx.config.ConfigRetrieverOptions; -import io.vertx.config.ConfigStoreOptions; -import io.vertx.core.Vertx; +import io.vertx.core.Future; +import io.vertx.core.Promise; import io.vertx.core.json.JsonObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static com.uid2.operator.Const.Config.*; import static com.uid2.operator.service.ConfigValidatorUtil.*; import static com.uid2.operator.service.UIDOperatorService.*; public class ConfigService implements IConfigService { - private static volatile ConfigService instance; private ConfigRetriever configRetriever; private static final Logger logger = LoggerFactory.getLogger(ConfigService.class); - private ConfigService(Vertx vertx, JsonObject bootstrapConfig) { - this.initialiseConfigRetriever(vertx, bootstrapConfig); - } - - public static ConfigService getInstance(Vertx vertx, JsonObject bootstrapConfig) { - ConfigService configService = instance; - - if (configService == null) { - synchronized (ConfigService.class) { - configService = instance; - if (configService == null) { - instance = configService = new ConfigService(vertx, bootstrapConfig); - } - } - } - - return configService; - } - - @Override - public JsonObject getConfig() { - return configRetriever.getCachedConfig(); + private ConfigService(ConfigRetriever configRetriever) { + this.configRetriever = configRetriever; + this.configRetriever.setConfigurationProcessor(this::configValidationHandler); } - private void initialiseConfigRetriever(Vertx vertx, JsonObject bootstrapConfig) { - String configPath = bootstrapConfig.getString(CoreConfigPath); + public static Future create(ConfigRetriever configRetriever) { + Promise promise = Promise.promise(); + ConfigService instance = new ConfigService(configRetriever); - ConfigStoreOptions httpStore = new ConfigStoreOptions() - .setType("http") - .setConfig(new JsonObject() - .put("host", "127.0.0.1") - .put("port", Const.Port.ServicePortForCore) - .put("path", configPath)); - - ConfigStoreOptions bootstrapStore = new ConfigStoreOptions() - .setType("json") - .setConfig(bootstrapConfig); - - ConfigRetrieverOptions retrieverOptions = new ConfigRetrieverOptions() - .setScanPeriod(bootstrapConfig.getLong(ConfigScanPeriodMs)) - .addStore(bootstrapStore) - .addStore(httpStore); - - this.configRetriever = ConfigRetriever.create(vertx, retrieverOptions); - - this.configRetriever.setConfigurationProcessor(this::configValidationHandler); - - this.configRetriever.getConfig(ar -> { + configRetriever.getConfig(ar -> { if (ar.succeeded()) { System.out.println("Successfully loaded config"); + promise.complete(instance); } else { System.err.println("Failed to load config: " + ar.cause().getMessage()); - logger.error("Failed to load config"); + logger.error("Failed to load config{}", ar.cause().getMessage()); + promise.fail(ar.cause()); } }); + return promise.future(); + } + + @Override + public JsonObject getConfig() { + return configRetriever.getCachedConfig(); } private JsonObject configValidationHandler(JsonObject config) { From e884433654895d7ed287c240ebed2d9cbdd8d094 Mon Sep 17 00:00:00 2001 From: sophia chen Date: Thu, 2 Jan 2025 09:20:23 +1100 Subject: [PATCH 191/431] Refactored path normalisation filtering into shared --- src/main/java/com/uid2/operator/Main.java | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index a307e3695..cc71114db 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -25,6 +25,7 @@ import com.uid2.shared.store.RotatingSaltProvider; import com.uid2.shared.store.reader.*; import com.uid2.shared.store.scope.GlobalScope; +import com.uid2.shared.util.HTTPPathMetricFilter; import com.uid2.shared.vertx.CloudSyncVerticle; import com.uid2.shared.vertx.ICloudSync; import com.uid2.shared.vertx.RotatingStoreVerticle; @@ -427,14 +428,8 @@ private static void setupMetrics(MicrometerMetricsOptions metricOptions) { prometheusRegistry.config() // providing common renaming for prometheus metric, e.g. "hello.world" to "hello_world" .meterFilter(new PrometheusRenameFilter()) - .meterFilter(MeterFilter.replaceTagValues(Label.HTTP_PATH.toString(), actualPath -> { - try { - String normalized = HttpUtils.normalizePath(actualPath).split("\\?")[0]; - return Endpoints.pathSet().contains(normalized) ? normalized : "/unknown"; - } catch (IllegalArgumentException e) { - return actualPath; - } - })) + .meterFilter(MeterFilter.replaceTagValues(Label.HTTP_PATH.toString(), + actualPath -> HTTPPathMetricFilter.filterPath(actualPath, Endpoints.pathSet()))) // Don't record metrics for 404s. .meterFilter(MeterFilter.deny(id -> id.getName().startsWith(MetricsDomain.HTTP_SERVER.getPrefix()) && From 4147b2dfcbff2aefe199a741df0c55a09a49d158 Mon Sep 17 00:00:00 2001 From: sophia chen Date: Thu, 2 Jan 2025 13:23:27 +1100 Subject: [PATCH 192/431] Updated pom.xml --- pom.xml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ef9b19c29..657ccf3f8 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ 2.1.0 2.1.0 2.1.0 - 8.0.9 + 8.0.29 ${project.version} 21 21 @@ -206,6 +206,12 @@ 4.3.2 test + + com.uid2 + uid2-shared + 8.0.25-DEV + compile + From 232b4a4d4c3781ebfcbb1b613ac3f9ca722a0e51 Mon Sep 17 00:00:00 2001 From: sophia-chen-ttd <152837545+sophia-chen-ttd@users.noreply.github.com> Date: Thu, 2 Jan 2025 13:27:30 +1100 Subject: [PATCH 193/431] Update pom.xml --- pom.xml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pom.xml b/pom.xml index 657ccf3f8..a57034855 100644 --- a/pom.xml +++ b/pom.xml @@ -206,12 +206,6 @@ 4.3.2 test - - com.uid2 - uid2-shared - 8.0.25-DEV - compile - From c1fd6dfbc470f819a60984135fa2ef0e165c0953 Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Thu, 2 Jan 2025 13:32:36 +1100 Subject: [PATCH 194/431] Updated ConfigServiceTest Removed use of reflection to access private attributes/methods Removed unnecessary tests Updated remaining tests to inject ConfigRetriever into ConfigService --- .../com/uid2/operator/ConfigServiceTest.java | 144 +++++++----------- 1 file changed, 55 insertions(+), 89 deletions(-) diff --git a/src/test/java/com/uid2/operator/ConfigServiceTest.java b/src/test/java/com/uid2/operator/ConfigServiceTest.java index e41954ee5..f00b2e900 100644 --- a/src/test/java/com/uid2/operator/ConfigServiceTest.java +++ b/src/test/java/com/uid2/operator/ConfigServiceTest.java @@ -1,7 +1,9 @@ package com.uid2.operator; +import com.uid2.operator.service.ConfigRetrieverFactory; import com.uid2.operator.service.ConfigService; import io.vertx.core.Vertx; +import io.vertx.core.http.HttpServer; import io.vertx.core.json.JsonObject; import io.vertx.config.ConfigRetriever; import io.vertx.ext.web.Router; @@ -11,9 +13,6 @@ import io.vertx.junit5.VertxTestContext; import org.junit.jupiter.api.extension.ExtendWith; -import java.lang.reflect.*; -import java.util.concurrent.TimeUnit; - import static com.uid2.operator.Const.Config.*; import static com.uid2.operator.service.UIDOperatorService.*; import static org.junit.jupiter.api.Assertions.*; @@ -23,6 +22,8 @@ class ConfigServiceTest { private Vertx vertx; private JsonObject bootstrapConfig; + private ConfigRetriever configRetriever; + private HttpServer server; @BeforeEach void setUp() { @@ -34,6 +35,9 @@ void setUp() { .put(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, 7200) .put(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS, 1800) .put(MaxBidstreamLifetimeSecondsProp, 7200); + + ConfigRetrieverFactory configRetrieverFactory = new ConfigRetrieverFactory(); + configRetriever = configRetrieverFactory.create(vertx, bootstrapConfig); } @AfterEach @@ -41,121 +45,83 @@ void tearDown() { vertx.close(); } - void startMockServer(VertxTestContext testContext, JsonObject config) throws InterruptedException { + void startMockServer(VertxTestContext testContext, JsonObject config) { + if (server != null) { + server.close(); + } + Router router = Router.router(vertx); router.route().handler(BodyHandler.create()); router.get("/config").handler(ctx -> ctx.response() .putHeader("content-type", "application/json") .end(config.encode())); - vertx.createHttpServer() + server = vertx.createHttpServer() .requestHandler(router) .listen(Const.Port.ServicePortForCore,"127.0.0.1", http -> { if (!http.succeeded()) { testContext.failNow(http.cause()); } }); - - testContext.awaitCompletion(5, TimeUnit.SECONDS); - } - - @Test - void testSingletonBehavior() { - ConfigService instance1 = ConfigService.getInstance(vertx, bootstrapConfig); - ConfigService instance2 = ConfigService.getInstance(vertx, bootstrapConfig); - assertSame(instance1, instance2, "getInstance should return the same instance"); - } - - @Test - void testGetConfig() { - ConfigRetriever mockConfigRetriever = mock(ConfigRetriever.class); - JsonObject cachedConfig = new JsonObject().put("key", "value"); - when(mockConfigRetriever.getCachedConfig()).thenReturn(cachedConfig); - ConfigService configService = ConfigService.getInstance(vertx, bootstrapConfig); - // Reflection to inject mocked ConfigRetriever - try { - Field configRetrieverField = ConfigService.class.getDeclaredField("configRetriever"); - configRetrieverField.setAccessible(true); - configRetrieverField.set(configService, mockConfigRetriever); - } catch (Exception e) { - fail("Failed to inject mock ConfigRetriever: " + e.getMessage()); - } - JsonObject result = configService.getConfig(); - assertEquals(cachedConfig, result, "getConfig should return the cached configuration"); } @Test - void testInitialiseConfigRetriever(VertxTestContext testContext) throws InterruptedException { + void testGetConfig(VertxTestContext testContext) { JsonObject httpStoreConfig = new JsonObject().put("http", "value"); this.startMockServer(testContext, httpStoreConfig); - ConfigService configService = ConfigService.getInstance(vertx, bootstrapConfig); - // Wait for the initialisation to finish - alternative is to have getInstance return a future - testContext.awaitCompletion(1, TimeUnit.SECONDS); - JsonObject retrievedConfig = configService.getConfig(); - assertNotNull(retrievedConfig, "Config retriever should initialise without error"); - assertTrue(retrievedConfig.fieldNames().containsAll(bootstrapConfig.fieldNames()), "Retrieved config should contain all keys in bootstrap config"); - assertTrue(retrievedConfig.fieldNames().containsAll(httpStoreConfig.fieldNames()), "Retrieved config should contain all keys in http store config"); - testContext.completeNow(); + ConfigService.create(configRetriever).onComplete(ar -> { + if (ar.succeeded()) { + ConfigService configService = ar.result(); + JsonObject retrievedConfig = configService.getConfig(); + assertNotNull(retrievedConfig, "Config retriever should initialise without error"); + assertTrue(retrievedConfig.fieldNames().containsAll(bootstrapConfig.fieldNames()), "Retrieved config should contain all keys in bootstrap config"); + assertTrue(retrievedConfig.fieldNames().containsAll(httpStoreConfig.fieldNames()), "Retrieved config should contain all keys in http store config"); + testContext.completeNow(); + } else { + testContext.failNow(ar.cause()); + } + }); + } @Test - void testInvalidConfigRevertsToPrevious() { - ConfigRetriever mockConfigRetriever = mock(ConfigRetriever.class); + void testInvalidConfigRevertsToPrevious(VertxTestContext testContext) { JsonObject lastConfig = new JsonObject().put("previous", "config"); + ConfigRetriever spyRetriever = spy(configRetriever); + when(spyRetriever.getCachedConfig()).thenReturn(lastConfig); JsonObject invalidConfig = new JsonObject() .put(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, 1000) .put(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, 2000); - when(mockConfigRetriever.getCachedConfig()).thenReturn(lastConfig); - ConfigService configService = ConfigService.getInstance(vertx, bootstrapConfig); - try { - Field configRetrieverField = ConfigService.class.getDeclaredField("configRetriever"); - configRetrieverField.setAccessible(true); - configRetrieverField.set(configService, mockConfigRetriever); - } catch (Exception e) { - fail("Failed to inject mock ConfigRetriever: " + e.getMessage()); - } - - try { - Method configValidationHandlerMethod = ConfigService.class.getDeclaredMethod("configValidationHandler", JsonObject.class); - configValidationHandlerMethod.setAccessible(true); - JsonObject validatedConfig = (JsonObject) configValidationHandlerMethod.invoke(configService, invalidConfig); - assertEquals(lastConfig, validatedConfig, "Invalid config not reverted to previous config"); - } catch (Exception e) { - fail("Failed to access and invoke the configValidationHandler method: " + e.getMessage()); - } + this.startMockServer(testContext, invalidConfig); + ConfigService.create(spyRetriever).onComplete(ar -> { + if (ar.succeeded()) { + reset(spyRetriever); + ConfigService configService = ar.result(); + assertEquals(lastConfig, configService.getConfig(), "Invalid config not reverted to previous config"); + testContext.completeNow(); + } + else { + testContext.failNow(ar.cause()); + } + }); } @Test - void testFirstInvalidConfigThrowsRuntimeException() { - ConfigRetriever mockConfigRetriever = mock(ConfigRetriever.class); + void testFirstInvalidConfigThrowsRuntimeException(VertxTestContext testContext) { JsonObject invalidConfig = new JsonObject() .put(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, 1000) .put(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, 2000); - when(mockConfigRetriever.getCachedConfig()).thenReturn(null); - ConfigService configService = ConfigService.getInstance(vertx, bootstrapConfig); - try { - Field configRetrieverField = ConfigService.class.getDeclaredField("configRetriever"); - configRetrieverField.setAccessible(true); - configRetrieverField.set(configService, mockConfigRetriever); - } catch (Exception e) { - fail("Failed to inject mock ConfigRetriever: " + e.getMessage()); - } - - - - try { - Method configValidationHandlerMethod = ConfigService.class.getDeclaredMethod("configValidationHandler", JsonObject.class); - configValidationHandlerMethod.setAccessible(true); - assertThrows(RuntimeException.class, () -> { - try { - configValidationHandlerMethod.invoke(configService, invalidConfig); - } catch (InvocationTargetException e) { - // Throw cause as InvocationTargetException wraps actual exception thrown by configValidationHandler - throw e.getCause(); - } - }); - } catch (Exception e) { - fail("Failed to access and invoke the configValidationHandler method: " + e.getMessage()); - } + this.startMockServer(testContext, invalidConfig); + ConfigService.create(configRetriever).onComplete(ar -> { + if (ar.succeeded()) { + testContext.failNow(new RuntimeException("Expected a RuntimeException but the creation succeeded")); + } + else { + assertThrows(RuntimeException.class, () -> { + throw ar.cause(); + }); + testContext.completeNow(); + } + }); } } \ No newline at end of file From ab3005459319c09253a7fcd1804187207f66f58c Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Thu, 2 Jan 2025 14:13:40 +1100 Subject: [PATCH 195/431] Updated ConfigRetrieverFactory Set http store to optional --- .../java/com/uid2/operator/service/ConfigRetrieverFactory.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java b/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java index 5b18cf3b1..a35b63416 100644 --- a/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java +++ b/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java @@ -17,6 +17,7 @@ public ConfigRetriever create(Vertx vertx, JsonObject bootstrapConfig) { ConfigStoreOptions httpStore = new ConfigStoreOptions() .setType("http") + .setOptional(true) .setConfig(new JsonObject() .put("host", "127.0.0.1") .put("port", Const.Port.ServicePortForCore) From 7d2a16dd979dad327003d74c4fe5f0621481276c Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Thu, 2 Jan 2025 15:09:50 +1100 Subject: [PATCH 196/431] Updated ConfigService Added default value for max_bidstream_lifetime_seconds --- src/main/java/com/uid2/operator/service/ConfigService.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/uid2/operator/service/ConfigService.java b/src/main/java/com/uid2/operator/service/ConfigService.java index c58e59fc7..cd5b8b9bc 100644 --- a/src/main/java/com/uid2/operator/service/ConfigService.java +++ b/src/main/java/com/uid2/operator/service/ConfigService.java @@ -13,7 +13,7 @@ public class ConfigService implements IConfigService { - private ConfigRetriever configRetriever; + private final ConfigRetriever configRetriever; private static final Logger logger = LoggerFactory.getLogger(ConfigService.class); private ConfigService(ConfigRetriever configRetriever) { @@ -32,7 +32,7 @@ public static Future create(ConfigRetriever configRetriever) { promise.complete(instance); } else { System.err.println("Failed to load config: " + ar.cause().getMessage()); - logger.error("Failed to load config{}", ar.cause().getMessage()); + logger.error("Failed to load config: {}", ar.cause().getMessage()); promise.fail(ar.cause()); } }); @@ -50,7 +50,7 @@ private JsonObject configValidationHandler(JsonObject config) { Integer identityExpiresAfter = config.getInteger(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS); Integer refreshExpiresAfter = config.getInteger(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS); Integer refreshIdentityAfter = config.getInteger(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS); - Integer maxBidstreamLifetimeSeconds = config.getInteger(Const.Config.MaxBidstreamLifetimeSecondsProp); + Integer maxBidstreamLifetimeSeconds = config.getInteger(Const.Config.MaxBidstreamLifetimeSecondsProp, identityExpiresAfter); isValid &= validateIdentityRefreshTokens(identityExpiresAfter, refreshExpiresAfter, refreshIdentityAfter); From 67b05b182e718217816774ca4a87c3af00442e03 Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Thu, 2 Jan 2025 16:34:12 +1100 Subject: [PATCH 197/431] Added ConfigServiceManager, StaticConfigService ConfigServiceManager handles switching between dynamic and static config StaticConfigService implements IConfigService, serving config from static files --- .../service/ConfigServiceManager.java | 58 +++++++++++++++++++ .../operator/service/StaticConfigService.java | 17 ++++++ 2 files changed, 75 insertions(+) create mode 100644 src/main/java/com/uid2/operator/service/ConfigServiceManager.java create mode 100644 src/main/java/com/uid2/operator/service/StaticConfigService.java diff --git a/src/main/java/com/uid2/operator/service/ConfigServiceManager.java b/src/main/java/com/uid2/operator/service/ConfigServiceManager.java new file mode 100644 index 000000000..fc69c79fc --- /dev/null +++ b/src/main/java/com/uid2/operator/service/ConfigServiceManager.java @@ -0,0 +1,58 @@ +package com.uid2.operator.service; + +import io.vertx.config.ConfigRetriever; +import io.vertx.core.Future; +import io.vertx.core.Promise; +import io.vertx.core.Vertx; +import io.vertx.core.json.JsonObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ConfigServiceManager { + private IConfigService currentConfigService; + private final ConfigService dynamicConfigService; + private final StaticConfigService staticConfigService; + private static final Logger logger = LoggerFactory.getLogger(ConfigServiceManager.class); + + private ConfigServiceManager(ConfigService dynamicConfigService, StaticConfigService staticConfigService, boolean useDynamicConfig) { + this.dynamicConfigService = dynamicConfigService; + this.staticConfigService = staticConfigService; + this.currentConfigService = useDynamicConfig ? dynamicConfigService : staticConfigService; + } + + public Future create(Vertx vertx, JsonObject bootstrapConfig, boolean useDynamicConfig) { + Promise promise = Promise.promise(); + + StaticConfigService staticConfigService = new StaticConfigService(bootstrapConfig); + + ConfigRetrieverFactory configRetrieverFactory = new ConfigRetrieverFactory(); + ConfigRetriever configRetriever = configRetrieverFactory.create(vertx, bootstrapConfig); + + ConfigService.create(configRetriever).onComplete(ar -> { + if (ar.succeeded()) { + ConfigService dynamicConfigService = ar.result(); + ConfigServiceManager instance = new ConfigServiceManager(dynamicConfigService, staticConfigService, useDynamicConfig); + promise.complete(instance); + } + else { + promise.fail(ar.cause()); + } + }); + + return promise.future(); + } + + public void updateConfigService(boolean useDynamicConfig) { + if (useDynamicConfig) { + logger.info("Switching to DynamicConfigService"); + this.currentConfigService = dynamicConfigService; + } else { + logger.info("Switching to StaticConfigService"); + this.currentConfigService = staticConfigService; + } + } + + public IConfigService getConfigService() { + return currentConfigService; + } +} diff --git a/src/main/java/com/uid2/operator/service/StaticConfigService.java b/src/main/java/com/uid2/operator/service/StaticConfigService.java new file mode 100644 index 000000000..e44440fc9 --- /dev/null +++ b/src/main/java/com/uid2/operator/service/StaticConfigService.java @@ -0,0 +1,17 @@ +package com.uid2.operator.service; + +import io.vertx.core.json.JsonObject; + + +public class StaticConfigService implements IConfigService { + private final JsonObject staticConfig; + + public StaticConfigService(JsonObject staticConfig) { + this.staticConfig = staticConfig; + } + + @Override + public JsonObject getConfig() { + return staticConfig; + } +} From 25ad265fa9035b035e426e2e3cfca91a32217898 Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Thu, 2 Jan 2025 17:40:48 +1100 Subject: [PATCH 198/431] Update UIDOperatorVerticle, Main Main creates new ConfigServiceManager instance and injects it into UIDOperatorVerticle Also Added feature flag config value to local-config.json, Made create method static in ConfigServiceManager --- conf/local-config.json | 3 ++- src/main/java/com/uid2/operator/Const.java | 1 + src/main/java/com/uid2/operator/Main.java | 7 ++----- .../com/uid2/operator/service/ConfigServiceManager.java | 2 +- .../java/com/uid2/operator/vertx/UIDOperatorVerticle.java | 6 ++++-- 5 files changed, 10 insertions(+), 9 deletions(-) diff --git a/conf/local-config.json b/conf/local-config.json index 6a357dba1..c73940613 100644 --- a/conf/local-config.json +++ b/conf/local-config.json @@ -36,5 +36,6 @@ "key_sharing_endpoint_provide_app_names": true, "client_side_token_generate_log_invalid_http_origins": true, "salts_expired_shutdown_hours": 12, - "operator_type": "public" + "operator_type": "public", + "remote_config_feature_flag": true } diff --git a/src/main/java/com/uid2/operator/Const.java b/src/main/java/com/uid2/operator/Const.java index 2b70c1cb0..b1c7175b0 100644 --- a/src/main/java/com/uid2/operator/Const.java +++ b/src/main/java/com/uid2/operator/Const.java @@ -34,5 +34,6 @@ public class Config extends com.uid2.shared.Const.Config { public static final String ConfigScanPeriodMs = "config_scan_period_ms"; public static final String Config = "config"; public static final String identityV3 = "identity_v3"; + public static final String RemoteConfigFeatureFlag = "remote_config_feature_flag"; } } diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index 85fcdef9c..5ef898c85 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -268,12 +268,9 @@ private void run() throws Exception { this.createVertxInstancesMetric(); this.createVertxEventLoopsMetric(); - ConfigRetrieverFactory configRetrieverFactory = new ConfigRetrieverFactory(); - ConfigRetriever configRetriever = configRetrieverFactory.create(vertx, config); - - ConfigService.create(configRetriever).compose(configService -> { + ConfigServiceManager.create(vertx, config, config.getBoolean(Const.Config.RemoteConfigFeatureFlag, true)).compose(configServiceManager -> { Supplier operatorVerticleSupplier = () -> { - UIDOperatorVerticle verticle = new UIDOperatorVerticle(configService, this.clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, getKeyManager(), saltProvider, optOutStore, Clock.systemUTC(), _statsCollectorQueue, new SecureLinkValidatorService(this.serviceLinkProvider, this.serviceProvider), this.shutdownHandler::handleSaltRetrievalResponse); + UIDOperatorVerticle verticle = new UIDOperatorVerticle(configServiceManager, this.clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, getKeyManager(), saltProvider, optOutStore, Clock.systemUTC(), _statsCollectorQueue, new SecureLinkValidatorService(this.serviceLinkProvider, this.serviceProvider), this.shutdownHandler::handleSaltRetrievalResponse); return verticle; }; diff --git a/src/main/java/com/uid2/operator/service/ConfigServiceManager.java b/src/main/java/com/uid2/operator/service/ConfigServiceManager.java index fc69c79fc..e21fca62f 100644 --- a/src/main/java/com/uid2/operator/service/ConfigServiceManager.java +++ b/src/main/java/com/uid2/operator/service/ConfigServiceManager.java @@ -20,7 +20,7 @@ private ConfigServiceManager(ConfigService dynamicConfigService, StaticConfigSer this.currentConfigService = useDynamicConfig ? dynamicConfigService : staticConfigService; } - public Future create(Vertx vertx, JsonObject bootstrapConfig, boolean useDynamicConfig) { + public static Future create(Vertx vertx, JsonObject bootstrapConfig, boolean useDynamicConfig) { Promise promise = Promise.promise(); StaticConfigService staticConfigService = new StaticConfigService(bootstrapConfig); diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index 4f1512e10..f3bd23937 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -96,6 +96,7 @@ public class UIDOperatorVerticle extends AbstractVerticle { private final IOptOutStore optOutStore; private final IClientKeyProvider clientKeyProvider; private final Clock clock; + private final ConfigServiceManager configServiceManager; protected IUIDOperatorService idService; private final Map _identityMapMetricSummaries = new HashMap<>(); private final Map, DistributionSummary> _refreshDurationMetricSummaries = new HashMap<>(); @@ -135,7 +136,7 @@ public class UIDOperatorVerticle extends AbstractVerticle { private static final String ERROR_INVALID_INPUT_EMAIL_TWICE = "Only one of email or email_hash can be specified"; public final static String ORIGIN_HEADER = "Origin"; - public UIDOperatorVerticle(IConfigService configService, + public UIDOperatorVerticle(ConfigServiceManager configServiceManager, boolean clientSideTokenGenerate, ISiteStore siteProvider, IClientKeyProvider clientKeyProvider, @@ -154,7 +155,8 @@ public UIDOperatorVerticle(IConfigService configService, } catch (NoSuchAlgorithmException | NoSuchPaddingException e) { throw new RuntimeException(e); } - this.configService = configService; + this.configServiceManager = configServiceManager; + this.configService = configServiceManager.getConfigService(); this.clientSideTokenGenerate = clientSideTokenGenerate; this.healthComponent.setHealthStatus(false, "not started"); this.auth = new AuthMiddleware(clientKeyProvider); From c3b2f037b5a4ce945fbe5f90753a83d261eefb1e Mon Sep 17 00:00:00 2001 From: abuabraham-ttd <130511456+abuabraham-ttd@users.noreply.github.com> Date: Thu, 2 Jan 2025 14:42:32 -0800 Subject: [PATCH 199/431] Create custom exceptions and link it to website (#1232) * Create custom exceptions and link it to website --- scripts/aws/ec2.py | 17 +++---- scripts/confidential_compute.py | 86 +++++++++++++++++---------------- 2 files changed, 53 insertions(+), 50 deletions(-) diff --git a/scripts/aws/ec2.py b/scripts/aws/ec2.py index 69fed7f91..ee47f4556 100644 --- a/scripts/aws/ec2.py +++ b/scripts/aws/ec2.py @@ -9,14 +9,14 @@ import requests import signal import argparse -from botocore.exceptions import ClientError +from botocore.exceptions import ClientError, NoCredentialsError from typing import Dict import sys import time import yaml sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from confidential_compute import ConfidentialCompute, ConfidentialComputeConfig, SecretNotFoundException, ConfidentialComputeStartupException +from confidential_compute import ConfidentialCompute, ConfidentialComputeConfig, MissingInstanceProfile, ConfigNotFound, InvalidConfigValue, ConfidentialComputeStartupException class AWSConfidentialComputeConfig(ConfidentialComputeConfig): enclave_memory_mb: int @@ -61,7 +61,7 @@ def __get_aws_token(self) -> str: ) return response.text except requests.RequestException as e: - raise RuntimeError(f"Failed to fetch aws token: {e}") + raise RuntimeError(f"Failed to fetch AWS token: {e}") def __get_current_region(self) -> str: """Fetches the current AWS region from EC2 instance metadata.""" @@ -97,16 +97,15 @@ def add_defaults(configs: Dict[str, any]) -> AWSConfidentialComputeConfig: region = self.__get_current_region() print(f"Running in {region}") - try: - client = boto3.client("secretsmanager", region_name=region) - except Exception as e: - raise RuntimeError("Please use IAM instance profile for your instance and make sure that has permission to access Secret Manager", e) + client = boto3.client("secretsmanager", region_name=region) try: secret = add_defaults(json.loads(client.get_secret_value(SecretId=secret_identifier)["SecretString"])) self.__validate_aws_specific_config(secret) return secret + except NoCredentialsError as _: + raise MissingInstanceProfile(self.__class__.__name__) except ClientError as _: - raise SecretNotFoundException(f"{secret_identifier} in {region}") + raise ConfigNotFound(self.__class__.__name__, f"Secret Manager {secret_identifier} in {region}") @staticmethod def __get_max_capacity(): @@ -255,5 +254,5 @@ def __kill_auxiliaries(self) -> None: except ConfidentialComputeStartupException as e: print("Failed starting up Confidential Compute. Please checks the logs for errors and retry \n", e) except Exception as e: - print("Unknown failure while starting up Confidential Compute. Please contact UID support team with this log \n ", e) + print("Unexpected failure while starting up Confidential Compute. Please contact UID support team with this log \n ", e) \ No newline at end of file diff --git a/scripts/confidential_compute.py b/scripts/confidential_compute.py index 306474e9c..4c80be659 100644 --- a/scripts/confidential_compute.py +++ b/scripts/confidential_compute.py @@ -13,6 +13,40 @@ class ConfidentialComputeConfig(TypedDict): environment: str skip_validations: NotRequired[bool] debug_mode: NotRequired[bool] + +class ConfidentialComputeStartupException(Exception): + def __init__(self, error_name, provider, extra_message=None): + urls = { + "EC2": "https://unifiedid.com/docs/guides/operator-guide-aws-marketplace#uid2-operator-error-codes", + "Azure": "https://unifiedid.com/docs/guides/operator-guide-azure-enclave#uid2-operator-error-codes", + "GCP": "https://unifiedid.com/docs/guides/operator-private-gcp-confidential-space#uid2-operator-error-codes", + } + url = urls.get(provider) + super().__init__(f"{error_name}\n" + (extra_message if extra_message else "") + f"\nVisit {url} for more details") + +class MissingInstanceProfile(ConfidentialComputeStartupException): + def __init__(self, cls): + super().__init__(error_name=f"E01: {self.__class__.__name__}", provider=cls) + +class ConfigNotFound(ConfidentialComputeStartupException): + def __init__(self, cls, message = None): + super().__init__(error_name=f"E02: {self.__class__.__name__}", provider=cls, extra_message=message) + +class MissingConfig(ConfidentialComputeStartupException): + def __init__(self, cls, missing_keys): + super().__init__(error_name=f"E03: {self.__class__.__name__}", provider=cls, extra_message=', '.join(missing_keys)) + +class InvalidConfigValue(ConfidentialComputeStartupException): + def __init__(self, cls, config_key = None): + super().__init__(error_name=f"E04: {self.__class__.__name__} " , provider=cls, extra_message=config_key) + +class InvalidOperatorKey(ConfidentialComputeStartupException): + def __init__(self, cls): + super().__init__(error_name=f"E05: {self.__class__.__name__}", provider=cls) + +class UID2ServicesUnreachable(ConfidentialComputeStartupException): + def __init__(self, cls, ip=None): + super().__init__(error_name=f"E06: {self.__class__.__name__}", provider=cls, extra_message=ip) class ConfidentialCompute(ABC): @@ -25,18 +59,13 @@ def validate_configuration(self): def validate_operator_key(): """ Validates the operator key format and its environment alignment.""" operator_key = self.configs.get("api_token") - if not operator_key: - raise ValueError("API token is missing from the configuration.") pattern = r"^(UID2|EUID)-.\-(I|P|L)-\d+-.*$" if re.match(pattern, operator_key): env = self.configs.get("environment", "").lower() debug_mode = self.configs.get("debug_mode", False) expected_env = "I" if debug_mode or env == "integ" else "P" - if operator_key.split("-")[2] != expected_env: - raise ValueError( - f"Operator key does not match the expected environment ({expected_env})." - ) + raise InvalidOperatorKey(self.__class__.__name__) print("Validated operator key matches environment") else: print("Skipping operator key validation") @@ -44,17 +73,12 @@ def validate_operator_key(): def validate_url(url_key, environment): """URL should include environment except in prod""" if environment != "prod" and environment not in self.configs[url_key]: - raise ValueError( - f"{url_key} must match the environment. Ensure the URL includes '{environment}'." - ) + raise InvalidConfigValue(self.__class__.__name__, url_key) parsed_url = urlparse(self.configs[url_key]) if parsed_url.scheme != 'https' and parsed_url.path: - raise ValueError( - f"{url_key} is invalid. Ensure {self.configs[url_key]} follows HTTPS, and doesn't have any path specified." - ) + raise InvalidConfigValue(self.__class__.__name__, url_key) print(f"Validated {self.configs[url_key]} matches other config parameters") - def validate_connectivity() -> None: """ Validates that the core URL is accessible.""" try: @@ -63,24 +87,22 @@ def validate_connectivity() -> None: requests.get(core_url, timeout=5) print(f"Validated connectivity to {core_url}") except (requests.ConnectionError, requests.Timeout) as e: - raise RuntimeError( - f"Failed to reach required URLs. Consider enabling {core_ip} in the egress firewall." - ) + raise UID2ServicesUnreachable(self.__class__.__name__, core_ip) except Exception as e: - raise Exception("Failed to reach the URLs.") from e + raise UID2ServicesUnreachable(self.__class__.__name__) + type_hints = get_type_hints(ConfidentialComputeConfig, include_extras=True) required_keys = [field for field, hint in type_hints.items() if "NotRequired" not in str(hint)] missing_keys = [key for key in required_keys if key not in self.configs] if missing_keys: - raise MissingConfigError(missing_keys) - + raise MissingConfig(self.__class__.__name__, missing_keys) + environment = self.configs["environment"] - if environment not in ["integ", "prod"]: - raise ValueError("Environment must be either prod/integ. It is currently set to", environment) + raise InvalidConfigValue(self.__class__.__name__, "environment") if self.configs.get("debug_mode") and environment == "prod": - raise ValueError("Debug mode cannot be enabled in the production environment.") + raise InvalidConfigValue(self.__class__.__name__, "debug_mode") validate_url("core_base_url", environment) validate_url("optout_base_url", environment) @@ -88,7 +110,6 @@ def validate_connectivity() -> None: validate_connectivity() print("Completed static validation of confidential compute config values") - @abstractmethod def _get_secret(self, secret_identifier: str) -> ConfidentialComputeConfig: """ @@ -124,21 +145,4 @@ def run_command(command, seperate_process=False): subprocess.run(command,check=True) except Exception as e: print(f"Failed to run command: {str(e)}") - raise RuntimeError (f"Failed to start {' '.join(command)} ") - -class ConfidentialComputeStartupException(Exception): - def __init__(self, message): - super().__init__(message) - -class MissingConfigError(ConfidentialComputeStartupException): - """Custom exception to handle missing config keys.""" - def __init__(self, missing_keys): - self.missing_keys = missing_keys - self.message = f"\n Missing configuration keys: {', '.join(missing_keys)} \n" - super().__init__(self.message) - -class SecretNotFoundException(ConfidentialComputeStartupException): - """Custom exception if secret manager is not found""" - def __init__(self, name): - self.message = f"Secret manager not found - {name}. Please check if secret exist and the Instance Profile has permission to read it" - super().__init__(self.message) + raise RuntimeError (f"Failed to start {' '.join(command)} ") \ No newline at end of file From 6bd4a350776194b2c233ac01f72ea9eabdc83d2e Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Mon, 6 Jan 2025 11:02:28 +1100 Subject: [PATCH 200/431] Update UIDOperatorVerticle Removed ConfigServiceManager from constructor --- .../java/com/uid2/operator/vertx/UIDOperatorVerticle.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index f3bd23937..4f1512e10 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -96,7 +96,6 @@ public class UIDOperatorVerticle extends AbstractVerticle { private final IOptOutStore optOutStore; private final IClientKeyProvider clientKeyProvider; private final Clock clock; - private final ConfigServiceManager configServiceManager; protected IUIDOperatorService idService; private final Map _identityMapMetricSummaries = new HashMap<>(); private final Map, DistributionSummary> _refreshDurationMetricSummaries = new HashMap<>(); @@ -136,7 +135,7 @@ public class UIDOperatorVerticle extends AbstractVerticle { private static final String ERROR_INVALID_INPUT_EMAIL_TWICE = "Only one of email or email_hash can be specified"; public final static String ORIGIN_HEADER = "Origin"; - public UIDOperatorVerticle(ConfigServiceManager configServiceManager, + public UIDOperatorVerticle(IConfigService configService, boolean clientSideTokenGenerate, ISiteStore siteProvider, IClientKeyProvider clientKeyProvider, @@ -155,8 +154,7 @@ public UIDOperatorVerticle(ConfigServiceManager configServiceManager, } catch (NoSuchAlgorithmException | NoSuchPaddingException e) { throw new RuntimeException(e); } - this.configServiceManager = configServiceManager; - this.configService = configServiceManager.getConfigService(); + this.configService = configService; this.clientSideTokenGenerate = clientSideTokenGenerate; this.healthComponent.setHealthStatus(false, "not started"); this.auth = new AuthMiddleware(clientKeyProvider); From 80b6b199053451fcb5790d400d5f3530629aa49f Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Mon, 6 Jan 2025 11:03:52 +1100 Subject: [PATCH 201/431] Add DelegatingConfigService Allows for switches between static and dynamic config services without having to change UIDOperatorVerticle --- .../service/DelegatingConfigService.java | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 src/main/java/com/uid2/operator/service/DelegatingConfigService.java diff --git a/src/main/java/com/uid2/operator/service/DelegatingConfigService.java b/src/main/java/com/uid2/operator/service/DelegatingConfigService.java new file mode 100644 index 000000000..36fd77da2 --- /dev/null +++ b/src/main/java/com/uid2/operator/service/DelegatingConfigService.java @@ -0,0 +1,22 @@ +package com.uid2.operator.service; + +import io.vertx.core.json.JsonObject; + +import java.util.concurrent.atomic.AtomicReference; + +public class DelegatingConfigService implements IConfigService{ + private AtomicReference activeConfigService; + + public DelegatingConfigService(IConfigService initialConfigService) { + this.activeConfigService = new AtomicReference<>(initialConfigService); + } + + public void updateConfigService(IConfigService newConfigService) { + this.activeConfigService = new AtomicReference<>(newConfigService); + } + + @Override + public JsonObject getConfig() { + return activeConfigService.get().getConfig(); + } +} From 7ef6af6e65923df6404dcf6d3aa0750f520c16bb Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Mon, 6 Jan 2025 11:07:59 +1100 Subject: [PATCH 202/431] Update ConfigServiceManager, Main ConfigRetriever in Main listens for feature flag changes and publishes to eventbus ConfigServiceManager listens to eventbus and updates DelegatingConfigService active ConfigService --- src/main/java/com/uid2/operator/Const.java | 1 + src/main/java/com/uid2/operator/Main.java | 23 +++++++++++++------ .../service/ConfigServiceManager.java | 23 ++++++++++++++----- 3 files changed, 34 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/uid2/operator/Const.java b/src/main/java/com/uid2/operator/Const.java index b1c7175b0..271b36170 100644 --- a/src/main/java/com/uid2/operator/Const.java +++ b/src/main/java/com/uid2/operator/Const.java @@ -35,5 +35,6 @@ public class Config extends com.uid2.shared.Const.Config { public static final String Config = "config"; public static final String identityV3 = "identity_v3"; public static final String RemoteConfigFeatureFlag = "remote_config_feature_flag"; + public static final String RemoteConfigFlagEventBus = "RemoteConfigFlag"; } } diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index 5ef898c85..1eb7f3763 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -203,7 +203,16 @@ else if (!Utils.isProductionEnvironment()) { } Vertx vertx = createVertx(); - VertxUtils.createConfigRetriever(vertx).getConfig(ar -> { + ConfigRetriever configRetriever = VertxUtils.createConfigRetriever(vertx); + + configRetriever.listen(configChange -> { + JsonObject newConfig = configChange.getNewConfiguration(); + boolean useDynamicConfig = newConfig.getBoolean(Const.Config.RemoteConfigFeatureFlag, true); + + vertx.eventBus().publish(Const.Config.RemoteConfigFlagEventBus, useDynamicConfig); + }); + + configRetriever.getConfig(ar -> { if (ar.failed()) { LOGGER.error("Unable to read config: " + ar.cause().getMessage(), ar.cause()); return; @@ -264,15 +273,16 @@ private ICloudStorage wrapCloudStorageForOptOut(ICloudStorage cloudStorage) { } } - private void run() throws Exception { + private void run() { this.createVertxInstancesMetric(); this.createVertxEventLoopsMetric(); ConfigServiceManager.create(vertx, config, config.getBoolean(Const.Config.RemoteConfigFeatureFlag, true)).compose(configServiceManager -> { - Supplier operatorVerticleSupplier = () -> { - UIDOperatorVerticle verticle = new UIDOperatorVerticle(configServiceManager, this.clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, getKeyManager(), saltProvider, optOutStore, Clock.systemUTC(), _statsCollectorQueue, new SecureLinkValidatorService(this.serviceLinkProvider, this.serviceProvider), this.shutdownHandler::handleSaltRetrievalResponse); - return verticle; - }; + IConfigService configService = configServiceManager.getDelegatingConfigService(); + Supplier operatorVerticleSupplier = () -> { + UIDOperatorVerticle verticle = new UIDOperatorVerticle(configService, this.clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, getKeyManager(), saltProvider, optOutStore, Clock.systemUTC(), _statsCollectorQueue, new SecureLinkValidatorService(this.serviceLinkProvider, this.serviceProvider), this.shutdownHandler::handleSaltRetrievalResponse); + return verticle; + }; DeploymentOptions options = new DeploymentOptions(); int svcInstances = this.config.getInteger(Const.Config.ServiceInstancesProp); @@ -296,7 +306,6 @@ private void run() throws Exception { .compose(v -> { metrics.setup(); vertx.setPeriodic(60000, id -> metrics.update()); - Promise promise = Promise.promise(); vertx.deployVerticle(operatorVerticleSupplier, options, promise); return promise.future(); diff --git a/src/main/java/com/uid2/operator/service/ConfigServiceManager.java b/src/main/java/com/uid2/operator/service/ConfigServiceManager.java index e21fca62f..1002fbecf 100644 --- a/src/main/java/com/uid2/operator/service/ConfigServiceManager.java +++ b/src/main/java/com/uid2/operator/service/ConfigServiceManager.java @@ -1,5 +1,6 @@ package com.uid2.operator.service; +import com.uid2.operator.Const; import io.vertx.config.ConfigRetriever; import io.vertx.core.Future; import io.vertx.core.Promise; @@ -9,7 +10,7 @@ import org.slf4j.LoggerFactory; public class ConfigServiceManager { - private IConfigService currentConfigService; + private final DelegatingConfigService delegatingConfigService; private final ConfigService dynamicConfigService; private final StaticConfigService staticConfigService; private static final Logger logger = LoggerFactory.getLogger(ConfigServiceManager.class); @@ -17,7 +18,7 @@ public class ConfigServiceManager { private ConfigServiceManager(ConfigService dynamicConfigService, StaticConfigService staticConfigService, boolean useDynamicConfig) { this.dynamicConfigService = dynamicConfigService; this.staticConfigService = staticConfigService; - this.currentConfigService = useDynamicConfig ? dynamicConfigService : staticConfigService; + this.delegatingConfigService = new DelegatingConfigService(useDynamicConfig ? dynamicConfigService : staticConfigService); } public static Future create(Vertx vertx, JsonObject bootstrapConfig, boolean useDynamicConfig) { @@ -32,6 +33,7 @@ public static Future create(Vertx vertx, JsonObject bootst if (ar.succeeded()) { ConfigService dynamicConfigService = ar.result(); ConfigServiceManager instance = new ConfigServiceManager(dynamicConfigService, staticConfigService, useDynamicConfig); + instance.initialiseListener(vertx); promise.complete(instance); } else { @@ -42,17 +44,26 @@ public static Future create(Vertx vertx, JsonObject bootst return promise.future(); } + private void initialiseListener(Vertx vertx) { + vertx.eventBus().consumer(Const.Config.RemoteConfigFlagEventBus, message -> { + boolean useDynamicConfig = Boolean.parseBoolean(message.body().toString()); + + this.updateConfigService(useDynamicConfig); + }); + } + public void updateConfigService(boolean useDynamicConfig) { if (useDynamicConfig) { logger.info("Switching to DynamicConfigService"); - this.currentConfigService = dynamicConfigService; + this.delegatingConfigService.updateConfigService(dynamicConfigService); } else { logger.info("Switching to StaticConfigService"); - this.currentConfigService = staticConfigService; + this.delegatingConfigService.updateConfigService(staticConfigService); } } - public IConfigService getConfigService() { - return currentConfigService; + public IConfigService getDelegatingConfigService() { + return delegatingConfigService; } + } From a65ac591fee570ce7847e64b3e59675b37e54fd8 Mon Sep 17 00:00:00 2001 From: sophia-chen-ttd <152837545+sophia-chen-ttd@users.noreply.github.com> Date: Mon, 6 Jan 2025 16:20:14 +1100 Subject: [PATCH 203/431] Update pom.xml --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a57034855..bf7c66a1a 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ 2.1.0 2.1.0 2.1.0 - 8.0.29 + 8.0.32 ${project.version} 21 21 From abe4a871c5e1717c3cd8120a8d9dbcbe032561ce Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Mon, 6 Jan 2025 22:39:31 +0000 Subject: [PATCH 204/431] [CI Pipeline] Released Patch version: 5.44.6 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index bf7c66a1a..5ebed11c9 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.44.0 + 5.44.6 UTF-8 From ed3141a7ae2a218bd63e9c49f98c94e8b00a0b9c Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Wed, 8 Jan 2025 11:16:16 +1100 Subject: [PATCH 205/431] Update UIDOperatorVerticleTest Added tests to verify config changes are reflected in endpoints --- .../operator/UIDOperatorVerticleTest.java | 97 +++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java index bcb9bfdb3..de105ab48 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java @@ -5090,4 +5090,101 @@ void secureLinkValidationFailsReturnsIdentityError(Vertx vertx, VertxTestContext testContext.completeNow(); }); } + + @Test + void tokenGenerateRespectsConfigValues(Vertx vertx, VertxTestContext testContext) { + final int clientSiteId = 201; + final String emailAddress = "test@uid2.com"; + fakeAuth(clientSiteId, Role.GENERATOR); + setupSalts(); + setupKeys(); + + String v1Param = "email=" + emailAddress; + JsonObject v2Payload = new JsonObject(); + v2Payload.put("email", emailAddress); + + Duration newIdentityExpiresAfter = Duration.ofMinutes(20); + Duration newRefreshExpiresAfter = Duration.ofMinutes(30); + Duration newRefreshIdentityAfter = Duration.ofMinutes(10); + + config.put(UIDOperatorService.IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, newIdentityExpiresAfter.toMillis() / 1000); + config.put(UIDOperatorService.REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, newRefreshExpiresAfter.toMillis() / 1000); + config.put(UIDOperatorService.REFRESH_IDENTITY_TOKEN_AFTER_SECONDS, newRefreshIdentityAfter.toMillis() / 1000); + when(configService.getConfig()).thenReturn(config); + + try { + sendTokenGenerate("v2", vertx, + v1Param, v2Payload, 200, + respJson -> { + JsonObject body = respJson.getJsonObject("body"); + testContext.verify(() -> { + assertNotNull(body); + assertEquals(now.plusMillis(newIdentityExpiresAfter.toMillis()).toEpochMilli(), body.getLong("identity_expires")); + assertEquals(now.plusMillis(newRefreshExpiresAfter.toMillis()).toEpochMilli(), body.getLong("refresh_expires")); + assertEquals(now.plusMillis(newRefreshIdentityAfter.toMillis()).toEpochMilli(), body.getLong("refresh_from")); + }); + testContext.completeNow(); + }); + } catch (Exception e) { + testContext.failNow(e); + } + } + + @Test + void keySharingRespectsConfigValues(Vertx vertx, VertxTestContext testContext) { + int newSharingTokenExpiry = config.getInteger(Const.Config.SharingTokenExpiryProp) + 1; + int newMaxSharingLifetimeSeconds = config.getInteger(Const.Config.SharingTokenExpiryProp) + 1; + + config.put(Const.Config.SharingTokenExpiryProp, newSharingTokenExpiry); + config.put(Const.Config.MaxSharingLifetimeProp, newMaxSharingLifetimeSeconds); + + String apiVersion = "v2"; + int siteId = 5; + fakeAuth(siteId, Role.SHARER); + Keyset[] keysets = { + new Keyset(MasterKeysetId, MasterKeySiteId, "test", null, now.getEpochSecond(), true, true), + new Keyset(10, 5, "siteKeyset", null, now.getEpochSecond(), true, true), + }; + KeysetKey[] encryptionKeys = { + new KeysetKey(101, "master key".getBytes(), now, now, now.plusSeconds(10), MasterKeysetId), + new KeysetKey(102, "site key".getBytes(), now, now, now.plusSeconds(10), 10), + }; + MultipleKeysetsTests test = new MultipleKeysetsTests(Arrays.asList(keysets), Arrays.asList(encryptionKeys)); + setupSiteDomainAndAppNameMock(true, false, 101, 102, 103, 104, 105); + send(apiVersion, vertx, apiVersion + "/key/sharing", true, null, null, 200, respJson -> { + + JsonObject body = respJson.getJsonObject("body"); + testContext.verify(() -> { + assertNotNull(body); + assertEquals(newSharingTokenExpiry, Integer.parseInt(body.getString("token_expiry_seconds"))); + assertEquals(newMaxSharingLifetimeSeconds + TOKEN_LIFETIME_TOLERANCE.toSeconds(), body.getLong(Const.Config.MaxSharingLifetimeProp)); + }); + testContext.completeNow(); + }); + } + + @Test + void keyBidstreamRespectsConfigValues(Vertx vertx, VertxTestContext testContext) { + int newMaxBidstreamLifetimeSeconds = 999999; + config.put(Const.Config.MaxBidstreamLifetimeSecondsProp, newMaxBidstreamLifetimeSeconds); + + final String apiVersion = "v2"; + final KeyDownloadEndpoint endpoint = KeyDownloadEndpoint.BIDSTREAM; + + final int clientSiteId = 101; + fakeAuth(clientSiteId, Role.ID_READER); + + // Required, sets up mock keys. + new MultipleKeysetsTests(); + + send(apiVersion, vertx, apiVersion + endpoint.getPath(), true, null, null, 200, respJson -> { + JsonObject body = respJson.getJsonObject("body"); + testContext.verify(() -> { + assertNotNull(body); + assertEquals(newMaxBidstreamLifetimeSeconds + TOKEN_LIFETIME_TOLERANCE.toSeconds(), body.getLong(Const.Config.MaxBidstreamLifetimeSecondsProp)); + }); + testContext.completeNow(); + }); + } + } From 5cde2b27a36d4a5879bc9acc91621c381e74feda Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Wed, 8 Jan 2025 11:47:39 +1100 Subject: [PATCH 206/431] Add static config to UIDOperatorVerticle constructor static config values are set from main config rather than ConfigService --- src/main/java/com/uid2/operator/Main.java | 2 +- .../java/com/uid2/operator/vertx/UIDOperatorVerticle.java | 2 +- .../java/com/uid2/operator/ExtendedUIDOperatorVerticle.java | 4 +++- src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index 1eb7f3763..a0546b25a 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -280,7 +280,7 @@ private void run() { ConfigServiceManager.create(vertx, config, config.getBoolean(Const.Config.RemoteConfigFeatureFlag, true)).compose(configServiceManager -> { IConfigService configService = configServiceManager.getDelegatingConfigService(); Supplier operatorVerticleSupplier = () -> { - UIDOperatorVerticle verticle = new UIDOperatorVerticle(configService, this.clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, getKeyManager(), saltProvider, optOutStore, Clock.systemUTC(), _statsCollectorQueue, new SecureLinkValidatorService(this.serviceLinkProvider, this.serviceProvider), this.shutdownHandler::handleSaltRetrievalResponse); + UIDOperatorVerticle verticle = new UIDOperatorVerticle(configService, config, this.clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, getKeyManager(), saltProvider, optOutStore, Clock.systemUTC(), _statsCollectorQueue, new SecureLinkValidatorService(this.serviceLinkProvider, this.serviceProvider), this.shutdownHandler::handleSaltRetrievalResponse); return verticle; }; diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index 4f1512e10..c0e6af231 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -136,6 +136,7 @@ public class UIDOperatorVerticle extends AbstractVerticle { public final static String ORIGIN_HEADER = "Origin"; public UIDOperatorVerticle(IConfigService configService, + JsonObject config, boolean clientSideTokenGenerate, ISiteStore siteProvider, IClientKeyProvider clientKeyProvider, @@ -164,7 +165,6 @@ public UIDOperatorVerticle(IConfigService configService, this.saltProvider = saltProvider; this.optOutStore = optOutStore; this.clock = clock; - JsonObject config = configService.getConfig(); this.identityScope = IdentityScope.fromString(config.getString("identity_scope", "uid2")); this.v2PayloadHandler = new V2PayloadHandler(keyManager, config.getBoolean("enable_v2_encryption", true), this.identityScope, siteProvider); this.phoneSupport = config.getBoolean("enable_phone_support", true); diff --git a/src/test/java/com/uid2/operator/ExtendedUIDOperatorVerticle.java b/src/test/java/com/uid2/operator/ExtendedUIDOperatorVerticle.java index 379d39719..24353eaf1 100644 --- a/src/test/java/com/uid2/operator/ExtendedUIDOperatorVerticle.java +++ b/src/test/java/com/uid2/operator/ExtendedUIDOperatorVerticle.java @@ -9,6 +9,7 @@ import com.uid2.operator.vertx.UIDOperatorVerticle; import com.uid2.shared.store.*; import io.vertx.core.Handler; +import io.vertx.core.json.JsonObject; import java.time.Clock; import java.time.Instant; @@ -18,6 +19,7 @@ //An extended UIDOperatorVerticle to expose classes for testing purposes public class ExtendedUIDOperatorVerticle extends UIDOperatorVerticle { public ExtendedUIDOperatorVerticle(IConfigService configService, + JsonObject config, boolean clientSideTokenGenerate, ISiteStore siteProvider, IClientKeyProvider clientKeyProvider, @@ -29,7 +31,7 @@ public ExtendedUIDOperatorVerticle(IConfigService configService, IStatsCollectorQueue statsCollectorQueue, SecureLinkValidatorService secureLinkValidationService, Handler saltRetrievalResponseHandler) { - super(configService, clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, keyManager, saltProvider, optOutStore, clock, statsCollectorQueue, secureLinkValidationService, saltRetrievalResponseHandler); + super(configService, config, clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, keyManager, saltProvider, optOutStore, clock, statsCollectorQueue, secureLinkValidationService, saltRetrievalResponseHandler); } public IUIDOperatorService getIdService() { diff --git a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java index de105ab48..2af239977 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java @@ -134,7 +134,7 @@ public void deployVerticle(Vertx vertx, VertxTestContext testContext, TestInfo t } when(configService.getConfig()).thenReturn(config); - this.uidOperatorVerticle = new ExtendedUIDOperatorVerticle(configService, config.getBoolean("client_side_token_generate"), siteProvider, clientKeyProvider, clientSideKeypairProvider, new KeyManager(keysetKeyStore, keysetProvider), saltProvider, optOutStore, clock, statsCollectorQueue, secureLinkValidatorService, shutdownHandler::handleSaltRetrievalResponse); + this.uidOperatorVerticle = new ExtendedUIDOperatorVerticle(configService, config, config.getBoolean("client_side_token_generate"), siteProvider, clientKeyProvider, clientSideKeypairProvider, new KeyManager(keysetKeyStore, keysetProvider), saltProvider, optOutStore, clock, statsCollectorQueue, secureLinkValidatorService, shutdownHandler::handleSaltRetrievalResponse); vertx.deployVerticle(uidOperatorVerticle, testContext.succeeding(id -> testContext.completeNow())); From 91ecc1d313047ab5b7dce0bcf50b596d0edc190e Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Thu, 9 Jan 2025 11:43:59 +1100 Subject: [PATCH 207/431] Update ConfigServiceTest to combine futures using compose - removed bootstrap config as a store from ConfigRetrieverFactory --- .../service/ConfigRetrieverFactory.java | 5 -- .../com/uid2/operator/ConfigServiceTest.java | 84 +++++++++---------- 2 files changed, 41 insertions(+), 48 deletions(-) diff --git a/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java b/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java index a35b63416..215b58482 100644 --- a/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java +++ b/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java @@ -23,13 +23,8 @@ public ConfigRetriever create(Vertx vertx, JsonObject bootstrapConfig) { .put("port", Const.Port.ServicePortForCore) .put("path", configPath)); - ConfigStoreOptions bootstrapStore = new ConfigStoreOptions() - .setType("json") - .setConfig(bootstrapConfig); - ConfigRetrieverOptions retrieverOptions = new ConfigRetrieverOptions() .setScanPeriod(bootstrapConfig.getLong(ConfigScanPeriodMs)) - .addStore(bootstrapStore) .addStore(httpStore); return ConfigRetriever.create(vertx, retrieverOptions); diff --git a/src/test/java/com/uid2/operator/ConfigServiceTest.java b/src/test/java/com/uid2/operator/ConfigServiceTest.java index f00b2e900..cf2b8fc6b 100644 --- a/src/test/java/com/uid2/operator/ConfigServiceTest.java +++ b/src/test/java/com/uid2/operator/ConfigServiceTest.java @@ -2,7 +2,10 @@ import com.uid2.operator.service.ConfigRetrieverFactory; import com.uid2.operator.service.ConfigService; +import io.vertx.core.Future; +import io.vertx.core.Promise; import io.vertx.core.Vertx; +import io.vertx.core.http.HttpHeaders; import io.vertx.core.http.HttpServer; import io.vertx.core.json.JsonObject; import io.vertx.config.ConfigRetriever; @@ -45,43 +48,44 @@ void tearDown() { vertx.close(); } - void startMockServer(VertxTestContext testContext, JsonObject config) { + private Future startMockServer(JsonObject config) { if (server != null) { server.close(); } + Promise promise = Promise.promise(); + Router router = Router.router(vertx); router.route().handler(BodyHandler.create()); router.get("/config").handler(ctx -> ctx.response() - .putHeader("content-type", "application/json") + .putHeader(HttpHeaders.CONTENT_TYPE, "application/json") .end(config.encode())); server = vertx.createHttpServer() .requestHandler(router) .listen(Const.Port.ServicePortForCore,"127.0.0.1", http -> { - if (!http.succeeded()) { - testContext.failNow(http.cause()); + if (http.succeeded()) { + promise.complete(); + } else { + promise.fail(http.cause()); } }); + + return promise.future(); } @Test void testGetConfig(VertxTestContext testContext) { - JsonObject httpStoreConfig = new JsonObject().put("http", "value"); - this.startMockServer(testContext, httpStoreConfig); - ConfigService.create(configRetriever).onComplete(ar -> { - if (ar.succeeded()) { - ConfigService configService = ar.result(); - JsonObject retrievedConfig = configService.getConfig(); - assertNotNull(retrievedConfig, "Config retriever should initialise without error"); - assertTrue(retrievedConfig.fieldNames().containsAll(bootstrapConfig.fieldNames()), "Retrieved config should contain all keys in bootstrap config"); - assertTrue(retrievedConfig.fieldNames().containsAll(httpStoreConfig.fieldNames()), "Retrieved config should contain all keys in http store config"); - testContext.completeNow(); - } else { - testContext.failNow(ar.cause()); - } - }); - + JsonObject httpStoreConfig = bootstrapConfig; + startMockServer(httpStoreConfig) + .compose(v -> ConfigService.create(configRetriever)) + .compose(configService -> { + JsonObject retrievedConfig = configService.getConfig(); + assertNotNull(retrievedConfig, "Config retriever should initialise without error"); + assertTrue(retrievedConfig.fieldNames().containsAll(httpStoreConfig.fieldNames()), "Retrieved config should contain all keys in http store config"); + return Future.succeededFuture(); + }) + .onComplete(testContext.succeedingThenComplete()); } @Test @@ -92,18 +96,14 @@ void testInvalidConfigRevertsToPrevious(VertxTestContext testContext) { JsonObject invalidConfig = new JsonObject() .put(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, 1000) .put(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, 2000); - this.startMockServer(testContext, invalidConfig); - ConfigService.create(spyRetriever).onComplete(ar -> { - if (ar.succeeded()) { - reset(spyRetriever); - ConfigService configService = ar.result(); - assertEquals(lastConfig, configService.getConfig(), "Invalid config not reverted to previous config"); - testContext.completeNow(); - } - else { - testContext.failNow(ar.cause()); - } - }); + startMockServer(invalidConfig) + .compose(v -> ConfigService.create(spyRetriever)) + .compose(configService -> { + reset(spyRetriever); + assertEquals(lastConfig, configService.getConfig(), "Invalid config not reverted to previous config"); + return Future.succeededFuture(); + }) + .onComplete(testContext.succeedingThenComplete()); } @Test @@ -111,17 +111,15 @@ void testFirstInvalidConfigThrowsRuntimeException(VertxTestContext testContext) JsonObject invalidConfig = new JsonObject() .put(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, 1000) .put(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, 2000); - this.startMockServer(testContext, invalidConfig); - ConfigService.create(configRetriever).onComplete(ar -> { - if (ar.succeeded()) { - testContext.failNow(new RuntimeException("Expected a RuntimeException but the creation succeeded")); - } - else { - assertThrows(RuntimeException.class, () -> { - throw ar.cause(); - }); - testContext.completeNow(); - } - }); + startMockServer(invalidConfig) + .compose(v -> ConfigService.create(configRetriever)) + .compose(configService -> Future.failedFuture(new RuntimeException("Expected a RuntimeException but the creation succeeded"))) + .recover(throwable -> { + assertThrows(RuntimeException.class, () -> { + throw throwable; + }); + return Future.succeededFuture(); + }) + .onComplete(testContext.succeedingThenComplete()); } } \ No newline at end of file From 233d9aa588a5f6e3baab88d91069e7595e1b5260 Mon Sep 17 00:00:00 2001 From: Cody Constine Date: Thu, 9 Jan 2025 09:10:43 -0700 Subject: [PATCH 208/431] Addressing last PR comments and updating shared --- pom.xml | 2 +- src/main/java/com/uid2/operator/reader/ApiStoreReader.java | 2 +- .../operator/reader/RotatingCloudEncryptionKeyApiProvider.java | 1 + src/test/java/com/uid2/operator/ApiStoreReaderTest.java | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index c11f47781..d8b22f0ad 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ 2.1.0 2.1.0 2.1.0 - 8.0.15-alpha-177-SNAPSHOT + 8.0.25 ${project.version} 21 21 diff --git a/src/main/java/com/uid2/operator/reader/ApiStoreReader.java b/src/main/java/com/uid2/operator/reader/ApiStoreReader.java index fda4c4e6a..13dc44855 100644 --- a/src/main/java/com/uid2/operator/reader/ApiStoreReader.java +++ b/src/main/java/com/uid2/operator/reader/ApiStoreReader.java @@ -36,7 +36,7 @@ public long loadContent(JsonObject contents, String dataType) throws IOException try { JsonArray dataArray = contents.getJsonArray(dataType); if (dataArray == null) { - throw new IllegalArgumentException("No array found in the contents"); + throw new IllegalArgumentException(String.format("No array of type: %s, found in the contents", dataType)); } String jsonString = dataArray.toString(); diff --git a/src/main/java/com/uid2/operator/reader/RotatingCloudEncryptionKeyApiProvider.java b/src/main/java/com/uid2/operator/reader/RotatingCloudEncryptionKeyApiProvider.java index 2b4d1b10d..eaac1b692 100644 --- a/src/main/java/com/uid2/operator/reader/RotatingCloudEncryptionKeyApiProvider.java +++ b/src/main/java/com/uid2/operator/reader/RotatingCloudEncryptionKeyApiProvider.java @@ -27,6 +27,7 @@ public RotatingCloudEncryptionKeyApiProvider(DownloadCloudStorage fileStreamProv @Override public long getVersion(JsonObject metadata) { + // Since we are pulling from an api not a data file, we use the epoch time we got the keys as the version return Instant.now().getEpochSecond(); } } diff --git a/src/test/java/com/uid2/operator/ApiStoreReaderTest.java b/src/test/java/com/uid2/operator/ApiStoreReaderTest.java index 7cd55fd1c..5359e02ed 100644 --- a/src/test/java/com/uid2/operator/ApiStoreReaderTest.java +++ b/src/test/java/com/uid2/operator/ApiStoreReaderTest.java @@ -61,7 +61,7 @@ void loadContentThrowsExceptionWhenArrayNotFound() { JsonObject contents = new JsonObject(); assertThatThrownBy(() -> reader.loadContent(contents)) .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("No array found in the contents"); + .hasMessageContaining("No array of type: test-data-type, found in the contents"); } @Test From 14360fe71c8f41e19d338166f3440ddea5ba65fb Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Thu, 9 Jan 2025 16:15:52 +0000 Subject: [PATCH 209/431] [CI Pipeline] Released Snapshot version: 5.44.7-alpha-148-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5ebed11c9..3f5f365fc 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.44.6 + 5.44.7-alpha-148-SNAPSHOT UTF-8 From ebe397187d2ac1b627e7550a0e6f357b16eca330 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Thu, 9 Jan 2025 20:01:44 +0000 Subject: [PATCH 210/431] [CI Pipeline] Released Minor version: 5.45.0 --- pom.xml | 2 +- version.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 3f5f365fc..8b40dde2a 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.44.7-alpha-148-SNAPSHOT + 5.45.0 UTF-8 diff --git a/version.json b/version.json index 0e3febcce..df99bb45c 100644 --- a/version.json +++ b/version.json @@ -1 +1 @@ -{ "$schema": "https://raw.githubusercontent.com/dotnet/Nerdbank.GitVersioning/master/src/NerdBank.GitVersioning/version.schema.json", "version": "5.44", "publicReleaseRefSpec": [ "^refs/heads/master$", "^refs/heads/v\\d+(?:\\.\\d+)?$" ], "cloudBuild": { "setVersionVariables": true, "buildNumber": { "enabled": true, "includeCommitId": { "when": "always" } } } } +{ "$schema": "https://raw.githubusercontent.com/dotnet/Nerdbank.GitVersioning/master/src/NerdBank.GitVersioning/version.schema.json", "version": "5.45", "publicReleaseRefSpec": [ "^refs/heads/master$", "^refs/heads/v\\d+(?:\\.\\d+)?$" ], "cloudBuild": { "setVersionVariables": true, "buildNumber": { "enabled": true, "includeCommitId": { "when": "always" } } } } From cd230c2bd5a3013e891fbe14a286c91621907083 Mon Sep 17 00:00:00 2001 From: Cody Constine Date: Thu, 9 Jan 2025 16:44:24 -0700 Subject: [PATCH 211/431] Reverting cloud encryption --- conf/default-config.json | 2 - conf/docker-config.json | 1 - conf/integ-config.json | 2 +- conf/local-config.json | 1 - conf/local-e2e-docker-private-config.json | 1 - conf/local-e2e-docker-public-config.json | 1 - conf/local-e2e-private-config.json | 1 - conf/local-e2e-public-config.json | 1 - ...dator-latest-e2e-docker-public-config.json | 1 - pom.xml | 2 +- scripts/aws/conf/integ-euid-config.json | 1 - scripts/aws/conf/integ-uid2-config.json | 1 - scripts/aws/conf/prod-euid-config.json | 1 - scripts/aws/conf/prod-uid2-config.json | 1 - scripts/azure-cc/conf/integ-uid2-config.json | 1 - scripts/azure-cc/conf/prod-uid2-config.json | 1 - scripts/gcp/conf/integ-config.json | 1 - scripts/gcp/conf/prod-config.json | 1 - src/main/java/com/uid2/operator/Main.java | 21 +--- .../uid2/operator/reader/ApiStoreReader.java | 57 ---------- ...RotatingCloudEncryptionKeyApiProvider.java | 33 ------ .../cloud_encryption_keys.json | 73 ------------- .../test/cloud_encryption_keys/metadata.json | 7 -- .../com/uid2/operator/ApiStoreReaderTest.java | 103 ------------------ ...tingCloudEncryptionKeyApiProviderTest.java | 102 ----------------- 25 files changed, 8 insertions(+), 409 deletions(-) delete mode 100644 src/main/java/com/uid2/operator/reader/ApiStoreReader.java delete mode 100644 src/main/java/com/uid2/operator/reader/RotatingCloudEncryptionKeyApiProvider.java delete mode 100644 src/main/resources/com.uid2.core/test/cloud_encryption_keys/cloud_encryption_keys.json delete mode 100644 src/main/resources/com.uid2.core/test/cloud_encryption_keys/metadata.json delete mode 100644 src/test/java/com/uid2/operator/ApiStoreReaderTest.java delete mode 100644 src/test/java/com/uid2/operator/RotatingCloudEncryptionKeyApiProviderTest.java diff --git a/conf/default-config.json b/conf/default-config.json index d4408bc34..224df8906 100644 --- a/conf/default-config.json +++ b/conf/default-config.json @@ -30,8 +30,6 @@ "salts_metadata_path": "salts/metadata.json", "services_metadata_path": "services/metadata.json", "service_links_metadata_path": "service_links/metadata.json", - "cloud_encryption_keys_metadata_path": "cloud_encryption_keys/metadata.json", - "cloud_encryption_keys_refresh_ms": 300000, "optout_metadata_path": null, "optout_inmem_cache": false, "enclave_platform": null, diff --git a/conf/docker-config.json b/conf/docker-config.json index 8afd14869..25f38e6ae 100644 --- a/conf/docker-config.json +++ b/conf/docker-config.json @@ -31,7 +31,6 @@ "salts_metadata_path": "/com.uid2.core/test/salts/metadata.json", "services_metadata_path": "/com.uid2.core/test/services/metadata.json", "service_links_metadata_path": "/com.uid2.core/test/service_links/metadata.json", - "cloud_encryption_keys_metadata_path": "/com.uid2.core/test/cloud_encryption_keys/metadata.json", "identity_token_expires_after_seconds": 3600, "optout_metadata_path": null, "optout_inmem_cache": false, diff --git a/conf/integ-config.json b/conf/integ-config.json index 3efa3fc7a..f1cf90742 100644 --- a/conf/integ-config.json +++ b/conf/integ-config.json @@ -14,6 +14,6 @@ "optout_api_token": "test-operator-key", "optout_api_uri": "http://localhost:8081/optout/replicate", "salts_expired_shutdown_hours": 12, - "cloud_encryption_keys_metadata_path": "http://localhost:8088/cloud_encryption_keys/retrieve", "operator_type": "public" + } \ No newline at end of file diff --git a/conf/local-config.json b/conf/local-config.json index d1bf225b5..6a357dba1 100644 --- a/conf/local-config.json +++ b/conf/local-config.json @@ -9,7 +9,6 @@ "salts_metadata_path": "/com.uid2.core/test/salts/metadata.json", "services_metadata_path": "/com.uid2.core/test/services/metadata.json", "service_links_metadata_path": "/com.uid2.core/test/service_links/metadata.json", - "cloud_encryption_keys_metadata_path":"/com.uid2.core/test/cloud_encryption_keys/metadata.json", "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, diff --git a/conf/local-e2e-docker-private-config.json b/conf/local-e2e-docker-private-config.json index 084e05004..8637e6da3 100644 --- a/conf/local-e2e-docker-private-config.json +++ b/conf/local-e2e-docker-private-config.json @@ -11,7 +11,6 @@ "keysets_metadata_path": "http://core:8088/key/keyset/refresh", "keyset_keys_metadata_path": "http://core:8088/key/keyset-keys/refresh", "salts_metadata_path": "http://core:8088/salt/refresh", - "cloud_encryption_keys_metadata_path": "http://core:8088/cloud_encryption_keys/retrieve", "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, diff --git a/conf/local-e2e-docker-public-config.json b/conf/local-e2e-docker-public-config.json index 49c07e2aa..a145c4d17 100644 --- a/conf/local-e2e-docker-public-config.json +++ b/conf/local-e2e-docker-public-config.json @@ -13,7 +13,6 @@ "salts_metadata_path": "http://core:8088/salt/refresh", "services_metadata_path": "http://core:8088/services/refresh", "service_links_metadata_path": "http://core:8088/service_links/refresh", - "cloud_encryption_keys_metadata_path": "http://core:8088/cloud_encryption_keys/retrieve", "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, diff --git a/conf/local-e2e-private-config.json b/conf/local-e2e-private-config.json index 3dc31bfdf..4ab52330f 100644 --- a/conf/local-e2e-private-config.json +++ b/conf/local-e2e-private-config.json @@ -13,7 +13,6 @@ "salts_metadata_path": "http://localhost:8088/salt/refresh", "services_metadata_path": "http://localhost:8088/services/refresh", "service_links_metadata_path": "http://localhost:8088/service_links/refresh", - "cloud_encryption_keys_metadata_path": "http://core:8088/cloud_encryption_keys/retrieve", "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, diff --git a/conf/local-e2e-public-config.json b/conf/local-e2e-public-config.json index 3b591de62..bfdc8e394 100644 --- a/conf/local-e2e-public-config.json +++ b/conf/local-e2e-public-config.json @@ -13,7 +13,6 @@ "salts_metadata_path": "http://localhost:8088/salt/refresh", "services_metadata_path": "http://localhost:8088/services/refresh", "service_links_metadata_path": "http://localhost:8088/service_links/refresh", - "cloud_encryption_keys_metadata_path": "http://core:8088/cloud_encryption_keys/retrieve", "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, diff --git a/conf/validator-latest-e2e-docker-public-config.json b/conf/validator-latest-e2e-docker-public-config.json index f607201c6..8f82b01a4 100644 --- a/conf/validator-latest-e2e-docker-public-config.json +++ b/conf/validator-latest-e2e-docker-public-config.json @@ -14,7 +14,6 @@ "salts_metadata_path": "http://core:8088/salt/refresh", "services_metadata_path": "http://core:8088/services/refresh", "service_links_metadata_path": "http://core:8088/service_links/refresh", - "cloud_encryption_keys_metadata_path": "https://core:8088/cloud_encryption_keys/retrieve", "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, diff --git a/pom.xml b/pom.xml index 8b40dde2a..5ebed11c9 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.45.0 + 5.44.6 UTF-8 diff --git a/scripts/aws/conf/integ-euid-config.json b/scripts/aws/conf/integ-euid-config.json index 22f8a15e7..45d3dbe94 100644 --- a/scripts/aws/conf/integ-euid-config.json +++ b/scripts/aws/conf/integ-euid-config.json @@ -9,7 +9,6 @@ "service_links_metadata_path": "https://core.integ.euid.eu/service_links/refresh", "optout_metadata_path": "https://optout.integ.euid.eu/optout/refresh", "core_attest_url": "https://core.integ.euid.eu/attest", - "cloud_encryption_keys_metadata_path": "https://core.integ.euid.eu/cloud_encryption_keys/retrieve", "optout_api_uri": "https://optout.integ.euid.eu/optout/replicate", "optout_s3_folder": "optout/", "allow_legacy_api": false diff --git a/scripts/aws/conf/integ-uid2-config.json b/scripts/aws/conf/integ-uid2-config.json index 11b0d6048..a7272a26a 100644 --- a/scripts/aws/conf/integ-uid2-config.json +++ b/scripts/aws/conf/integ-uid2-config.json @@ -8,7 +8,6 @@ "services_metadata_path": "https://core-integ.uidapi.com/services/refresh", "service_links_metadata_path": "https://core-integ.uidapi.com/service_links/refresh", "optout_metadata_path": "https://optout-integ.uidapi.com/optout/refresh", - "cloud_encryption_keys_metadata_path": "https://core-integ.uidapi.com/cloud_encryption_keys/retrieve", "core_attest_url": "https://core-integ.uidapi.com/attest", "optout_api_uri": "https://optout-integ.uidapi.com/optout/replicate", "optout_s3_folder": "uid-optout-integ/", diff --git a/scripts/aws/conf/prod-euid-config.json b/scripts/aws/conf/prod-euid-config.json index 7f7ecdef4..0fbf5d69c 100644 --- a/scripts/aws/conf/prod-euid-config.json +++ b/scripts/aws/conf/prod-euid-config.json @@ -9,7 +9,6 @@ "services_metadata_path": "https://core.prod.euid.eu/services/refresh", "service_links_metadata_path": "https://core.prod.euid.eu/service_links/refresh", "optout_metadata_path": "https://optout.prod.euid.eu/optout/refresh", - "cloud_encryption_keys_metadata_path": "https://core.prod.euid.eu/cloud_encryption_keys/retrieve", "core_attest_url": "https://core.prod.euid.eu/attest", "core_api_token": "your-api-token", "optout_s3_path_compat": false, diff --git a/scripts/aws/conf/prod-uid2-config.json b/scripts/aws/conf/prod-uid2-config.json index 8dd9b63d5..5da450033 100644 --- a/scripts/aws/conf/prod-uid2-config.json +++ b/scripts/aws/conf/prod-uid2-config.json @@ -8,7 +8,6 @@ "salts_metadata_path": "https://core-prod.uidapi.com/salt/refresh", "services_metadata_path": "https://core-prod.uidapi.com/services/refresh", "service_links_metadata_path": "https://core-prod.uidapi.com/service_links/refresh", - "cloud_encryption_keys_metadata_path": "https://core-prod.uidapi.com/cloud_encryption_keys/retrieve", "optout_metadata_path": "https://optout-prod.uidapi.com/optout/refresh", "core_attest_url": "https://core-prod.uidapi.com/attest", "core_api_token": "your-api-token", diff --git a/scripts/azure-cc/conf/integ-uid2-config.json b/scripts/azure-cc/conf/integ-uid2-config.json index f75f3717d..2cd4be5c3 100644 --- a/scripts/azure-cc/conf/integ-uid2-config.json +++ b/scripts/azure-cc/conf/integ-uid2-config.json @@ -7,7 +7,6 @@ "salts_metadata_path": "https://core-integ.uidapi.com/salt/refresh", "services_metadata_path": "https://core-integ.uidapi.com/services/refresh", "service_links_metadata_path": "https://core-integ.uidapi.com/service_links/refresh", - "cloud_encryption_keys_metadata_path": "https://core-integ.uidapi.com/cloud_encryption_keys/retrieve", "optout_metadata_path": "https://optout-integ.uidapi.com/optout/refresh", "core_attest_url": "https://core-integ.uidapi.com/attest", "optout_api_uri": "https://optout-integ.uidapi.com/optout/replicate", diff --git a/scripts/azure-cc/conf/prod-uid2-config.json b/scripts/azure-cc/conf/prod-uid2-config.json index 3703337a4..02e2cde20 100644 --- a/scripts/azure-cc/conf/prod-uid2-config.json +++ b/scripts/azure-cc/conf/prod-uid2-config.json @@ -7,7 +7,6 @@ "salts_metadata_path": "https://core-prod.uidapi.com/salt/refresh", "services_metadata_path": "https://core-prod.uidapi.com/services/refresh", "service_links_metadata_path": "https://core-prod.uidapi.com/service_links/refresh", - "cloud_encryption_keys_metadata_path": "https://core-prod.uidapi.com/cloud_encryption_keys/retrieve", "optout_metadata_path": "https://optout-prod.uidapi.com/optout/refresh", "core_attest_url": "https://core-prod.uidapi.com/attest", "optout_api_uri": "https://optout-prod.uidapi.com/optout/replicate", diff --git a/scripts/gcp/conf/integ-config.json b/scripts/gcp/conf/integ-config.json index 09e93dfcc..d3fb9e9ff 100644 --- a/scripts/gcp/conf/integ-config.json +++ b/scripts/gcp/conf/integ-config.json @@ -5,7 +5,6 @@ "salts_metadata_path": "https://core-integ.uidapi.com/salt/refresh", "core_attest_url": "https://core-integ.uidapi.com/attest", "optout_metadata_path": "https://optout-integ.uidapi.com/optout/refresh", - "cloud_encryption_keys_metadata_path": "https://core-integ.uidapi.com/cloud_encryption_keys/retrieve", "optout_api_uri": "https://optout-integ.uidapi.com/optout/replicate", "optout_s3_folder": "optout-v2/", "optout_inmem_cache": true, diff --git a/scripts/gcp/conf/prod-config.json b/scripts/gcp/conf/prod-config.json index 32bc0085c..836349c19 100644 --- a/scripts/gcp/conf/prod-config.json +++ b/scripts/gcp/conf/prod-config.json @@ -6,7 +6,6 @@ "core_attest_url": "https://core-prod.uidapi.com/attest", "optout_metadata_path": "https://optout-prod.uidapi.com/optout/refresh", "optout_api_uri": "https://optout-prod.uidapi.com/optout/replicate", - "cloud_encryption_keys_metadata_path": "https://core-prod.uidapi.com/cloud_encryption_keys/retrieve", "optout_s3_folder": "optout-v2/", "optout_inmem_cache": true, "identity_token_expires_after_seconds": 14400, diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index 69dff7c38..cc71114db 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -8,7 +8,6 @@ import com.uid2.operator.monitoring.IStatsCollectorQueue; import com.uid2.operator.monitoring.OperatorMetrics; import com.uid2.operator.monitoring.StatsCollectorVerticle; -import com.uid2.operator.reader.RotatingCloudEncryptionKeyApiProvider; import com.uid2.operator.service.SecureLinkValidatorService; import com.uid2.operator.service.ShutdownService; import com.uid2.operator.vertx.Endpoints; @@ -23,7 +22,6 @@ import com.uid2.shared.jmx.AdminApi; import com.uid2.shared.optout.OptOutCloudSync; import com.uid2.shared.store.CloudPath; -import com.uid2.shared.store.EncryptedRotatingSaltProvider; import com.uid2.shared.store.RotatingSaltProvider; import com.uid2.shared.store.reader.*; import com.uid2.shared.store.scope.GlobalScope; @@ -84,7 +82,6 @@ public class Main { private IStatsCollectorQueue _statsCollectorQueue; private RotatingServiceStore serviceProvider; private RotatingServiceLinkStore serviceLinkProvider; - private RotatingCloudEncryptionKeyApiProvider cloudEncryptionKeyProvider; public Main(Vertx vertx, JsonObject config) throws Exception { this.vertx = vertx; @@ -136,19 +133,17 @@ public Main(Vertx vertx, JsonObject config) throws Exception { this.fsOptOut = configureCloudOptOutStore(); } - String cloudEncryptionKeyMdPath = this.config.getString(Const.Config.CloudEncryptionKeysMetadataPathProp); - this.cloudEncryptionKeyProvider = new RotatingCloudEncryptionKeyApiProvider(fsStores, new GlobalScope(new CloudPath(cloudEncryptionKeyMdPath))); String sitesMdPath = this.config.getString(Const.Config.SitesMetadataPathProp); String keypairMdPath = this.config.getString(Const.Config.ClientSideKeypairsMetadataPathProp); - this.clientSideKeypairProvider = new RotatingClientSideKeypairStore(fsStores, new GlobalScope(new CloudPath(keypairMdPath)), cloudEncryptionKeyProvider); + this.clientSideKeypairProvider = new RotatingClientSideKeypairStore(fsStores, new GlobalScope(new CloudPath(keypairMdPath))); String clientsMdPath = this.config.getString(Const.Config.ClientsMetadataPathProp); - this.clientKeyProvider = new RotatingClientKeyProvider(fsStores, new GlobalScope(new CloudPath(clientsMdPath)), cloudEncryptionKeyProvider); + this.clientKeyProvider = new RotatingClientKeyProvider(fsStores, new GlobalScope(new CloudPath(clientsMdPath))); String keysetKeysMdPath = this.config.getString(Const.Config.KeysetKeysMetadataPathProp); - this.keysetKeyStore = new RotatingKeysetKeyStore(fsStores, new GlobalScope(new CloudPath(keysetKeysMdPath)), cloudEncryptionKeyProvider); + this.keysetKeyStore = new RotatingKeysetKeyStore(fsStores, new GlobalScope(new CloudPath(keysetKeysMdPath))); String keysetMdPath = this.config.getString(Const.Config.KeysetsMetadataPathProp); - this.keysetProvider = new RotatingKeysetProvider(fsStores, new GlobalScope(new CloudPath(keysetMdPath)), cloudEncryptionKeyProvider); + this.keysetProvider = new RotatingKeysetProvider(fsStores, new GlobalScope(new CloudPath(keysetMdPath))); String saltsMdPath = this.config.getString(Const.Config.SaltsMetadataPathProp); - this.saltProvider = new EncryptedRotatingSaltProvider(fsStores, cloudEncryptionKeyProvider, new GlobalScope(new CloudPath(saltsMdPath))); + this.saltProvider = new RotatingSaltProvider(fsStores, saltsMdPath); this.optOutStore = new CloudSyncOptOutStore(vertx, fsLocal, this.config, operatorKey, Clock.systemUTC()); if (this.validateServiceLinks) { @@ -158,7 +153,7 @@ public Main(Vertx vertx, JsonObject config) throws Exception { this.serviceLinkProvider = new RotatingServiceLinkStore(fsStores, new GlobalScope(new CloudPath(serviceLinkMdPath))); } - this.siteProvider = clientSideTokenGenerate ? new RotatingSiteStore(fsStores, new GlobalScope(new CloudPath(sitesMdPath)), cloudEncryptionKeyProvider) : null; + this.siteProvider = clientSideTokenGenerate ? new RotatingSiteStore(fsStores, new GlobalScope(new CloudPath(sitesMdPath))) : null; if (useStorageMock && coreAttestUrl == null) { if (clientSideTokenGenerate) { @@ -169,7 +164,6 @@ public Main(Vertx vertx, JsonObject config) throws Exception { this.saltProvider.loadContent(); this.keysetProvider.loadContent(); this.keysetKeyStore.loadContent(); - this.cloudEncryptionKeyProvider.loadContent(); if (this.validateServiceLinks) { this.serviceProvider.loadContent(); @@ -311,8 +305,6 @@ private void run() throws Exception { private Future createStoreVerticles() throws Exception { // load metadatas for the first time - cloudEncryptionKeyProvider.loadContent(); - if (clientSideTokenGenerate) { siteProvider.getMetadata(); clientSideKeypairProvider.getMetadata(); @@ -341,7 +333,6 @@ private Future createStoreVerticles() throws Exception { fs.add(createAndDeployRotatingStoreVerticle("auth", clientKeyProvider, "auth_refresh_ms")); fs.add(createAndDeployRotatingStoreVerticle("keyset", keysetProvider, "keyset_refresh_ms")); fs.add(createAndDeployRotatingStoreVerticle("keysetkey", keysetKeyStore, "keysetkey_refresh_ms")); - fs.add(createAndDeployRotatingStoreVerticle("cloud_encryption_keys", cloudEncryptionKeyProvider, "cloud_encryption_keys_refresh_ms")); fs.add(createAndDeployRotatingStoreVerticle("salt", saltProvider, "salt_refresh_ms")); fs.add(createAndDeployCloudSyncStoreVerticle("optout", fsOptOut, optOutCloudSync)); CompositeFuture.all(fs).onComplete(ar -> { diff --git a/src/main/java/com/uid2/operator/reader/ApiStoreReader.java b/src/main/java/com/uid2/operator/reader/ApiStoreReader.java deleted file mode 100644 index 13dc44855..000000000 --- a/src/main/java/com/uid2/operator/reader/ApiStoreReader.java +++ /dev/null @@ -1,57 +0,0 @@ -package com.uid2.operator.reader; - -import com.uid2.shared.cloud.DownloadCloudStorage; -import com.uid2.shared.store.ScopedStoreReader; -import com.uid2.shared.store.parser.Parser; -import com.uid2.shared.store.parser.ParsingResult; -import com.uid2.shared.store.scope.StoreScope; -import io.vertx.core.json.JsonArray; -import io.vertx.core.json.JsonObject; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; - -public class ApiStoreReader extends ScopedStoreReader { - private static final Logger LOGGER = LoggerFactory.getLogger(ApiStoreReader.class); - - public ApiStoreReader(DownloadCloudStorage fileStreamProvider, StoreScope scope, Parser parser, String dataTypeName) { - super(fileStreamProvider, scope, parser, dataTypeName); - } - - - public long loadContent(JsonObject contents) throws Exception { - return loadContent(contents, dataTypeName); - } - - @Override - public long loadContent(JsonObject contents, String dataType) throws IOException { - if (contents == null) { - throw new IllegalArgumentException(String.format("No contents provided for loading data type %s, cannot load content", dataType)); - } - - try { - JsonArray dataArray = contents.getJsonArray(dataType); - if (dataArray == null) { - throw new IllegalArgumentException(String.format("No array of type: %s, found in the contents", dataType)); - } - - String jsonString = dataArray.toString(); - InputStream inputStream = new ByteArrayInputStream(jsonString.getBytes(StandardCharsets.UTF_8)); - - ParsingResult parsed = parser.deserialize(inputStream); - latestSnapshot.set(parsed.getData()); - - final int count = parsed.getCount(); - latestEntryCount.set(count); - LOGGER.info(String.format("Loaded %d %s", count, dataType)); - return count; - } catch (Exception e) { - LOGGER.error(String.format("Unable to load %s", dataType)); - throw e; - } - } -} diff --git a/src/main/java/com/uid2/operator/reader/RotatingCloudEncryptionKeyApiProvider.java b/src/main/java/com/uid2/operator/reader/RotatingCloudEncryptionKeyApiProvider.java deleted file mode 100644 index eaac1b692..000000000 --- a/src/main/java/com/uid2/operator/reader/RotatingCloudEncryptionKeyApiProvider.java +++ /dev/null @@ -1,33 +0,0 @@ -package com.uid2.operator.reader; - -import com.uid2.shared.cloud.DownloadCloudStorage; -import com.uid2.shared.model.CloudEncryptionKey; -import com.uid2.shared.store.CloudPath; -import com.uid2.shared.store.parser.CloudEncryptionKeyParser; -import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider; -import com.uid2.shared.store.scope.StoreScope; -import io.vertx.core.json.JsonObject; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.time.Instant; -import java.util.*; - -public class RotatingCloudEncryptionKeyApiProvider extends RotatingCloudEncryptionKeyProvider { - private static final Logger LOGGER = LoggerFactory.getLogger(RotatingCloudEncryptionKeyApiProvider.class); - - public RotatingCloudEncryptionKeyApiProvider(DownloadCloudStorage fileStreamProvider, StoreScope scope) { - super(fileStreamProvider, scope, new ApiStoreReader<>(fileStreamProvider, scope, new CloudEncryptionKeyParser(), "cloud_encryption_keys")); - } - - public RotatingCloudEncryptionKeyApiProvider(DownloadCloudStorage fileStreamProvider, StoreScope scope, ApiStoreReader> reader) { - super(fileStreamProvider, scope, reader); - } - - - @Override - public long getVersion(JsonObject metadata) { - // Since we are pulling from an api not a data file, we use the epoch time we got the keys as the version - return Instant.now().getEpochSecond(); - } -} diff --git a/src/main/resources/com.uid2.core/test/cloud_encryption_keys/cloud_encryption_keys.json b/src/main/resources/com.uid2.core/test/cloud_encryption_keys/cloud_encryption_keys.json deleted file mode 100644 index b875d6797..000000000 --- a/src/main/resources/com.uid2.core/test/cloud_encryption_keys/cloud_encryption_keys.json +++ /dev/null @@ -1,73 +0,0 @@ -[ { - "id" : 1, - "siteId" : 999, - "activates" : 1720641670, - "created" : 1720641670, - "secret" : "mydrCudb2PZOm01Qn0SpthltmexHUAA11Hy1m+uxjVw=" -}, { - "id" : 2, - "siteId" : 999, - "activates" : 1720728070, - "created" : 1720641670, - "secret" : "FtdslrFSsvVXOuhOWGwEI+0QTkCvM8SGZAP3k2u3PgY=" -}, { - "id" : 3, - "siteId" : 999, - "activates" : 1720814470, - "created" : 1720641670, - "secret" : "/7zO6QbKrhZKIV36G+cU9UR4hZUVg5bD+KjbczICjHw=" -}, { - "id" : 4, - "siteId" : 123, - "activates" : 1720641671, - "created" : 1720641671, - "secret" : "XjiqRlWQQJGLr7xfV1qbueKwyzt881GVohuUkQt/ht4=" -}, { - "id" : 5, - "siteId" : 123, - "activates" : 1720728071, - "created" : 1720641671, - "secret" : "QmpIf5NzO+UROjl5XjB/BmF6paefM8n6ub9B2plC9aI=" -}, { - "id" : 6, - "siteId" : 123, - "activates" : 1720814471, - "created" : 1720641671, - "secret" : "40w9UMSYxGm+KldOWOXhBGI8QgjvUUQjivtkP4VpKV8=" -}, { - "id" : 7, - "siteId" : 124, - "activates" : 1720641671, - "created" : 1720641671, - "secret" : "QdwD0kQV1BwmLRD0PH1YpqgaOrgpVTfu08o98mSZ6uE=" -}, { - "id" : 8, - "siteId" : 124, - "activates" : 1720728071, - "created" : 1720641671, - "secret" : "yCVCM/HLf9/6k+aUNrx7w17VbyfSzI8JykLQLSR+CW0=" -}, { - "id" : 9, - "siteId" : 124, - "activates" : 1720814471, - "created" : 1720641671, - "secret" : "JqHl8BrTyx9XpR2lYj/5xvUpzgnibGeomETTwF4rn1U=" -}, { - "id" : 10, - "siteId" : 127, - "activates" : 1720641671, - "created" : 1720641671, - "secret" : "JqiG1b34AvrdO3Aj6cCcjOBJMijrDzTmrR+p9ZtP2es=" -}, { - "id" : 11, - "siteId" : 127, - "activates" : 1720728072, - "created" : 1720641672, - "secret" : "lp1CyHdfc7K0aO5JGpA+Ve5Z/V5LImtGEQwCg/YB0kY=" -}, { - "id" : 12, - "siteId" : 127, - "activates" : 1720814472, - "created" : 1720641672, - "secret" : "G99rFYJF+dnSlk/xG6fuC3WNqQxTLJbDIdVyPMbGQ6s=" -} ] \ No newline at end of file diff --git a/src/main/resources/com.uid2.core/test/cloud_encryption_keys/metadata.json b/src/main/resources/com.uid2.core/test/cloud_encryption_keys/metadata.json deleted file mode 100644 index af9de38c2..000000000 --- a/src/main/resources/com.uid2.core/test/cloud_encryption_keys/metadata.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "version": 1, - "generated": 1620253519, - "cloud_encryption_keys": { - "location": "/com.uid2.core/test/cloud_encryption_keys/cloud_encryption_keys.json" - } -} \ No newline at end of file diff --git a/src/test/java/com/uid2/operator/ApiStoreReaderTest.java b/src/test/java/com/uid2/operator/ApiStoreReaderTest.java deleted file mode 100644 index 5359e02ed..000000000 --- a/src/test/java/com/uid2/operator/ApiStoreReaderTest.java +++ /dev/null @@ -1,103 +0,0 @@ -package com.uid2.operator; - -import com.uid2.operator.reader.ApiStoreReader; -import com.uid2.shared.cloud.DownloadCloudStorage; -import com.uid2.shared.store.CloudPath; -import com.uid2.shared.store.parser.Parser; -import com.uid2.shared.store.parser.ParsingResult; -import com.uid2.shared.store.scope.GlobalScope; -import io.vertx.core.json.JsonArray; -import io.vertx.core.json.JsonObject; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.Mock; -import org.mockito.MockitoAnnotations; - -import java.io.InputStream; -import java.util.Arrays; -import java.util.Collection; -import java.util.List; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.when; - - class ApiStoreReaderTest { - - @Mock - private DownloadCloudStorage mockStorage; - - @Mock - private Parser> mockParser; - - private final CloudPath metadataPath = new CloudPath("test/test-metadata.json"); - private final String dataType = "test-data-type"; - private final GlobalScope scope = new GlobalScope(metadataPath); - - private ApiStoreReader> reader; - - @BeforeEach - void setUp() { - MockitoAnnotations.openMocks(this); - reader = new ApiStoreReader<>(mockStorage, scope, mockParser, dataType); - } - - @Test - void getMetadataPathReturnsPathFromScope() { - CloudPath actual = reader.getMetadataPath(); - assertThat(actual).isEqualTo(metadataPath); - } - - @Test - void loadContentThrowsExceptionWhenContentsAreNull() { - assertThatThrownBy(() -> reader.loadContent(null)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("No contents provided for loading data type"); - } - - @Test - void loadContentThrowsExceptionWhenArrayNotFound() { - JsonObject contents = new JsonObject(); - assertThatThrownBy(() -> reader.loadContent(contents)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("No array of type: test-data-type, found in the contents"); - } - - @Test - void loadContentSuccessfullyLoadsData() throws Exception { - JsonObject contents = new JsonObject() - .put(dataType, new JsonArray().add("value1").add("value2")); - - List expectedData = Arrays.asList(new TestData("value1"), new TestData("value2")); - when(mockParser.deserialize(any(InputStream.class))) - .thenReturn(new ParsingResult<>(expectedData, expectedData.size())); - - long count = reader.loadContent(contents); - - assertThat(count).isEqualTo(2); - assertThat(reader.getSnapshot()).isEqualTo(expectedData); - } - - private static class TestData { - private final String value; - - TestData(String value) { - this.value = value; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - TestData testData = (TestData) o; - return value.equals(testData.value); - } - - @Override - public int hashCode() { - return value.hashCode(); - } - } - } - diff --git a/src/test/java/com/uid2/operator/RotatingCloudEncryptionKeyApiProviderTest.java b/src/test/java/com/uid2/operator/RotatingCloudEncryptionKeyApiProviderTest.java deleted file mode 100644 index 254276658..000000000 --- a/src/test/java/com/uid2/operator/RotatingCloudEncryptionKeyApiProviderTest.java +++ /dev/null @@ -1,102 +0,0 @@ -package com.uid2.operator; - -import com.uid2.operator.reader.ApiStoreReader; -import com.uid2.operator.reader.RotatingCloudEncryptionKeyApiProvider; -import com.uid2.shared.cloud.DownloadCloudStorage; -import com.uid2.shared.model.CloudEncryptionKey; -import com.uid2.shared.store.CloudPath; -import com.uid2.shared.store.scope.StoreScope; -import io.vertx.core.json.JsonObject; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.Mock; -import org.mockito.MockitoAnnotations; - -import java.util.HashMap; -import java.util.Map; - -import static org.junit.jupiter.api.Assertions.*; -import static org.mockito.Mockito.*; - -class RotatingCloudEncryptionKeyApiProviderTest { - - @Mock - private DownloadCloudStorage mockFileStreamProvider; - - @Mock - private StoreScope mockScope; - - @Mock - private ApiStoreReader> mockApiStoreReader; - - private RotatingCloudEncryptionKeyApiProvider rotatingCloudEncryptionKeyApiProvider; - - @BeforeEach - void setUp() { - MockitoAnnotations.openMocks(this); - rotatingCloudEncryptionKeyApiProvider = new RotatingCloudEncryptionKeyApiProvider(mockFileStreamProvider, mockScope, mockApiStoreReader); - } - - @Test - void testGetMetadata() throws Exception { - JsonObject expectedMetadata = new JsonObject().put("version", 1L); - when(mockApiStoreReader.getMetadata()).thenReturn(expectedMetadata); - - JsonObject metadata = rotatingCloudEncryptionKeyApiProvider.getMetadata(); - assertEquals(expectedMetadata, metadata); - verify(mockApiStoreReader).getMetadata(); - } - - @Test - void testGetMetadataPath() { - CloudPath expectedPath = new CloudPath("test/path"); - when(mockApiStoreReader.getMetadataPath()).thenReturn(expectedPath); - - CloudPath path = rotatingCloudEncryptionKeyApiProvider.getMetadataPath(); - assertEquals(expectedPath, path); - verify(mockApiStoreReader).getMetadataPath(); - } - - @Test - void testLoadContentWithMetadata() throws Exception { - JsonObject metadata = new JsonObject(); - when(mockApiStoreReader.loadContent(metadata, "cloud_encryption_keys")).thenReturn(1L); - - long version = rotatingCloudEncryptionKeyApiProvider.loadContent(metadata); - assertEquals(1L, version); - verify(mockApiStoreReader).loadContent(metadata, "cloud_encryption_keys"); - } - - @Test - void testGetAll() { - Map expectedKeys = new HashMap<>(); - CloudEncryptionKey key = new CloudEncryptionKey(1, 123, 1687635529, 1687808329, "secret"); - expectedKeys.put(1, key); - when(mockApiStoreReader.getSnapshot()).thenReturn(expectedKeys); - - Map keys = rotatingCloudEncryptionKeyApiProvider.getAll(); - assertEquals(expectedKeys, keys); - verify(mockApiStoreReader).getSnapshot(); - } - - @Test - void testGetAllWithNullSnapshot() { - when(mockApiStoreReader.getSnapshot()).thenReturn(null); - - Map keys = rotatingCloudEncryptionKeyApiProvider.getAll(); - assertNotNull(keys); - assertTrue(keys.isEmpty()); - verify(mockApiStoreReader).getSnapshot(); - } - - @Test - void testLoadContent() throws Exception { - JsonObject metadata = new JsonObject().put("version", 1L); - when(mockApiStoreReader.getMetadata()).thenReturn(metadata); - when(mockApiStoreReader.loadContent(metadata, "cloud_encryption_keys")).thenReturn(1L); - - rotatingCloudEncryptionKeyApiProvider.loadContent(); - verify(mockApiStoreReader).getMetadata(); - verify(mockApiStoreReader).loadContent(metadata, "cloud_encryption_keys"); - } -} \ No newline at end of file From 41e7489f52b024f2c7e1ae3d3bde44720e70844a Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Tue, 14 Jan 2025 17:12:18 +1100 Subject: [PATCH 212/431] Update feature flag functionality - exclusive config retriever to listen to changes in mounted configmap - listener directly interacts with ConfigServiceManager rather than publishing to EventBus --- conf/default-config.json | 2 +- src/main/java/com/uid2/operator/Const.java | 4 +- src/main/java/com/uid2/operator/Main.java | 112 +++++++++++------- .../service/ConfigRetrieverFactory.java | 29 ++++- .../service/ConfigServiceManager.java | 69 +++++------ .../service/DelegatingConfigService.java | 4 +- .../operator/service/StaticConfigService.java | 17 --- 7 files changed, 130 insertions(+), 107 deletions(-) delete mode 100644 src/main/java/com/uid2/operator/service/StaticConfigService.java diff --git a/conf/default-config.json b/conf/default-config.json index d160858de..932f8dbbd 100644 --- a/conf/default-config.json +++ b/conf/default-config.json @@ -36,6 +36,6 @@ "failure_shutdown_wait_hours": 120, "sharing_token_expiry_seconds": 2592000, "operator_type": "public", - "core_config_path": "/config", + "core_config_path": "/operator/config", "config_scan_period_ms": 300000 } diff --git a/src/main/java/com/uid2/operator/Const.java b/src/main/java/com/uid2/operator/Const.java index 271b36170..2be8eb38a 100644 --- a/src/main/java/com/uid2/operator/Const.java +++ b/src/main/java/com/uid2/operator/Const.java @@ -30,11 +30,11 @@ public class Config extends com.uid2.shared.Const.Config { public static final String MaxInvalidPaths = "logging_limit_max_invalid_paths_per_interval"; public static final String MaxVersionBucketsPerSite = "logging_limit_max_version_buckets_per_site"; - public static final String CoreConfigPath = "core_config_path"; //TODO: update when endpoint name finalised + public static final String CoreConfigPath = "core_config_path"; public static final String ConfigScanPeriodMs = "config_scan_period_ms"; public static final String Config = "config"; public static final String identityV3 = "identity_v3"; public static final String RemoteConfigFeatureFlag = "remote_config_feature_flag"; - public static final String RemoteConfigFlagEventBus = "RemoteConfigFlag"; + public static final String RemoteConfigFlagConfigMapPath = "remote_config_feature_flag_path"; } } diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index a0546b25a..6da6f699f 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -205,13 +205,6 @@ else if (!Utils.isProductionEnvironment()) { Vertx vertx = createVertx(); ConfigRetriever configRetriever = VertxUtils.createConfigRetriever(vertx); - configRetriever.listen(configChange -> { - JsonObject newConfig = configChange.getNewConfiguration(); - boolean useDynamicConfig = newConfig.getBoolean(Const.Config.RemoteConfigFeatureFlag, true); - - vertx.eventBus().publish(Const.Config.RemoteConfigFlagEventBus, useDynamicConfig); - }); - configRetriever.getConfig(ar -> { if (ar.failed()) { LOGGER.error("Unable to read config: " + ar.cause().getMessage(), ar.cause()); @@ -277,45 +270,80 @@ private void run() { this.createVertxInstancesMetric(); this.createVertxEventLoopsMetric(); - ConfigServiceManager.create(vertx, config, config.getBoolean(Const.Config.RemoteConfigFeatureFlag, true)).compose(configServiceManager -> { - IConfigService configService = configServiceManager.getDelegatingConfigService(); - Supplier operatorVerticleSupplier = () -> { - UIDOperatorVerticle verticle = new UIDOperatorVerticle(configService, config, this.clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, getKeyManager(), saltProvider, optOutStore, Clock.systemUTC(), _statsCollectorQueue, new SecureLinkValidatorService(this.serviceLinkProvider, this.serviceProvider), this.shutdownHandler::handleSaltRetrievalResponse); - return verticle; - }; + ConfigRetrieverFactory configRetrieverFactory = new ConfigRetrieverFactory(); + ConfigRetriever dynamicConfigRetriever = configRetrieverFactory.createHttpRetriever(vertx, config); + ConfigRetriever staticConfigRetriever = configRetrieverFactory.createJsonRetriever(vertx, config); - DeploymentOptions options = new DeploymentOptions(); - int svcInstances = this.config.getInteger(Const.Config.ServiceInstancesProp); - options.setInstances(svcInstances); + Future dynamicConfigFuture = ConfigService.create(dynamicConfigRetriever); + Future staticConfigFuture = ConfigService.create(staticConfigRetriever); - Promise compositePromise = Promise.promise(); - List fs = new ArrayList<>(); - fs.add(createAndDeployStatsCollector()); - try { - fs.add(createStoreVerticles()); - } catch (Exception e) { - throw new RuntimeException(e); - } + ConfigRetriever featureFlagConfigRetriever = configRetrieverFactory.createFileRetriever( + vertx, + config.getString(Const.Config.RemoteConfigFlagConfigMapPath, "conf/local-config.json") + ); + + Future featureFlagFuture = featureFlagConfigRetriever.getConfig(); + + Future.all(dynamicConfigFuture, staticConfigFuture, featureFlagFuture) + .compose(configServiceManagerCompositeFuture -> { + ConfigService dynamicConfigService = configServiceManagerCompositeFuture.resultAt(0); + ConfigService staticConfigService = configServiceManagerCompositeFuture.resultAt(1); + JsonObject featureFlagConfig = configServiceManagerCompositeFuture.resultAt(2); + + boolean featureFlag = featureFlagConfig.getBoolean(Const.Config.RemoteConfigFeatureFlag, true); + + ConfigServiceManager configServiceManager = new ConfigServiceManager(vertx, dynamicConfigService, staticConfigService, featureFlag); - CompositeFuture.all(fs).onComplete(ar -> { - if (ar.failed()) compositePromise.fail(new Exception(ar.cause())); - else compositePromise.complete(); - }); - - return compositePromise.future() - .compose(v -> { - metrics.setup(); - vertx.setPeriodic(60000, id -> metrics.update()); - Promise promise = Promise.promise(); - vertx.deployVerticle(operatorVerticleSupplier, options, promise); - return promise.future(); - }) - .onFailure(t -> { - LOGGER.error("Failed to bootstrap operator: " + t.getMessage(), new Exception(t)); - vertx.close(); - System.exit(1); + featureFlagConfigRetriever.listen(change -> { + JsonObject newConfig = change.getNewConfiguration(); + boolean useDynamicConfig = newConfig.getBoolean(Const.Config.RemoteConfigFeatureFlag, true); + configServiceManager.updateConfigService(useDynamicConfig).onComplete(update -> { + if (update.succeeded()) { + LOGGER.info("Remote config feature flag toggled successfully"); + } else { + LOGGER.error("Failed to toggle remote config feature flag: " + update.cause()); + } + }); }); - }); + + IConfigService configService = configServiceManager.getDelegatingConfigService(); + Supplier operatorVerticleSupplier = () -> { + UIDOperatorVerticle verticle = new UIDOperatorVerticle(configService, config, this.clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, getKeyManager(), saltProvider, optOutStore, Clock.systemUTC(), _statsCollectorQueue, new SecureLinkValidatorService(this.serviceLinkProvider, this.serviceProvider), this.shutdownHandler::handleSaltRetrievalResponse); + return verticle; + }; + + DeploymentOptions options = new DeploymentOptions(); + int svcInstances = this.config.getInteger(Const.Config.ServiceInstancesProp); + options.setInstances(svcInstances); + + Promise compositePromise = Promise.promise(); + List fs = new ArrayList<>(); + fs.add(createAndDeployStatsCollector()); + try { + fs.add(createStoreVerticles()); + } catch (Exception e) { + throw new RuntimeException(e); + } + + CompositeFuture.all(fs).onComplete(ar -> { + if (ar.failed()) compositePromise.fail(new Exception(ar.cause())); + else compositePromise.complete(); + }); + + return compositePromise.future() + .compose(v -> { + metrics.setup(); + vertx.setPeriodic(60000, id -> metrics.update()); + Promise promise = Promise.promise(); + vertx.deployVerticle(operatorVerticleSupplier, options, promise); + return promise.future(); + }) + .onFailure(t -> { + LOGGER.error("Failed to bootstrap operator: " + t.getMessage(), new Exception(t)); + vertx.close(); + System.exit(1); + }); + }); } private Future createStoreVerticles() throws Exception { diff --git a/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java b/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java index 215b58482..11957fbe6 100644 --- a/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java +++ b/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java @@ -11,10 +11,9 @@ import static com.uid2.operator.Const.Config.CoreConfigPath; public class ConfigRetrieverFactory { - public ConfigRetriever create(Vertx vertx, JsonObject bootstrapConfig) { + public ConfigRetriever createHttpRetriever(Vertx vertx, JsonObject bootstrapConfig) { String configPath = bootstrapConfig.getString(CoreConfigPath); - ConfigStoreOptions httpStore = new ConfigStoreOptions() .setType("http") .setOptional(true) @@ -29,4 +28,30 @@ public ConfigRetriever create(Vertx vertx, JsonObject bootstrapConfig) { return ConfigRetriever.create(vertx, retrieverOptions); } + + public ConfigRetriever createJsonRetriever(Vertx vertx, JsonObject config) { + ConfigStoreOptions jsonStore = new ConfigStoreOptions() + .setType("json") + .setConfig(config); + + ConfigRetrieverOptions retrieverOptions = new ConfigRetrieverOptions() + .setScanPeriod(-1) + .addStore(jsonStore); + + + return ConfigRetriever.create(vertx, retrieverOptions); + } + + public ConfigRetriever createFileRetriever(Vertx vertx, String path) { + ConfigStoreOptions fileStore = new ConfigStoreOptions() + .setType("file") + .setConfig(new JsonObject() + .put("path", path) + .put("format", "json")); + + ConfigRetrieverOptions retrieverOptions = new ConfigRetrieverOptions() + .addStore(fileStore); + + return ConfigRetriever.create(vertx, retrieverOptions); + } } diff --git a/src/main/java/com/uid2/operator/service/ConfigServiceManager.java b/src/main/java/com/uid2/operator/service/ConfigServiceManager.java index 1002fbecf..5fa3ffac2 100644 --- a/src/main/java/com/uid2/operator/service/ConfigServiceManager.java +++ b/src/main/java/com/uid2/operator/service/ConfigServiceManager.java @@ -1,67 +1,54 @@ package com.uid2.operator.service; -import com.uid2.operator.Const; -import io.vertx.config.ConfigRetriever; import io.vertx.core.Future; import io.vertx.core.Promise; import io.vertx.core.Vertx; -import io.vertx.core.json.JsonObject; +import io.vertx.core.shareddata.Lock; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class ConfigServiceManager { + private final Vertx vertx; private final DelegatingConfigService delegatingConfigService; - private final ConfigService dynamicConfigService; - private final StaticConfigService staticConfigService; + private final IConfigService dynamicConfigService; + private final IConfigService staticConfigService; private static final Logger logger = LoggerFactory.getLogger(ConfigServiceManager.class); - private ConfigServiceManager(ConfigService dynamicConfigService, StaticConfigService staticConfigService, boolean useDynamicConfig) { + public ConfigServiceManager(Vertx vertx, IConfigService dynamicConfigService, IConfigService staticConfigService, boolean useDynamicConfig) { + this.vertx = vertx; this.dynamicConfigService = dynamicConfigService; this.staticConfigService = staticConfigService; this.delegatingConfigService = new DelegatingConfigService(useDynamicConfig ? dynamicConfigService : staticConfigService); } - public static Future create(Vertx vertx, JsonObject bootstrapConfig, boolean useDynamicConfig) { - Promise promise = Promise.promise(); - - StaticConfigService staticConfigService = new StaticConfigService(bootstrapConfig); - - ConfigRetrieverFactory configRetrieverFactory = new ConfigRetrieverFactory(); - ConfigRetriever configRetriever = configRetrieverFactory.create(vertx, bootstrapConfig); - - ConfigService.create(configRetriever).onComplete(ar -> { - if (ar.succeeded()) { - ConfigService dynamicConfigService = ar.result(); - ConfigServiceManager instance = new ConfigServiceManager(dynamicConfigService, staticConfigService, useDynamicConfig); - instance.initialiseListener(vertx); - promise.complete(instance); - } - else { - promise.fail(ar.cause()); + public Future updateConfigService(boolean useDynamicConfig) { + Promise promise = Promise.promise(); + vertx.sharedData().getLocalLock("updateConfigServiceLock", lockAsyncResult -> { + if (lockAsyncResult.succeeded()) { + Lock lock = lockAsyncResult.result(); + try { + if (useDynamicConfig) { + logger.info("Switching to DynamicConfigService"); + delegatingConfigService.updateConfigService(dynamicConfigService); + } else { + logger.info("Switching to StaticConfigService"); + delegatingConfigService.updateConfigService(staticConfigService); + } + promise.complete(); + } catch (Exception e) { + promise.fail(e); + } finally { + lock.release(); + } + } else { + logger.error("Failed to acquire lock for updating active ConfigService", lockAsyncResult.cause()); + promise.fail(lockAsyncResult.cause()); } }); return promise.future(); } - private void initialiseListener(Vertx vertx) { - vertx.eventBus().consumer(Const.Config.RemoteConfigFlagEventBus, message -> { - boolean useDynamicConfig = Boolean.parseBoolean(message.body().toString()); - - this.updateConfigService(useDynamicConfig); - }); - } - - public void updateConfigService(boolean useDynamicConfig) { - if (useDynamicConfig) { - logger.info("Switching to DynamicConfigService"); - this.delegatingConfigService.updateConfigService(dynamicConfigService); - } else { - logger.info("Switching to StaticConfigService"); - this.delegatingConfigService.updateConfigService(staticConfigService); - } - } - public IConfigService getDelegatingConfigService() { return delegatingConfigService; } diff --git a/src/main/java/com/uid2/operator/service/DelegatingConfigService.java b/src/main/java/com/uid2/operator/service/DelegatingConfigService.java index 36fd77da2..a34d8b1c7 100644 --- a/src/main/java/com/uid2/operator/service/DelegatingConfigService.java +++ b/src/main/java/com/uid2/operator/service/DelegatingConfigService.java @@ -5,14 +5,14 @@ import java.util.concurrent.atomic.AtomicReference; public class DelegatingConfigService implements IConfigService{ - private AtomicReference activeConfigService; + private final AtomicReference activeConfigService; public DelegatingConfigService(IConfigService initialConfigService) { this.activeConfigService = new AtomicReference<>(initialConfigService); } public void updateConfigService(IConfigService newConfigService) { - this.activeConfigService = new AtomicReference<>(newConfigService); + this.activeConfigService.set(newConfigService); } @Override diff --git a/src/main/java/com/uid2/operator/service/StaticConfigService.java b/src/main/java/com/uid2/operator/service/StaticConfigService.java deleted file mode 100644 index e44440fc9..000000000 --- a/src/main/java/com/uid2/operator/service/StaticConfigService.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.uid2.operator.service; - -import io.vertx.core.json.JsonObject; - - -public class StaticConfigService implements IConfigService { - private final JsonObject staticConfig; - - public StaticConfigService(JsonObject staticConfig) { - this.staticConfig = staticConfig; - } - - @Override - public JsonObject getConfig() { - return staticConfig; - } -} From 272ac08b823210df67c993be57e04a037d1eecd2 Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Tue, 14 Jan 2025 17:13:25 +1100 Subject: [PATCH 213/431] Add ConfigServiceManager tests --- .../operator/ConfigServiceManagerTest.java | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 src/test/java/com/uid2/operator/ConfigServiceManagerTest.java diff --git a/src/test/java/com/uid2/operator/ConfigServiceManagerTest.java b/src/test/java/com/uid2/operator/ConfigServiceManagerTest.java new file mode 100644 index 000000000..188949562 --- /dev/null +++ b/src/test/java/com/uid2/operator/ConfigServiceManagerTest.java @@ -0,0 +1,60 @@ +package com.uid2.operator; + +import com.uid2.operator.service.ConfigServiceManager; +import com.uid2.operator.service.IConfigService; +import io.vertx.core.Vertx; +import io.vertx.core.json.JsonObject; +import io.vertx.junit5.VertxExtension; +import io.vertx.junit5.VertxTestContext; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +import static com.uid2.operator.Const.Config.*; +import static org.junit.jupiter.api.Assertions.*; +import static com.uid2.operator.service.UIDOperatorService.*; +import static org.mockito.Mockito.*; + +@ExtendWith(VertxExtension.class) +public class ConfigServiceManagerTest { + private JsonObject bootstrapConfig; + private JsonObject staticConfig; + private ConfigServiceManager configServiceManager; + + @BeforeEach + void setUp(Vertx vertx) { + bootstrapConfig = new JsonObject() + .put(CoreConfigPath, "/operator/config") + .put(ConfigScanPeriodMs, 300000) + .put(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, 3600) + .put(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, 7200) + .put(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS, 1800) + .put(MaxBidstreamLifetimeSecondsProp, 7200); + staticConfig = new JsonObject(bootstrapConfig.toString()) + .put(MaxBidstreamLifetimeSecondsProp, 7201); + + IConfigService dynamicConfigService = mock(IConfigService.class); + when(dynamicConfigService.getConfig()).thenReturn(bootstrapConfig); + IConfigService staticConfigService = mock(IConfigService.class); + when(staticConfigService.getConfig()).thenReturn(staticConfig); + + configServiceManager = new ConfigServiceManager(vertx, dynamicConfigService, staticConfigService, true); + } + + @Test + void testRemoteFeatureFlag(VertxTestContext testContext) { + IConfigService delegatingConfigService = configServiceManager.getDelegatingConfigService(); + + configServiceManager.updateConfigService(true) + .compose(updateToDynamic -> { + testContext.verify(() -> assertEquals(bootstrapConfig, delegatingConfigService.getConfig())); + + return configServiceManager.updateConfigService(false); + }) + .onSuccess(updateToStatic -> testContext.verify(() -> { + assertEquals(staticConfig, delegatingConfigService.getConfig()); + testContext.completeNow(); + })) + .onFailure(testContext::failNow); + } +} From 68530b4017a97b1c7c9e91f5c56a9c98d5cffe86 Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Tue, 14 Jan 2025 17:17:17 +1100 Subject: [PATCH 214/431] Update ConfigServiceTest to remove unnecessary use of mock server --- .../com/uid2/operator/ConfigServiceTest.java | 39 ++++++++++--------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/src/test/java/com/uid2/operator/ConfigServiceTest.java b/src/test/java/com/uid2/operator/ConfigServiceTest.java index cf2b8fc6b..fe943f61e 100644 --- a/src/test/java/com/uid2/operator/ConfigServiceTest.java +++ b/src/test/java/com/uid2/operator/ConfigServiceTest.java @@ -25,26 +25,29 @@ class ConfigServiceTest { private Vertx vertx; private JsonObject bootstrapConfig; - private ConfigRetriever configRetriever; private HttpServer server; + private ConfigRetrieverFactory configRetrieverFactory; @BeforeEach void setUp() { vertx = Vertx.vertx(); bootstrapConfig = new JsonObject() - .put(CoreConfigPath, "/config") + .put(CoreConfigPath, "/operator/config") .put(ConfigScanPeriodMs, 300000) .put(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, 3600) .put(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, 7200) .put(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS, 1800) .put(MaxBidstreamLifetimeSecondsProp, 7200); - ConfigRetrieverFactory configRetrieverFactory = new ConfigRetrieverFactory(); - configRetriever = configRetrieverFactory.create(vertx, bootstrapConfig); + configRetrieverFactory = new ConfigRetrieverFactory(); + } @AfterEach void tearDown() { + if (server != null) { + server.close(); + } vertx.close(); } @@ -57,7 +60,7 @@ private Future startMockServer(JsonObject config) { Router router = Router.router(vertx); router.route().handler(BodyHandler.create()); - router.get("/config").handler(ctx -> ctx.response() + router.get("/operator/config").handler(ctx -> ctx.response() .putHeader(HttpHeaders.CONTENT_TYPE, "application/json") .end(config.encode())); @@ -76,6 +79,7 @@ private Future startMockServer(JsonObject config) { @Test void testGetConfig(VertxTestContext testContext) { + ConfigRetriever configRetriever = configRetrieverFactory.createHttpRetriever(vertx, bootstrapConfig); JsonObject httpStoreConfig = bootstrapConfig; startMockServer(httpStoreConfig) .compose(v -> ConfigService.create(configRetriever)) @@ -91,13 +95,12 @@ void testGetConfig(VertxTestContext testContext) { @Test void testInvalidConfigRevertsToPrevious(VertxTestContext testContext) { JsonObject lastConfig = new JsonObject().put("previous", "config"); - ConfigRetriever spyRetriever = spy(configRetriever); - when(spyRetriever.getCachedConfig()).thenReturn(lastConfig); JsonObject invalidConfig = new JsonObject() .put(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, 1000) .put(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, 2000); - startMockServer(invalidConfig) - .compose(v -> ConfigService.create(spyRetriever)) + ConfigRetriever spyRetriever = spy(configRetrieverFactory.createJsonRetriever(vertx, invalidConfig)); + when(spyRetriever.getCachedConfig()).thenReturn(lastConfig); + ConfigService.create(spyRetriever) .compose(configService -> { reset(spyRetriever); assertEquals(lastConfig, configService.getConfig(), "Invalid config not reverted to previous config"); @@ -111,15 +114,13 @@ void testFirstInvalidConfigThrowsRuntimeException(VertxTestContext testContext) JsonObject invalidConfig = new JsonObject() .put(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, 1000) .put(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, 2000); - startMockServer(invalidConfig) - .compose(v -> ConfigService.create(configRetriever)) - .compose(configService -> Future.failedFuture(new RuntimeException("Expected a RuntimeException but the creation succeeded"))) - .recover(throwable -> { - assertThrows(RuntimeException.class, () -> { - throw throwable; - }); - return Future.succeededFuture(); - }) - .onComplete(testContext.succeedingThenComplete()); + ConfigRetriever configRetriever = configRetrieverFactory.createJsonRetriever(vertx, invalidConfig); + ConfigService.create(configRetriever) + .onComplete(testContext.failing(throwable -> { + assertThrows(RuntimeException.class, () -> { + throw throwable; + }, "Expected a RuntimeException but the creation succeeded"); + testContext.completeNow(); + })); } } \ No newline at end of file From 1dfdea331deda4fa59d8444b9cfb596d1a0e9e40 Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Wed, 15 Jan 2025 16:19:41 +1100 Subject: [PATCH 215/431] Add authorisation header to config retriever --- conf/integ-config.json | 4 ++-- scripts/aws/conf/integ-uid2-config.json | 3 ++- src/main/java/com/uid2/operator/Const.java | 2 +- src/main/java/com/uid2/operator/Main.java | 4 ++-- .../operator/service/ConfigRetrieverFactory.java | 15 ++++++++++----- .../java/com/uid2/operator/ConfigServiceTest.java | 8 +++++--- 6 files changed, 22 insertions(+), 14 deletions(-) diff --git a/conf/integ-config.json b/conf/integ-config.json index f1cf90742..4aa307c35 100644 --- a/conf/integ-config.json +++ b/conf/integ-config.json @@ -14,6 +14,6 @@ "optout_api_token": "test-operator-key", "optout_api_uri": "http://localhost:8081/optout/replicate", "salts_expired_shutdown_hours": 12, - "operator_type": "public" - + "operator_type": "public", + "core_operator_config_path": "http://localhost:8088/operator/config" } \ No newline at end of file diff --git a/scripts/aws/conf/integ-uid2-config.json b/scripts/aws/conf/integ-uid2-config.json index a7272a26a..91b53b1f2 100644 --- a/scripts/aws/conf/integ-uid2-config.json +++ b/scripts/aws/conf/integ-uid2-config.json @@ -11,5 +11,6 @@ "core_attest_url": "https://core-integ.uidapi.com/attest", "optout_api_uri": "https://optout-integ.uidapi.com/optout/replicate", "optout_s3_folder": "uid-optout-integ/", - "allow_legacy_api": false + "allow_legacy_api": false, + "core_operator_config_path": "https://core-integ.uidapi.com/operator/config" } diff --git a/src/main/java/com/uid2/operator/Const.java b/src/main/java/com/uid2/operator/Const.java index 2be8eb38a..b1623f5ee 100644 --- a/src/main/java/com/uid2/operator/Const.java +++ b/src/main/java/com/uid2/operator/Const.java @@ -30,7 +30,7 @@ public class Config extends com.uid2.shared.Const.Config { public static final String MaxInvalidPaths = "logging_limit_max_invalid_paths_per_interval"; public static final String MaxVersionBucketsPerSite = "logging_limit_max_version_buckets_per_site"; - public static final String CoreConfigPath = "core_config_path"; + public static final String CoreConfigPath = "core_operator_config_path"; public static final String ConfigScanPeriodMs = "config_scan_period_ms"; public static final String Config = "config"; public static final String identityV3 = "identity_v3"; diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index 6da6f699f..470fafd80 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -266,12 +266,12 @@ private ICloudStorage wrapCloudStorageForOptOut(ICloudStorage cloudStorage) { } } - private void run() { + private void run() throws Exception { this.createVertxInstancesMetric(); this.createVertxEventLoopsMetric(); ConfigRetrieverFactory configRetrieverFactory = new ConfigRetrieverFactory(); - ConfigRetriever dynamicConfigRetriever = configRetrieverFactory.createHttpRetriever(vertx, config); + ConfigRetriever dynamicConfigRetriever = configRetrieverFactory.createRemoteConfigRetriever(vertx, config, this.createOperatorKeyRetriever().retrieve()); ConfigRetriever staticConfigRetriever = configRetrieverFactory.createJsonRetriever(vertx, config); Future dynamicConfigFuture = ConfigService.create(dynamicConfigRetriever); diff --git a/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java b/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java index 11957fbe6..488c9d69e 100644 --- a/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java +++ b/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java @@ -1,26 +1,31 @@ package com.uid2.operator.service; -import com.uid2.operator.Const; import io.vertx.config.ConfigRetriever; import io.vertx.config.ConfigRetrieverOptions; import io.vertx.config.ConfigStoreOptions; import io.vertx.core.Vertx; import io.vertx.core.json.JsonObject; +import java.net.URI; +import java.net.URISyntaxException; + import static com.uid2.operator.Const.Config.ConfigScanPeriodMs; import static com.uid2.operator.Const.Config.CoreConfigPath; public class ConfigRetrieverFactory { - public ConfigRetriever createHttpRetriever(Vertx vertx, JsonObject bootstrapConfig) { + public ConfigRetriever createRemoteConfigRetriever(Vertx vertx, JsonObject bootstrapConfig, String operatorKey) throws URISyntaxException { String configPath = bootstrapConfig.getString(CoreConfigPath); + URI uri = new URI(configPath); ConfigStoreOptions httpStore = new ConfigStoreOptions() .setType("http") .setOptional(true) .setConfig(new JsonObject() - .put("host", "127.0.0.1") - .put("port", Const.Port.ServicePortForCore) - .put("path", configPath)); + .put("host", uri.getHost()) + .put("port", uri.getPort()) + .put("path", uri.getPath()) + .put("headers", new JsonObject() + .put("Authorization", "Bearer " + operatorKey))); ConfigRetrieverOptions retrieverOptions = new ConfigRetrieverOptions() .setScanPeriod(bootstrapConfig.getLong(ConfigScanPeriodMs)) diff --git a/src/test/java/com/uid2/operator/ConfigServiceTest.java b/src/test/java/com/uid2/operator/ConfigServiceTest.java index fe943f61e..d7ff8e9e7 100644 --- a/src/test/java/com/uid2/operator/ConfigServiceTest.java +++ b/src/test/java/com/uid2/operator/ConfigServiceTest.java @@ -16,6 +16,8 @@ import io.vertx.junit5.VertxTestContext; import org.junit.jupiter.api.extension.ExtendWith; +import java.net.URISyntaxException; + import static com.uid2.operator.Const.Config.*; import static com.uid2.operator.service.UIDOperatorService.*; import static org.junit.jupiter.api.Assertions.*; @@ -32,7 +34,7 @@ class ConfigServiceTest { void setUp() { vertx = Vertx.vertx(); bootstrapConfig = new JsonObject() - .put(CoreConfigPath, "/operator/config") + .put(CoreConfigPath, "http://localhost:8088/operator/config") .put(ConfigScanPeriodMs, 300000) .put(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, 3600) .put(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, 7200) @@ -78,8 +80,8 @@ private Future startMockServer(JsonObject config) { } @Test - void testGetConfig(VertxTestContext testContext) { - ConfigRetriever configRetriever = configRetrieverFactory.createHttpRetriever(vertx, bootstrapConfig); + void testGetConfig(VertxTestContext testContext) throws URISyntaxException { + ConfigRetriever configRetriever = configRetrieverFactory.createRemoteConfigRetriever(vertx, bootstrapConfig, ""); JsonObject httpStoreConfig = bootstrapConfig; startMockServer(httpStoreConfig) .compose(v -> ConfigService.create(configRetriever)) From 908557032b6f4379005584c87d2e4876bb23dd7a Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Thu, 16 Jan 2025 18:34:04 +1100 Subject: [PATCH 216/431] Add fallback to static config when dynamic config fails --- conf/default-config.json | 1 - conf/local-config.json | 2 +- .../remote-config-feat-flag.json | 3 + src/main/java/com/uid2/operator/Const.java | 2 - src/main/java/com/uid2/operator/Main.java | 57 ++++++++++++------- .../uid2/operator/service/ConfigService.java | 4 +- .../operator/service/ConfigValidatorUtil.java | 25 ++++---- .../operator/vertx/UIDOperatorVerticle.java | 1 + .../service/ConfigValidatorUtilTest.java | 12 ---- 9 files changed, 58 insertions(+), 49 deletions(-) create mode 100644 conf/remote-config-feat-flag/remote-config-feat-flag.json diff --git a/conf/default-config.json b/conf/default-config.json index 932f8dbbd..a72c1f9df 100644 --- a/conf/default-config.json +++ b/conf/default-config.json @@ -36,6 +36,5 @@ "failure_shutdown_wait_hours": 120, "sharing_token_expiry_seconds": 2592000, "operator_type": "public", - "core_config_path": "/operator/config", "config_scan_period_ms": 300000 } diff --git a/conf/local-config.json b/conf/local-config.json index c73940613..76b0395da 100644 --- a/conf/local-config.json +++ b/conf/local-config.json @@ -37,5 +37,5 @@ "client_side_token_generate_log_invalid_http_origins": true, "salts_expired_shutdown_hours": 12, "operator_type": "public", - "remote_config_feature_flag": true + "core_operator_config_path": "http://localhost:8088/operator/config" } diff --git a/conf/remote-config-feat-flag/remote-config-feat-flag.json b/conf/remote-config-feat-flag/remote-config-feat-flag.json new file mode 100644 index 000000000..aaf4b6db4 --- /dev/null +++ b/conf/remote-config-feat-flag/remote-config-feat-flag.json @@ -0,0 +1,3 @@ +{ + "remote_config_feature_flag": false +} \ No newline at end of file diff --git a/src/main/java/com/uid2/operator/Const.java b/src/main/java/com/uid2/operator/Const.java index b1623f5ee..3e584877f 100644 --- a/src/main/java/com/uid2/operator/Const.java +++ b/src/main/java/com/uid2/operator/Const.java @@ -32,9 +32,7 @@ public class Config extends com.uid2.shared.Const.Config { public static final String CoreConfigPath = "core_operator_config_path"; public static final String ConfigScanPeriodMs = "config_scan_period_ms"; - public static final String Config = "config"; public static final String identityV3 = "identity_v3"; public static final String RemoteConfigFeatureFlag = "remote_config_feature_flag"; - public static final String RemoteConfigFlagConfigMapPath = "remote_config_feature_flag_path"; } } diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index 470fafd80..8c00c772e 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -271,40 +271,41 @@ private void run() throws Exception { this.createVertxEventLoopsMetric(); ConfigRetrieverFactory configRetrieverFactory = new ConfigRetrieverFactory(); - ConfigRetriever dynamicConfigRetriever = configRetrieverFactory.createRemoteConfigRetriever(vertx, config, this.createOperatorKeyRetriever().retrieve()); ConfigRetriever staticConfigRetriever = configRetrieverFactory.createJsonRetriever(vertx, config); - - Future dynamicConfigFuture = ConfigService.create(dynamicConfigRetriever); + ConfigRetriever dynamicConfigRetriever = configRetrieverFactory.createRemoteConfigRetriever(vertx, config, this.createOperatorKeyRetriever().retrieve()); Future staticConfigFuture = ConfigService.create(staticConfigRetriever); + Future dynamicConfigFuture = ConfigService.create(dynamicConfigRetriever); ConfigRetriever featureFlagConfigRetriever = configRetrieverFactory.createFileRetriever( vertx, - config.getString(Const.Config.RemoteConfigFlagConfigMapPath, "conf/local-config.json") + "conf/remote-config-feat-flag/remote-config-feat-flag.json" ); - Future featureFlagFuture = featureFlagConfigRetriever.getConfig(); - Future.all(dynamicConfigFuture, staticConfigFuture, featureFlagFuture) + featureFlagFuture.compose(featureFlagConfig -> { + boolean featureFlag = featureFlagConfig.getBoolean(Const.Config.RemoteConfigFeatureFlag, true); + // use static config if dynamic config fails and feature flag toggled off + return dynamicConfigFuture + .recover(throwable -> { + if (!featureFlag) { + LOGGER.warn("Dynamic config service creation failed: " + throwable.getMessage()); + return staticConfigFuture; + } else { + return Future.failedFuture(new Exception("Dynamic config service creation failed and feature flag is enabled: " + throwable.getMessage())); + } + }) + .compose(dynamicConfigService -> Future.all(Future.succeededFuture(dynamicConfigService), staticConfigFuture)); + }) .compose(configServiceManagerCompositeFuture -> { ConfigService dynamicConfigService = configServiceManagerCompositeFuture.resultAt(0); ConfigService staticConfigService = configServiceManagerCompositeFuture.resultAt(1); - JsonObject featureFlagConfig = configServiceManagerCompositeFuture.resultAt(2); - boolean featureFlag = featureFlagConfig.getBoolean(Const.Config.RemoteConfigFeatureFlag, true); + boolean featureFlag = featureFlagConfigRetriever.getCachedConfig().getBoolean(Const.Config.RemoteConfigFeatureFlag, false); - ConfigServiceManager configServiceManager = new ConfigServiceManager(vertx, dynamicConfigService, staticConfigService, featureFlag); - - featureFlagConfigRetriever.listen(change -> { - JsonObject newConfig = change.getNewConfiguration(); - boolean useDynamicConfig = newConfig.getBoolean(Const.Config.RemoteConfigFeatureFlag, true); - configServiceManager.updateConfigService(useDynamicConfig).onComplete(update -> { - if (update.succeeded()) { - LOGGER.info("Remote config feature flag toggled successfully"); - } else { - LOGGER.error("Failed to toggle remote config feature flag: " + update.cause()); - } - }); - }); + ConfigServiceManager configServiceManager = new ConfigServiceManager( + vertx, dynamicConfigService, staticConfigService, featureFlag); + + setupFeatureFlagListener(configServiceManager, featureFlagConfigRetriever); IConfigService configService = configServiceManager.getDelegatingConfigService(); Supplier operatorVerticleSupplier = () -> { @@ -346,6 +347,20 @@ private void run() throws Exception { }); } + private void setupFeatureFlagListener(ConfigServiceManager manager, ConfigRetriever retriever) { + retriever.listen(change -> { + JsonObject newConfig = change.getNewConfiguration(); + boolean useDynamicConfig = newConfig.getBoolean(Const.Config.RemoteConfigFeatureFlag, true); + manager.updateConfigService(useDynamicConfig).onComplete(update -> { + if (update.succeeded()) { + LOGGER.info("Remote config feature flag toggled successfully"); + } else { + LOGGER.error("Failed to toggle remote config feature flag: " + update.cause()); + } + }); + }); + } + private Future createStoreVerticles() throws Exception { // load metadatas for the first time if (clientSideTokenGenerate) { diff --git a/src/main/java/com/uid2/operator/service/ConfigService.java b/src/main/java/com/uid2/operator/service/ConfigService.java index cd5b8b9bc..4326fa1ad 100644 --- a/src/main/java/com/uid2/operator/service/ConfigService.java +++ b/src/main/java/com/uid2/operator/service/ConfigService.java @@ -28,10 +28,9 @@ public static Future create(ConfigRetriever configRetriever) { configRetriever.getConfig(ar -> { if (ar.succeeded()) { - System.out.println("Successfully loaded config"); + logger.info("Successfully loaded config"); promise.complete(instance); } else { - System.err.println("Failed to load config: " + ar.cause().getMessage()); logger.error("Failed to load config: {}", ar.cause().getMessage()); promise.fail(ar.cause()); } @@ -65,6 +64,7 @@ private JsonObject configValidationHandler(JsonObject config) { return lastConfig; } + logger.info("Successfully updated config"); return config; } } diff --git a/src/main/java/com/uid2/operator/service/ConfigValidatorUtil.java b/src/main/java/com/uid2/operator/service/ConfigValidatorUtil.java index c7643d707..1859fb766 100644 --- a/src/main/java/com/uid2/operator/service/ConfigValidatorUtil.java +++ b/src/main/java/com/uid2/operator/service/ConfigValidatorUtil.java @@ -3,6 +3,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static com.uid2.operator.Const.Config.MaxBidstreamLifetimeSecondsProp; import static com.uid2.operator.service.UIDOperatorService.*; public class ConfigValidatorUtil { @@ -12,36 +13,40 @@ public class ConfigValidatorUtil { public static Boolean validateIdentityRefreshTokens(Integer identityExpiresAfter, Integer refreshExpiresAfter, Integer refreshIdentityAfter) { boolean isValid = true; - if (identityExpiresAfter == null || refreshExpiresAfter == null || refreshIdentityAfter == null) { + if (areValuesNull(identityExpiresAfter, refreshExpiresAfter, refreshIdentityAfter)) { logger.error(VALUES_ARE_NULL); return false; } - if (identityExpiresAfter > refreshExpiresAfter) { + if (refreshExpiresAfter < identityExpiresAfter) { logger.error(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS + " must be >= " + IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS); isValid = false; } - if (refreshIdentityAfter > identityExpiresAfter) { + if (identityExpiresAfter < refreshIdentityAfter) { logger.error(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS + " must be >= " + REFRESH_IDENTITY_TOKEN_AFTER_SECONDS); isValid = false; } - if (refreshIdentityAfter > refreshExpiresAfter) { + if (refreshExpiresAfter < refreshIdentityAfter) { logger.error(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS + " must be >= " + REFRESH_IDENTITY_TOKEN_AFTER_SECONDS); } return isValid; } public static Boolean validateBidstreamLifetime(Integer maxBidstreamLifetimeSeconds, Integer identityTokenExpiresAfterSeconds) { - if (maxBidstreamLifetimeSeconds == null || identityTokenExpiresAfterSeconds == null) { - logger.error(VALUES_ARE_NULL); - return false; - } - if (maxBidstreamLifetimeSeconds < identityTokenExpiresAfterSeconds) { - logger.error("Max bidstream lifetime seconds ({} seconds) is less than identity token lifetime ({} seconds)", maxBidstreamLifetimeSeconds, identityTokenExpiresAfterSeconds); + logger.error(MaxBidstreamLifetimeSecondsProp + " must be >= " + IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS); return false; } return true; } + + private static boolean areValuesNull(Integer... values) { + for (Integer value : values) { + if (value == null) { + return true; + } + } + return false; + } } diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index c0e6af231..ed9cceed7 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -133,6 +133,7 @@ public class UIDOperatorVerticle extends AbstractVerticle { private static final String ERROR_INVALID_INPUT_WITH_PHONE_SUPPORT = "Required Parameter Missing: exactly one of [email, email_hash, phone, phone_hash] must be specified"; private static final String ERROR_INVALID_INPUT_EMAIL_MISSING = "Required Parameter Missing: exactly one of email or email_hash must be specified"; private static final String ERROR_INVALID_INPUT_EMAIL_TWICE = "Only one of email or email_hash can be specified"; + public static final String Config = "config"; public final static String ORIGIN_HEADER = "Origin"; public UIDOperatorVerticle(IConfigService configService, diff --git a/src/test/java/com/uid2/operator/service/ConfigValidatorUtilTest.java b/src/test/java/com/uid2/operator/service/ConfigValidatorUtilTest.java index dde9b6080..f9b18e4cc 100644 --- a/src/test/java/com/uid2/operator/service/ConfigValidatorUtilTest.java +++ b/src/test/java/com/uid2/operator/service/ConfigValidatorUtilTest.java @@ -44,16 +44,4 @@ void testValidateIdentityRefreshTokensWithNullValues() { // all values are null assertFalse(ConfigValidatorUtil.validateIdentityRefreshTokens(null, null, null)); } - - @Test - void testValidateBidstreamLifetimeWithNullValues() { - // maxBidstreamLifetimeSeconds is null - assertFalse(ConfigValidatorUtil.validateBidstreamLifetime(null, 10)); - - // identityTokenExpiresAfterSeconds is null - assertFalse(ConfigValidatorUtil.validateBidstreamLifetime(10, null)); - - // both values are null - assertFalse(ConfigValidatorUtil.validateBidstreamLifetime(null, null)); - } } From ca885888b906dddc6d43d6f35b06eaa66a8f7aa6 Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Thu, 16 Jan 2025 19:06:26 +1100 Subject: [PATCH 217/431] Add null check to ConfigValidatorUtil validateBidstreamLifetime --- .../java/com/uid2/operator/service/ConfigValidatorUtil.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/uid2/operator/service/ConfigValidatorUtil.java b/src/main/java/com/uid2/operator/service/ConfigValidatorUtil.java index 1859fb766..1c2ae471d 100644 --- a/src/main/java/com/uid2/operator/service/ConfigValidatorUtil.java +++ b/src/main/java/com/uid2/operator/service/ConfigValidatorUtil.java @@ -18,7 +18,6 @@ public static Boolean validateIdentityRefreshTokens(Integer identityExpiresAfter return false; } - if (refreshExpiresAfter < identityExpiresAfter) { logger.error(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS + " must be >= " + IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS); isValid = false; @@ -34,6 +33,10 @@ public static Boolean validateIdentityRefreshTokens(Integer identityExpiresAfter } public static Boolean validateBidstreamLifetime(Integer maxBidstreamLifetimeSeconds, Integer identityTokenExpiresAfterSeconds) { + if (areValuesNull(maxBidstreamLifetimeSeconds, identityTokenExpiresAfterSeconds)) { + logger.error(VALUES_ARE_NULL); + return false; + } if (maxBidstreamLifetimeSeconds < identityTokenExpiresAfterSeconds) { logger.error(MaxBidstreamLifetimeSecondsProp + " must be >= " + IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS); return false; From 174b0c176e39e2b9093233dd95696d5e2614fce1 Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Tue, 21 Jan 2025 15:08:52 +1100 Subject: [PATCH 218/431] Addressed Review Comments - Made feature flag json file generic - Small refactoring --- .../feat-flag.json} | 0 scripts/aws/conf/integ-uid2-config.json | 3 +-- src/main/java/com/uid2/operator/Const.java | 2 +- src/main/java/com/uid2/operator/Main.java | 7 +++---- .../uid2/operator/vertx/UIDOperatorVerticle.java | 13 ++++++------- .../com/uid2/operator/UIDOperatorServiceTest.java | 14 +++++++------- .../com/uid2/operator/UIDOperatorVerticleTest.java | 2 +- .../uid2/operator/benchmark/BenchmarkCommon.java | 4 ++-- 8 files changed, 21 insertions(+), 24 deletions(-) rename conf/{remote-config-feat-flag/remote-config-feat-flag.json => feat-flag/feat-flag.json} (100%) diff --git a/conf/remote-config-feat-flag/remote-config-feat-flag.json b/conf/feat-flag/feat-flag.json similarity index 100% rename from conf/remote-config-feat-flag/remote-config-feat-flag.json rename to conf/feat-flag/feat-flag.json diff --git a/scripts/aws/conf/integ-uid2-config.json b/scripts/aws/conf/integ-uid2-config.json index 91b53b1f2..a7272a26a 100644 --- a/scripts/aws/conf/integ-uid2-config.json +++ b/scripts/aws/conf/integ-uid2-config.json @@ -11,6 +11,5 @@ "core_attest_url": "https://core-integ.uidapi.com/attest", "optout_api_uri": "https://optout-integ.uidapi.com/optout/replicate", "optout_s3_folder": "uid-optout-integ/", - "allow_legacy_api": false, - "core_operator_config_path": "https://core-integ.uidapi.com/operator/config" + "allow_legacy_api": false } diff --git a/src/main/java/com/uid2/operator/Const.java b/src/main/java/com/uid2/operator/Const.java index 3e584877f..7450b7389 100644 --- a/src/main/java/com/uid2/operator/Const.java +++ b/src/main/java/com/uid2/operator/Const.java @@ -32,7 +32,7 @@ public class Config extends com.uid2.shared.Const.Config { public static final String CoreConfigPath = "core_operator_config_path"; public static final String ConfigScanPeriodMs = "config_scan_period_ms"; - public static final String identityV3 = "identity_v3"; + public static final String IdentityV3 = "identity_v3"; public static final String RemoteConfigFeatureFlag = "remote_config_feature_flag"; } } diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index 80b0c15bc..fb120ea4b 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -40,7 +40,6 @@ import io.vertx.config.ConfigRetriever; import io.vertx.core.*; import io.vertx.core.http.HttpServerOptions; -import io.vertx.core.http.impl.HttpUtils; import io.vertx.core.json.JsonObject; import io.vertx.micrometer.*; import io.vertx.micrometer.backends.BackendRegistries; @@ -279,7 +278,7 @@ private void run() throws Exception { ConfigRetriever featureFlagConfigRetriever = configRetrieverFactory.createFileRetriever( vertx, - "conf/remote-config-feat-flag/remote-config-feat-flag.json" + "conf/feat-flag/feat-flag.json" ); Future featureFlagFuture = featureFlagConfigRetriever.getConfig(); @@ -289,10 +288,10 @@ private void run() throws Exception { return dynamicConfigFuture .recover(throwable -> { if (!featureFlag) { - LOGGER.warn("Dynamic config service creation failed: " + throwable.getMessage()); + LOGGER.warn("Dynamic config service creation failed: ", throwable); return staticConfigFuture; } else { - return Future.failedFuture(new Exception("Dynamic config service creation failed and feature flag is enabled: " + throwable.getMessage())); + return Future.failedFuture(new Exception("Dynamic config service creation failed and feature flag is enabled: ", throwable)); } }) .compose(dynamicConfigService -> Future.all(Future.succeededFuture(dynamicConfigService), staticConfigFuture)); diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index ed9cceed7..7422ad380 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -96,6 +96,8 @@ public class UIDOperatorVerticle extends AbstractVerticle { private final IOptOutStore optOutStore; private final IClientKeyProvider clientKeyProvider; private final Clock clock; + private final Boolean allowLegacyAPI; + private final Boolean identityV3Enabled; protected IUIDOperatorService idService; private final Map _identityMapMetricSummaries = new HashMap<>(); private final Map, DistributionSummary> _refreshDurationMetricSummaries = new HashMap<>(); @@ -179,11 +181,12 @@ public UIDOperatorVerticle(IConfigService configService, this.saltRetrievalResponseHandler = saltRetrievalResponseHandler; this.optOutStatusApiEnabled = config.getBoolean(Const.Config.OptOutStatusApiEnabled, true); this.optOutStatusMaxRequestSize = config.getInteger(Const.Config.OptOutStatusMaxRequestSize, 5000); + this.allowLegacyAPI = config.getBoolean(Const.Config.AllowLegacyAPIProp, true); + this.identityV3Enabled = config.getBoolean(IdentityV3, false); } @Override public void start(Promise startPromise) throws Exception { - Boolean identityV3Enabled = this.configService.getConfig().getBoolean(identityV3, false); this.healthComponent.setHealthStatus(false, "still starting"); this.idService = new UIDOperatorService( this.optOutStore, @@ -192,7 +195,7 @@ public void start(Promise startPromise) throws Exception { this.clock, this.identityScope, this.saltRetrievalResponseHandler, - identityV3Enabled + this.identityV3Enabled ); final Router router = createRoutesSetup(); @@ -245,7 +248,7 @@ private Router createRoutesSetup() throws IOException { // Static and health check router.get(OPS_HEALTHCHECK.toString()).handler(this::handleHealthCheck); - if (this.configService.getConfig().getBoolean(Const.Config.AllowLegacyAPIProp, true)) { + if (this.allowLegacyAPI) { // V1 APIs router.get(V1_TOKEN_GENERATE.toString()).handler(auth.handleV1(this::handleTokenGenerateV1, Role.GENERATOR)); router.get(V1_TOKEN_VALIDATE.toString()).handler(this::handleTokenValidateV1); @@ -618,10 +621,6 @@ public void handleKeysRequest(RoutingContext rc) { } } -// private String getSharingTokenExpirySeconds() { -// return config.getString(Const.Config.SharingTokenExpiryProp); -// } - public void handleKeysSharing(RoutingContext rc) { JsonObject config = this.getConfigFromRc(rc); Integer maxSharingLifetimeSeconds = config.getInteger(Const.Config.MaxSharingLifetimeProp, config.getInteger(Const.Config.SharingTokenExpiryProp)); diff --git a/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java b/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java index a818d29df..a7b37a71a 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java @@ -22,7 +22,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import static com.uid2.operator.Const.Config.identityV3; +import static com.uid2.operator.Const.Config.IdentityV3; import static org.junit.jupiter.api.Assertions.*; import org.junit.jupiter.params.ParameterizedTest; @@ -91,7 +91,7 @@ void setup() throws Exception { uid2Config.put(UIDOperatorService.IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS); uid2Config.put(UIDOperatorService.REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, REFRESH_TOKEN_EXPIRES_AFTER_SECONDS); uid2Config.put(UIDOperatorService.REFRESH_IDENTITY_TOKEN_AFTER_SECONDS, REFRESH_IDENTITY_TOKEN_AFTER_SECONDS); - uid2Config.put(identityV3, false); + uid2Config.put(IdentityV3, false); uid2Service = new ExtendedUIDOperatorService( optOutStore, @@ -100,14 +100,14 @@ void setup() throws Exception { this.clock, IdentityScope.UID2, this.shutdownHandler::handleSaltRetrievalResponse, - uid2Config.getBoolean(identityV3) + uid2Config.getBoolean(IdentityV3) ); euidConfig = new JsonObject(); euidConfig.put(UIDOperatorService.IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS); euidConfig.put(UIDOperatorService.REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, REFRESH_TOKEN_EXPIRES_AFTER_SECONDS); euidConfig.put(UIDOperatorService.REFRESH_IDENTITY_TOKEN_AFTER_SECONDS, REFRESH_IDENTITY_TOKEN_AFTER_SECONDS); - euidConfig.put(identityV3, true); + euidConfig.put(IdentityV3, true); euidService = new ExtendedUIDOperatorService( optOutStore, @@ -116,7 +116,7 @@ void setup() throws Exception { this.clock, IdentityScope.EUID, this.shutdownHandler::handleSaltRetrievalResponse, - euidConfig.getBoolean(identityV3) + euidConfig.getBoolean(IdentityV3) ); } @@ -783,7 +783,7 @@ void testExpiredSaltsNotifiesShutdownHandler(TestIdentityInputType type, String this.clock, IdentityScope.UID2, this.shutdownHandler::handleSaltRetrievalResponse, - uid2Config.getBoolean(identityV3) + uid2Config.getBoolean(IdentityV3) ); UIDOperatorService euidService = new UIDOperatorService( @@ -793,7 +793,7 @@ void testExpiredSaltsNotifiesShutdownHandler(TestIdentityInputType type, String this.clock, IdentityScope.EUID, this.shutdownHandler::handleSaltRetrievalResponse, - euidConfig.getBoolean(identityV3) + euidConfig.getBoolean(IdentityV3) ); when(this.optOutStore.getLatestEntry(any())).thenReturn(null); diff --git a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java index 2af239977..f064a0f75 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java @@ -157,7 +157,7 @@ private void setupConfig(JsonObject config) { config.put(Const.Config.SharingTokenExpiryProp, 60 * 60 * 24 * 30); config.put("identity_scope", getIdentityScope().toString()); - config.put("identity_v3", useRawUidV3()); + config.put(Const.Config.IdentityV3, useRawUidV3()); config.put("client_side_token_generate", true); config.put("key_sharing_endpoint_provide_app_names", true); config.put("client_side_token_generate_log_invalid_http_origins", true); diff --git a/src/test/java/com/uid2/operator/benchmark/BenchmarkCommon.java b/src/test/java/com/uid2/operator/benchmark/BenchmarkCommon.java index 11b0f4a3b..111d17706 100644 --- a/src/test/java/com/uid2/operator/benchmark/BenchmarkCommon.java +++ b/src/test/java/com/uid2/operator/benchmark/BenchmarkCommon.java @@ -41,7 +41,7 @@ import java.util.List; import java.util.Random; -import static com.uid2.operator.Const.Config.identityV3; +import static com.uid2.operator.Const.Config.IdentityV3; public class BenchmarkCommon { @@ -77,7 +77,7 @@ static IUIDOperatorService createUidOperatorService() throws Exception { Clock.systemUTC(), IdentityScope.UID2, null, - config.getBoolean(identityV3) + config.getBoolean(IdentityV3) ); } From 4c8e8729ba5cd8083286c666822ec92a3a4a4267 Mon Sep 17 00:00:00 2001 From: abuabraham-ttd <130511456+abuabraham-ttd@users.noreply.github.com> Date: Tue, 21 Jan 2025 15:19:41 -0800 Subject: [PATCH 219/431] Abu UI d2 4557 gcp validation support (#1286) * Adding validations in GCP --------- Co-authored-by: Release Workflow --- .../publish-gcp-oidc-enclave-docker.yaml | 1 + pom.xml | 2 +- scripts/confidential_compute.py | 6 +- scripts/gcp-oidc/Dockerfile | 14 ++- scripts/gcp-oidc/conf/default-config.json | 84 ++++++++-------- scripts/gcp-oidc/conf/integ-config.json | 14 +++ scripts/gcp-oidc/conf/integ-uid2-config.json | 14 --- ...prod-uid2-config.json => prod-config.json} | 30 +++--- scripts/gcp-oidc/entrypoint.sh | 64 ------------- scripts/gcp-oidc/gcp.py | 95 +++++++++++++++++++ 10 files changed, 180 insertions(+), 144 deletions(-) create mode 100644 scripts/gcp-oidc/conf/integ-config.json delete mode 100644 scripts/gcp-oidc/conf/integ-uid2-config.json rename scripts/gcp-oidc/conf/{prod-uid2-config.json => prod-config.json} (98%) delete mode 100644 scripts/gcp-oidc/entrypoint.sh create mode 100644 scripts/gcp-oidc/gcp.py diff --git a/.github/workflows/publish-gcp-oidc-enclave-docker.yaml b/.github/workflows/publish-gcp-oidc-enclave-docker.yaml index 9f042a916..742d73b8e 100644 --- a/.github/workflows/publish-gcp-oidc-enclave-docker.yaml +++ b/.github/workflows/publish-gcp-oidc-enclave-docker.yaml @@ -93,6 +93,7 @@ jobs: echo "jar_version=$(mvn help:evaluate -Dexpression=project.version | grep -e '^[1-9][^\[]')" >> $GITHUB_OUTPUT echo "git_commit=$(git show --format="%h" --no-patch)" >> $GITHUB_OUTPUT cp -r target ${{ env.DOCKER_CONTEXT_PATH }}/ + cp scripts/confidential_compute.py ${{ env.DOCKER_CONTEXT_PATH }}/ - name: Log in to the Docker container registry uses: docker/login-action@v3 diff --git a/pom.xml b/pom.xml index 5ebed11c9..740f874c6 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.44.6 + 5.45.7-alpha-169-SNAPSHOT UTF-8 diff --git a/scripts/confidential_compute.py b/scripts/confidential_compute.py index 4c80be659..827ab8f6a 100644 --- a/scripts/confidential_compute.py +++ b/scripts/confidential_compute.py @@ -19,14 +19,14 @@ def __init__(self, error_name, provider, extra_message=None): urls = { "EC2": "https://unifiedid.com/docs/guides/operator-guide-aws-marketplace#uid2-operator-error-codes", "Azure": "https://unifiedid.com/docs/guides/operator-guide-azure-enclave#uid2-operator-error-codes", - "GCP": "https://unifiedid.com/docs/guides/operator-private-gcp-confidential-space#uid2-operator-error-codes", + "GCPEntrypoint": "https://unifiedid.com/docs/guides/operator-private-gcp-confidential-space#uid2-operator-error-codes", } url = urls.get(provider) super().__init__(f"{error_name}\n" + (extra_message if extra_message else "") + f"\nVisit {url} for more details") class MissingInstanceProfile(ConfidentialComputeStartupException): - def __init__(self, cls): - super().__init__(error_name=f"E01: {self.__class__.__name__}", provider=cls) + def __init__(self, cls, message = None): + super().__init__(error_name=f"E01: {self.__class__.__name__}", provider=cls, extra_message=message) class ConfigNotFound(ConfidentialComputeStartupException): def __init__(self, cls, message = None): diff --git a/scripts/gcp-oidc/Dockerfile b/scripts/gcp-oidc/Dockerfile index 76b302e30..5d03ee49b 100644 --- a/scripts/gcp-oidc/Dockerfile +++ b/scripts/gcp-oidc/Dockerfile @@ -5,7 +5,11 @@ LABEL "tee.launch_policy.allow_env_override"="API_TOKEN_SECRET_NAME,DEPLOYMENT_E LABEL "tee.launch_policy.log_redirect"="always" # Install Packages -RUN apk update && apk add jq +RUN apk update && apk add --no-cache jq python3 py3-pip && \ + python3 -m venv /venv && \ + . /venv/bin/activate && \ + pip install --no-cache-dir google-cloud-secret-manager google-auth google-api-core && \ + rm -rf /var/cache/apk/* WORKDIR /app EXPOSE 8080 @@ -18,7 +22,6 @@ ENV JAR_NAME=${JAR_NAME} ENV JAR_VERSION=${JAR_VERSION} ENV IMAGE_VERSION=${IMAGE_VERSION} ENV REGION=default -ENV LOKI_HOSTNAME=loki COPY ./target/${JAR_NAME}-${JAR_VERSION}-jar-with-dependencies.jar /app/${JAR_NAME}-${JAR_VERSION}.jar COPY ./target/${JAR_NAME}-${JAR_VERSION}-sources.jar /app @@ -28,9 +31,10 @@ COPY ./conf/*.xml /app/conf/ RUN tar xzvf /app/static.tar.gz --no-same-owner --no-same-permissions && rm -f /app/static.tar.gz -COPY ./entrypoint.sh /app/ -RUN chmod a+x /app/entrypoint.sh +COPY ./gcp.py /app/ +COPY ./confidential_compute.py /app +RUN chmod a+x /app/gcp.py RUN mkdir -p /opt/uid2 && chmod 777 -R /opt/uid2 && mkdir -p /app && chmod 705 -R /app && mkdir -p /app/file-uploads && chmod 777 -R /app/file-uploads -CMD ["/app/entrypoint.sh"] +CMD ["/venv/bin/python", "/app/gcp.py"] diff --git a/scripts/gcp-oidc/conf/default-config.json b/scripts/gcp-oidc/conf/default-config.json index 6a65ee2d0..5d54e792d 100644 --- a/scripts/gcp-oidc/conf/default-config.json +++ b/scripts/gcp-oidc/conf/default-config.json @@ -1,42 +1,42 @@ -{ - "service_verbose": true, - "service_instances": 12, - "core_s3_bucket": null, - "core_attest_url": null, - "core_api_token": null, - "storage_mock": false, - "optout_s3_bucket": null, - "optout_s3_folder": "optout/", - "optout_s3_path_compat": false, - "optout_data_dir": "/opt/uid2/operator-optout/", - "optout_api_token": null, - "optout_api_uri": null, - "optout_bloom_filter_size": 8192, - "optout_delta_rotate_interval": 300, - "optout_delta_backtrack_in_days": 1, - "optout_partition_interval": 86400, - "optout_max_partitions": 30, - "optout_heap_default_capacity": 8192, - "cloud_download_threads": 8, - "cloud_upload_threads": 2, - "cloud_refresh_interval": 60, - "sites_metadata_path": "sites/metadata.json", - "clients_metadata_path": "clients/metadata.json", - "client_side_keypairs_metadata_path": "client_side_keypairs/metadata.json", - "keysets_metadata_path": "keysets/metadata.json", - "keyset_keys_metadata_path": "keyset_keys/metadata.json", - "salts_metadata_path": "salts/metadata.json", - "services_metadata_path": "services/metadata.json", - "service_links_metadata_path": "service_links/metadata.json", - "optout_metadata_path": null, - "enclave_platform": "gcp-oidc", - "optout_inmem_cache": true, - "identity_token_expires_after_seconds": 86400, - "refresh_token_expires_after_seconds": 2592000, - "refresh_identity_token_after_seconds": 3600, - "allow_legacy_api": false, - "failure_shutdown_wait_hours": 120, - "sharing_token_expiry_seconds": 2592000, - "validate_service_links": false, - "operator_type": "private" -} +{ + "service_verbose": true, + "service_instances": 12, + "core_s3_bucket": null, + "core_attest_url": null, + "core_api_token": null, + "storage_mock": false, + "optout_s3_bucket": null, + "optout_s3_folder": "optout/", + "optout_s3_path_compat": false, + "optout_data_dir": "/opt/uid2/operator-optout/", + "optout_api_token": null, + "optout_api_uri": null, + "optout_bloom_filter_size": 8192, + "optout_delta_rotate_interval": 300, + "optout_delta_backtrack_in_days": 1, + "optout_partition_interval": 86400, + "optout_max_partitions": 30, + "optout_heap_default_capacity": 8192, + "cloud_download_threads": 8, + "cloud_upload_threads": 2, + "cloud_refresh_interval": 60, + "sites_metadata_path": "sites/metadata.json", + "clients_metadata_path": "clients/metadata.json", + "client_side_keypairs_metadata_path": "client_side_keypairs/metadata.json", + "keysets_metadata_path": "keysets/metadata.json", + "keyset_keys_metadata_path": "keyset_keys/metadata.json", + "salts_metadata_path": "salts/metadata.json", + "services_metadata_path": "services/metadata.json", + "service_links_metadata_path": "service_links/metadata.json", + "optout_metadata_path": null, + "enclave_platform": "gcp-oidc", + "optout_inmem_cache": true, + "identity_token_expires_after_seconds": 86400, + "refresh_token_expires_after_seconds": 2592000, + "refresh_identity_token_after_seconds": 3600, + "allow_legacy_api": false, + "failure_shutdown_wait_hours": 120, + "sharing_token_expiry_seconds": 2592000, + "validate_service_links": false, + "operator_type": "private" +} \ No newline at end of file diff --git a/scripts/gcp-oidc/conf/integ-config.json b/scripts/gcp-oidc/conf/integ-config.json new file mode 100644 index 000000000..48f13b5d6 --- /dev/null +++ b/scripts/gcp-oidc/conf/integ-config.json @@ -0,0 +1,14 @@ +{ + "sites_metadata_path": "https://core.uidapi.com/sites/refresh", + "clients_metadata_path": "https://core.uidapi.com/clients/refresh", + "keysets_metadata_path": "https://core.uidapi.com/key/keyset/refresh", + "keyset_keys_metadata_path": "https://core.uidapi.com/key/keyset-keys/refresh", + "client_side_keypairs_metadata_path": "https://core.uidapi.com/client_side_keypairs/refresh", + "salts_metadata_path": "https://core.uidapi.com/salt/refresh", + "services_metadata_path": "https://core.uidapi.com/services/refresh", + "service_links_metadata_path": "https://core.uidapi.com/service_links/refresh", + "optout_metadata_path": "https://optout.uidapi.com/optout/refresh", + "core_attest_url": "https://core.uidapi.com/attest", + "optout_api_uri": "https://optout.uidapi.com/optout/replicate", + "optout_s3_folder": "uid-optout-integ/" +} \ No newline at end of file diff --git a/scripts/gcp-oidc/conf/integ-uid2-config.json b/scripts/gcp-oidc/conf/integ-uid2-config.json deleted file mode 100644 index 935514b5a..000000000 --- a/scripts/gcp-oidc/conf/integ-uid2-config.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "sites_metadata_path": "https://core.uidapi.com/sites/refresh", - "clients_metadata_path": "https://core.uidapi.com/clients/refresh", - "keysets_metadata_path": "https://core.uidapi.com/key/keyset/refresh", - "keyset_keys_metadata_path": "https://core.uidapi.com/key/keyset-keys/refresh", - "client_side_keypairs_metadata_path": "https://core.uidapi.com/client_side_keypairs/refresh", - "salts_metadata_path": "https://core.uidapi.com/salt/refresh", - "services_metadata_path": "https://core.uidapi.com/services/refresh", - "service_links_metadata_path": "https://core.uidapi.com/service_links/refresh", - "optout_metadata_path": "https://optout.uidapi.com/optout/refresh", - "core_attest_url": "https://core.uidapi.com/attest", - "optout_api_uri": "https://optout.uidapi.com/optout/replicate", - "optout_s3_folder": "uid-optout-integ/" -} diff --git a/scripts/gcp-oidc/conf/prod-uid2-config.json b/scripts/gcp-oidc/conf/prod-config.json similarity index 98% rename from scripts/gcp-oidc/conf/prod-uid2-config.json rename to scripts/gcp-oidc/conf/prod-config.json index f5445a9ec..18ca8769a 100644 --- a/scripts/gcp-oidc/conf/prod-uid2-config.json +++ b/scripts/gcp-oidc/conf/prod-config.json @@ -1,15 +1,15 @@ -{ - "sites_metadata_path": "https://core.uidapi.com/sites/refresh", - "clients_metadata_path": "https://core.uidapi.com/clients/refresh", - "keysets_metadata_path": "https://core.uidapi.com/key/keyset/refresh", - "keyset_keys_metadata_path": "https://core.uidapi.com/key/keyset-keys/refresh", - "client_side_keypairs_metadata_path": "https://core.uidapi.com/client_side_keypairs/refresh", - "salts_metadata_path": "https://core.uidapi.com/salt/refresh", - "services_metadata_path": "https://core.uidapi.com/services/refresh", - "service_links_metadata_path": "https://core.uidapi.com/service_links/refresh", - "optout_metadata_path": "https://optout.uidapi.com/optout/refresh", - "core_attest_url": "https://core.uidapi.com/attest", - "optout_api_uri": "https://optout.uidapi.com/optout/replicate", - "optout_s3_folder": "optout-v2/", - "identity_token_expires_after_seconds": 259200 -} +{ + "sites_metadata_path": "https://core.uidapi.com/sites/refresh", + "clients_metadata_path": "https://core.uidapi.com/clients/refresh", + "keysets_metadata_path": "https://core.uidapi.com/key/keyset/refresh", + "keyset_keys_metadata_path": "https://core.uidapi.com/key/keyset-keys/refresh", + "client_side_keypairs_metadata_path": "https://core.uidapi.com/client_side_keypairs/refresh", + "salts_metadata_path": "https://core.uidapi.com/salt/refresh", + "services_metadata_path": "https://core.uidapi.com/services/refresh", + "service_links_metadata_path": "https://core.uidapi.com/service_links/refresh", + "optout_metadata_path": "https://optout.uidapi.com/optout/refresh", + "core_attest_url": "https://core.uidapi.com/attest", + "optout_api_uri": "https://optout.uidapi.com/optout/replicate", + "optout_s3_folder": "optout-v2/", + "identity_token_expires_after_seconds": 259200 +} diff --git a/scripts/gcp-oidc/entrypoint.sh b/scripts/gcp-oidc/entrypoint.sh deleted file mode 100644 index 133b54486..000000000 --- a/scripts/gcp-oidc/entrypoint.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/sh -# -# This script must be compatible with Ash (provided in eclipse-temurin Docker image) and Bash - -# -- set API tokens -if [ -z "${API_TOKEN_SECRET_NAME}" ]; then - echo "API_TOKEN_SECRET_NAME cannot be empty" - exit 1 -fi - -if [ -z "${CORE_BASE_URL}" ]; then - echo "CORE_BASE_URL cannot be empty" - exit 1 -fi - -if [ -z "${OPTOUT_BASE_URL}" ]; then - echo "OPTOUT_BASE_URL cannot be empty" - exit 1 -fi - -export gcp_secret_version_name="${API_TOKEN_SECRET_NAME}" - -# -- locate config file -if [ -z "${DEPLOYMENT_ENVIRONMENT}" ]; then - echo "DEPLOYMENT_ENVIRONMENT cannot be empty" - exit 1 -fi -if [ "${DEPLOYMENT_ENVIRONMENT}" != 'prod' -a "${DEPLOYMENT_ENVIRONMENT}" != 'integ' ]; then - echo "Unrecognized DEPLOYMENT_ENVIRONMENT ${DEPLOYMENT_ENVIRONMENT}" - exit 1 -fi - -TARGET_CONFIG="/app/conf/${DEPLOYMENT_ENVIRONMENT}-uid2-config.json" -if [ ! -f "${TARGET_CONFIG}" ]; then - echo "Unrecognized config ${TARGET_CONFIG}" - exit 1 -fi - -FINAL_CONFIG="/tmp/final-config.json" -echo "-- copying ${TARGET_CONFIG} to ${FINAL_CONFIG}" -cp ${TARGET_CONFIG} ${FINAL_CONFIG} -if [ $? -ne 0 ]; then - echo "Failed to create ${FINAL_CONFIG} with error code $?" - exit 1 -fi - -# -- using hardcoded domains is fine because they should not be changed frequently -echo "-- replacing URLs by ${CORE_BASE_URL} and ${OPTOUT_BASE_URL}" -sed -i "s#https://core.uidapi.com#${CORE_BASE_URL}#g" ${FINAL_CONFIG} - -sed -i "s#https://optout.uidapi.com#${OPTOUT_BASE_URL}#g" ${FINAL_CONFIG} - - -cat $FINAL_CONFIG - -# -- start operator -echo "-- starting java application" -java \ - -XX:MaxRAMPercentage=95 -XX:-UseCompressedOops -XX:+PrintFlagsFinal \ - -Djava.security.egd=file:/dev/./urandom \ - -Dvertx.logger-delegate-factory-class-name=io.vertx.core.logging.SLF4JLogDelegateFactory \ - -Dlogback.configurationFile=/app/conf/logback.xml \ - -Dvertx-config-path=${FINAL_CONFIG} \ - -jar ${JAR_NAME}-${JAR_VERSION}.jar diff --git a/scripts/gcp-oidc/gcp.py b/scripts/gcp-oidc/gcp.py new file mode 100644 index 000000000..a2a4a5013 --- /dev/null +++ b/scripts/gcp-oidc/gcp.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 + +import os +import shutil +from typing import Dict +import sys +from google.cloud import secretmanager +from google.auth import default +from google.auth.exceptions import DefaultCredentialsError +from google.api_core.exceptions import PermissionDenied, NotFound + +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from confidential_compute import ConfidentialCompute, ConfidentialComputeConfig, MissingConfig, ConfigNotFound, MissingInstanceProfile, ConfidentialComputeStartupException + +class GCPEntrypoint(ConfidentialCompute): + + def __init__(self): + super().__init__() + + def _get_secret(self, secret_identifier=None) -> ConfidentialComputeConfig: + keys_mapping = { + "core_base_url": "CORE_BASE_URL", + "optout_base_url": "OPTOUT_BASE_URL", + "environment": "DEPLOYMENT_ENVIRONMENT", + "skip_validations": "SKIP_VALIDATIONS", + "debug_mode": "DEBUG_MODE", + } + config: ConfidentialComputeConfig = { + key: (os.environ[env_var].lower() == "true" if key in ["skip_validations", "debug_mode"] else os.environ[env_var]) + for key, env_var in keys_mapping.items() if env_var in os.environ + } + + if not os.getenv("API_TOKEN_SECRET_NAME"): + raise MissingConfig(self.__class__.__name__, ["API_TOKEN_SECRET_NAME"]) + try: + client = secretmanager.SecretManagerServiceClient() + secret_version_name = f"{os.getenv("API_TOKEN_SECRET_NAME")}" + response = client.access_secret_version(name=secret_version_name) + secret_value = response.payload.data.decode("UTF-8") + except (PermissionDenied, DefaultCredentialsError) as e: + raise MissingInstanceProfile(self.__class__.__name__, str(e)) + except NotFound: + raise ConfigNotFound(self.__class__.__name__, f"Secret Manager {os.getenv("API_TOKEN_SECRET_NAME")}") + config["api_token"] = secret_value + return config + + def __populate_operator_config(self, destination): + target_config = f"/app/conf/{self.configs["environment"].lower()}-config.json" + shutil.copy(target_config, destination) + with open(destination, 'r') as file: + config = file.read() + config = config.replace("https://core.uidapi.com", self.configs.get("core_base_url")) + config = config.replace("https://optout.uidapi.com", self.configs.get("optout_base_url")) + with open(destination, 'w') as file: + file.write(config) + + def _setup_auxiliaries(self) -> None: + """ No Auxiliariy service required for GCP Confidential compute. """ + pass + + def _validate_auxiliaries(self) -> None: + """ No Auxiliariy service required for GCP Confidential compute. """ + pass + + def run_compute(self) -> None: + self.configs = self._get_secret('read_from_env_vars') + print(f"Fetched configs") + if not self.configs.get("skip_validations"): + self.validate_configuration() + config_locaton = "/tmp/final-config.json" + self.__populate_operator_config(config_locaton) + os.environ["gcp_secret_version_name"] = os.getenv("API_TOKEN_SECRET_NAME") + java_command = [ + "java", + "-XX:MaxRAMPercentage=95", + "-XX:-UseCompressedOops", + "-XX:+PrintFlagsFinal", + "-Djava.security.egd=file:/dev/./urandom", + "-Dvertx.logger-delegate-factory-class-name=io.vertx.core.logging.SLF4JLogDelegateFactory", + "-Dlogback.configurationFile=/app/conf/logback.xml", + f"-Dvertx-config-path={config_locaton}", + "-jar", + f"{os.getenv("JAR_NAME")}-{os.getenv("JAR_VERSION")}.jar" + ] + self.run_command(java_command) + +if __name__ == "__main__": + try: + gcp = GCP() + gcp.run_compute() + except ConfidentialComputeStartupException as e: + print("Failed starting up Confidential Compute. Please checks the logs for errors and retry \n", e) + except Exception as e: + print("Unexpected failure while starting up Confidential Compute. Please contact UID support team with this log \n ", e) + From 917b0a1ad91ef06f2b644f7a19b982b7b0b53a96 Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Wed, 22 Jan 2025 18:04:05 +1100 Subject: [PATCH 220/431] Update fallback to static config for running locally - ConfigRetrieverFactory now creates ConfigRetriever from bootstrap config - Add runtime-config-defaults.json to serve as defaults for running locally --- conf/default-config.json | 3 +- conf/feat-flag/feat-flag.json | 14 ++++- conf/integ-config.json | 10 +++- conf/local-config.json | 9 ++- conf/local-e2e-docker-public-config.json | 12 +++- conf/runtime-config-defaults.json | 6 ++ src/main/java/com/uid2/operator/Const.java | 1 - src/main/java/com/uid2/operator/Main.java | 38 ++++++------- .../service/ConfigRetrieverFactory.java | 56 ++++--------------- .../operator/ConfigServiceManagerTest.java | 2 - .../com/uid2/operator/ConfigServiceTest.java | 31 +++++++--- 11 files changed, 98 insertions(+), 84 deletions(-) create mode 100644 conf/runtime-config-defaults.json diff --git a/conf/default-config.json b/conf/default-config.json index a72c1f9df..a57739a7f 100644 --- a/conf/default-config.json +++ b/conf/default-config.json @@ -35,6 +35,5 @@ "enclave_platform": null, "failure_shutdown_wait_hours": 120, "sharing_token_expiry_seconds": 2592000, - "operator_type": "public", - "config_scan_period_ms": 300000 + "operator_type": "public" } diff --git a/conf/feat-flag/feat-flag.json b/conf/feat-flag/feat-flag.json index aaf4b6db4..45e2f6d7c 100644 --- a/conf/feat-flag/feat-flag.json +++ b/conf/feat-flag/feat-flag.json @@ -1,3 +1,15 @@ { - "remote_config_feature_flag": false + "remote_config": { + "enabled": true, + "runtime_config_store": { + "type": "json", + "config": { + "identity_token_expires_after_seconds": 3600, + "refresh_token_expires_after_seconds": 86400, + "refresh_identity_token_after_seconds": 900, + "sharing_token_expiry_seconds": 2592000 + }, + "config_scan_period_ms": -1 + } + } } \ No newline at end of file diff --git a/conf/integ-config.json b/conf/integ-config.json index 4aa307c35..8f7fa2385 100644 --- a/conf/integ-config.json +++ b/conf/integ-config.json @@ -15,5 +15,13 @@ "optout_api_uri": "http://localhost:8081/optout/replicate", "salts_expired_shutdown_hours": 12, "operator_type": "public", - "core_operator_config_path": "http://localhost:8088/operator/config" + "runtime_config_store": { + "type": "http", + "config" : { + "host": "localhost", + "port": 8088, + "path": "/operator/config" + }, + "config_scan_period_ms": 300000 + } } \ No newline at end of file diff --git a/conf/local-config.json b/conf/local-config.json index 76b0395da..f1abb67f7 100644 --- a/conf/local-config.json +++ b/conf/local-config.json @@ -37,5 +37,12 @@ "client_side_token_generate_log_invalid_http_origins": true, "salts_expired_shutdown_hours": 12, "operator_type": "public", - "core_operator_config_path": "http://localhost:8088/operator/config" + "runtime_config_store": { + "type": "file", + "config" : { + "path": "conf/runtime-config-defaults.json", + "format": "json" + }, + "config_scan_period_ms": -1 + } } diff --git a/conf/local-e2e-docker-public-config.json b/conf/local-e2e-docker-public-config.json index a145c4d17..3357a2b85 100644 --- a/conf/local-e2e-docker-public-config.json +++ b/conf/local-e2e-docker-public-config.json @@ -33,6 +33,14 @@ "optout_status_api_enabled": true, "cloud_refresh_interval": 30, "salts_expired_shutdown_hours": 12, - "operator_type": "public" - + "operator_type": "public", + "runtime_config_store": { + "type": "http", + "config" : { + "host": "core", + "port": 8088, + "path": "/operator/config" + }, + "config_scan_period_ms": 300000 + } } diff --git a/conf/runtime-config-defaults.json b/conf/runtime-config-defaults.json new file mode 100644 index 000000000..817d714dd --- /dev/null +++ b/conf/runtime-config-defaults.json @@ -0,0 +1,6 @@ +{ + "identity_token_expires_after_seconds": 3600, + "refresh_token_expires_after_seconds": 86400, + "refresh_identity_token_after_seconds": 900, + "sharing_token_expiry_seconds": 2592000 +} \ No newline at end of file diff --git a/src/main/java/com/uid2/operator/Const.java b/src/main/java/com/uid2/operator/Const.java index 7450b7389..3433d4617 100644 --- a/src/main/java/com/uid2/operator/Const.java +++ b/src/main/java/com/uid2/operator/Const.java @@ -30,7 +30,6 @@ public class Config extends com.uid2.shared.Const.Config { public static final String MaxInvalidPaths = "logging_limit_max_invalid_paths_per_interval"; public static final String MaxVersionBucketsPerSite = "logging_limit_max_version_buckets_per_site"; - public static final String CoreConfigPath = "core_operator_config_path"; public static final String ConfigScanPeriodMs = "config_scan_period_ms"; public static final String IdentityV3 = "identity_v3"; public static final String RemoteConfigFeatureFlag = "remote_config_feature_flag"; diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index fb120ea4b..5e4c7e0a9 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -57,6 +57,7 @@ import java.util.*; import java.util.function.Supplier; +import static com.uid2.operator.Const.Config.ConfigScanPeriodMs; import static io.micrometer.core.instrument.Metrics.globalRegistry; public class Main { @@ -271,36 +272,31 @@ private void run() throws Exception { this.createVertxEventLoopsMetric(); ConfigRetrieverFactory configRetrieverFactory = new ConfigRetrieverFactory(); - ConfigRetriever staticConfigRetriever = configRetrieverFactory.createJsonRetriever(vertx, config); - ConfigRetriever dynamicConfigRetriever = configRetrieverFactory.createRemoteConfigRetriever(vertx, config, this.createOperatorKeyRetriever().retrieve()); - Future staticConfigFuture = ConfigService.create(staticConfigRetriever); + ConfigRetriever dynamicConfigRetriever = configRetrieverFactory.create(vertx, config.getJsonObject("runtime_config_store"), this.createOperatorKeyRetriever().retrieve()); Future dynamicConfigFuture = ConfigService.create(dynamicConfigRetriever); - ConfigRetriever featureFlagConfigRetriever = configRetrieverFactory.createFileRetriever( + ConfigRetriever featureFlagConfigRetriever = configRetrieverFactory.create( vertx, - "conf/feat-flag/feat-flag.json" + new JsonObject() + .put("type", "file") + .put("config", new JsonObject() + .put("path", "conf/feat-flag/feat-flag.json") + .put("format", "json")) + .put(ConfigScanPeriodMs, 60000), + "" ); - Future featureFlagFuture = featureFlagConfigRetriever.getConfig(); - - featureFlagFuture.compose(featureFlagConfig -> { - boolean featureFlag = featureFlagConfig.getBoolean(Const.Config.RemoteConfigFeatureFlag, true); - // use static config if dynamic config fails and feature flag toggled off - return dynamicConfigFuture - .recover(throwable -> { - if (!featureFlag) { - LOGGER.warn("Dynamic config service creation failed: ", throwable); - return staticConfigFuture; - } else { - return Future.failedFuture(new Exception("Dynamic config service creation failed and feature flag is enabled: ", throwable)); - } - }) - .compose(dynamicConfigService -> Future.all(Future.succeededFuture(dynamicConfigService), staticConfigFuture)); + + featureFlagConfigRetriever.getConfig().compose(featureFlagConfig -> { + JsonObject remoteConfigJson = featureFlagConfig.getJsonObject("remote_config"); + JsonObject featureFlagBootstrapConfig = remoteConfigJson.getJsonObject("runtime_config_store"); + ConfigRetriever staticConfigRetriever = configRetrieverFactory.create(vertx, featureFlagBootstrapConfig, ""); + return Future.all(dynamicConfigFuture, ConfigService.create(staticConfigRetriever)); }) .compose(configServiceManagerCompositeFuture -> { ConfigService dynamicConfigService = configServiceManagerCompositeFuture.resultAt(0); ConfigService staticConfigService = configServiceManagerCompositeFuture.resultAt(1); - boolean featureFlag = featureFlagConfigRetriever.getCachedConfig().getBoolean(Const.Config.RemoteConfigFeatureFlag, false); + boolean featureFlag = featureFlagConfigRetriever.getCachedConfig().getJsonObject("remote_config").getBoolean(Const.Config.RemoteConfigFeatureFlag, false); ConfigServiceManager configServiceManager = new ConfigServiceManager( vertx, dynamicConfigService, staticConfigService, featureFlag); diff --git a/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java b/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java index 488c9d69e..a04690e96 100644 --- a/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java +++ b/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java @@ -6,56 +6,24 @@ import io.vertx.core.Vertx; import io.vertx.core.json.JsonObject; -import java.net.URI; -import java.net.URISyntaxException; - import static com.uid2.operator.Const.Config.ConfigScanPeriodMs; -import static com.uid2.operator.Const.Config.CoreConfigPath; public class ConfigRetrieverFactory { - public ConfigRetriever createRemoteConfigRetriever(Vertx vertx, JsonObject bootstrapConfig, String operatorKey) throws URISyntaxException { - String configPath = bootstrapConfig.getString(CoreConfigPath); - URI uri = new URI(configPath); - - ConfigStoreOptions httpStore = new ConfigStoreOptions() - .setType("http") - .setOptional(true) - .setConfig(new JsonObject() - .put("host", uri.getHost()) - .put("port", uri.getPort()) - .put("path", uri.getPath()) - .put("headers", new JsonObject() - .put("Authorization", "Bearer " + operatorKey))); + public ConfigRetriever create(Vertx vertx, JsonObject bootstrapConfig, String operatorKey) { + String type = bootstrapConfig.getString("type"); + JsonObject storeConfig = bootstrapConfig.getJsonObject("config"); + if (type.equals("http")) { + storeConfig.put("headers", new JsonObject() + .put("Authorization", "Bearer " + operatorKey)); + } + + ConfigStoreOptions storeOptions = new ConfigStoreOptions() + .setType(type) + .setConfig(storeConfig); ConfigRetrieverOptions retrieverOptions = new ConfigRetrieverOptions() .setScanPeriod(bootstrapConfig.getLong(ConfigScanPeriodMs)) - .addStore(httpStore); - - return ConfigRetriever.create(vertx, retrieverOptions); - } - - public ConfigRetriever createJsonRetriever(Vertx vertx, JsonObject config) { - ConfigStoreOptions jsonStore = new ConfigStoreOptions() - .setType("json") - .setConfig(config); - - ConfigRetrieverOptions retrieverOptions = new ConfigRetrieverOptions() - .setScanPeriod(-1) - .addStore(jsonStore); - - - return ConfigRetriever.create(vertx, retrieverOptions); - } - - public ConfigRetriever createFileRetriever(Vertx vertx, String path) { - ConfigStoreOptions fileStore = new ConfigStoreOptions() - .setType("file") - .setConfig(new JsonObject() - .put("path", path) - .put("format", "json")); - - ConfigRetrieverOptions retrieverOptions = new ConfigRetrieverOptions() - .addStore(fileStore); + .addStore(storeOptions); return ConfigRetriever.create(vertx, retrieverOptions); } diff --git a/src/test/java/com/uid2/operator/ConfigServiceManagerTest.java b/src/test/java/com/uid2/operator/ConfigServiceManagerTest.java index 188949562..74bba3400 100644 --- a/src/test/java/com/uid2/operator/ConfigServiceManagerTest.java +++ b/src/test/java/com/uid2/operator/ConfigServiceManagerTest.java @@ -24,8 +24,6 @@ public class ConfigServiceManagerTest { @BeforeEach void setUp(Vertx vertx) { bootstrapConfig = new JsonObject() - .put(CoreConfigPath, "/operator/config") - .put(ConfigScanPeriodMs, 300000) .put(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, 3600) .put(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, 7200) .put(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS, 1800) diff --git a/src/test/java/com/uid2/operator/ConfigServiceTest.java b/src/test/java/com/uid2/operator/ConfigServiceTest.java index d7ff8e9e7..8c751e57e 100644 --- a/src/test/java/com/uid2/operator/ConfigServiceTest.java +++ b/src/test/java/com/uid2/operator/ConfigServiceTest.java @@ -16,8 +16,6 @@ import io.vertx.junit5.VertxTestContext; import org.junit.jupiter.api.extension.ExtendWith; -import java.net.URISyntaxException; - import static com.uid2.operator.Const.Config.*; import static com.uid2.operator.service.UIDOperatorService.*; import static org.junit.jupiter.api.Assertions.*; @@ -27,6 +25,7 @@ class ConfigServiceTest { private Vertx vertx; private JsonObject bootstrapConfig; + private JsonObject runtimeConfig; private HttpServer server; private ConfigRetrieverFactory configRetrieverFactory; @@ -34,8 +33,14 @@ class ConfigServiceTest { void setUp() { vertx = Vertx.vertx(); bootstrapConfig = new JsonObject() - .put(CoreConfigPath, "http://localhost:8088/operator/config") - .put(ConfigScanPeriodMs, 300000) + .put("type", "http") + .put("config", new JsonObject() + .put("host", "localhost") + .put("port", 8088) + .put("path", "/operator/config")) + .put(ConfigScanPeriodMs, 300000); + + runtimeConfig = new JsonObject() .put(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, 3600) .put(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, 7200) .put(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS, 1800) @@ -80,9 +85,9 @@ private Future startMockServer(JsonObject config) { } @Test - void testGetConfig(VertxTestContext testContext) throws URISyntaxException { - ConfigRetriever configRetriever = configRetrieverFactory.createRemoteConfigRetriever(vertx, bootstrapConfig, ""); - JsonObject httpStoreConfig = bootstrapConfig; + void testGetConfig(VertxTestContext testContext) { + ConfigRetriever configRetriever = configRetrieverFactory.create(vertx, bootstrapConfig, ""); + JsonObject httpStoreConfig = runtimeConfig; startMockServer(httpStoreConfig) .compose(v -> ConfigService.create(configRetriever)) .compose(configService -> { @@ -100,7 +105,11 @@ void testInvalidConfigRevertsToPrevious(VertxTestContext testContext) { JsonObject invalidConfig = new JsonObject() .put(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, 1000) .put(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, 2000); - ConfigRetriever spyRetriever = spy(configRetrieverFactory.createJsonRetriever(vertx, invalidConfig)); + JsonObject jsonBootstrapConfig = new JsonObject() + .put("type", "json") + .put("config", invalidConfig) + .put(ConfigScanPeriodMs, -1); + ConfigRetriever spyRetriever = spy(configRetrieverFactory.create(vertx, jsonBootstrapConfig, "")); when(spyRetriever.getCachedConfig()).thenReturn(lastConfig); ConfigService.create(spyRetriever) .compose(configService -> { @@ -116,7 +125,11 @@ void testFirstInvalidConfigThrowsRuntimeException(VertxTestContext testContext) JsonObject invalidConfig = new JsonObject() .put(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, 1000) .put(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, 2000); - ConfigRetriever configRetriever = configRetrieverFactory.createJsonRetriever(vertx, invalidConfig); + JsonObject jsonBootstrapConfig = new JsonObject() + .put("type", "json") + .put("config", invalidConfig) + .put(ConfigScanPeriodMs, -1); + ConfigRetriever configRetriever = configRetrieverFactory.create(vertx, jsonBootstrapConfig, ""); ConfigService.create(configRetriever) .onComplete(testContext.failing(throwable -> { assertThrows(RuntimeException.class, () -> { From 12fd4b384eb54f31abba7b62816c6491a2caae3d Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Thu, 23 Jan 2025 14:46:30 +1100 Subject: [PATCH 221/431] Address Review Comments: - Refactor Const names in Const.Config - Update ConfigValidatorUtil log messages --- src/main/java/com/uid2/operator/Const.java | 6 ++--- src/main/java/com/uid2/operator/Main.java | 8 +++---- .../service/ConfigRetrieverFactory.java | 4 ++-- .../uid2/operator/service/ConfigService.java | 3 +++ .../operator/service/ConfigValidatorUtil.java | 23 +++++++++++++------ .../operator/vertx/UIDOperatorVerticle.java | 2 +- .../com/uid2/operator/ConfigServiceTest.java | 9 ++++---- .../uid2/operator/UIDOperatorServiceTest.java | 14 +++++------ .../operator/UIDOperatorVerticleTest.java | 2 +- .../operator/benchmark/BenchmarkCommon.java | 4 ++-- 10 files changed, 44 insertions(+), 31 deletions(-) diff --git a/src/main/java/com/uid2/operator/Const.java b/src/main/java/com/uid2/operator/Const.java index 3433d4617..4eb52198d 100644 --- a/src/main/java/com/uid2/operator/Const.java +++ b/src/main/java/com/uid2/operator/Const.java @@ -30,8 +30,8 @@ public class Config extends com.uid2.shared.Const.Config { public static final String MaxInvalidPaths = "logging_limit_max_invalid_paths_per_interval"; public static final String MaxVersionBucketsPerSite = "logging_limit_max_version_buckets_per_site"; - public static final String ConfigScanPeriodMs = "config_scan_period_ms"; - public static final String IdentityV3 = "identity_v3"; - public static final String RemoteConfigFeatureFlag = "remote_config_feature_flag"; + public static final String ConfigScanPeriodMsProp = "config_scan_period_ms"; + public static final String IdentityV3Prop = "identity_v3"; + public static final String RemoteConfigFeatureFlagProp = "remote_config_feature_flag"; } } diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index 5e4c7e0a9..57560ac6e 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -57,7 +57,7 @@ import java.util.*; import java.util.function.Supplier; -import static com.uid2.operator.Const.Config.ConfigScanPeriodMs; +import static com.uid2.operator.Const.Config.ConfigScanPeriodMsProp; import static io.micrometer.core.instrument.Metrics.globalRegistry; public class Main { @@ -282,7 +282,7 @@ private void run() throws Exception { .put("config", new JsonObject() .put("path", "conf/feat-flag/feat-flag.json") .put("format", "json")) - .put(ConfigScanPeriodMs, 60000), + .put(ConfigScanPeriodMsProp, 60000), "" ); @@ -296,7 +296,7 @@ private void run() throws Exception { ConfigService dynamicConfigService = configServiceManagerCompositeFuture.resultAt(0); ConfigService staticConfigService = configServiceManagerCompositeFuture.resultAt(1); - boolean featureFlag = featureFlagConfigRetriever.getCachedConfig().getJsonObject("remote_config").getBoolean(Const.Config.RemoteConfigFeatureFlag, false); + boolean featureFlag = featureFlagConfigRetriever.getCachedConfig().getJsonObject("remote_config").getBoolean(Const.Config.RemoteConfigFeatureFlagProp, false); ConfigServiceManager configServiceManager = new ConfigServiceManager( vertx, dynamicConfigService, staticConfigService, featureFlag); @@ -346,7 +346,7 @@ private void run() throws Exception { private void setupFeatureFlagListener(ConfigServiceManager manager, ConfigRetriever retriever) { retriever.listen(change -> { JsonObject newConfig = change.getNewConfiguration(); - boolean useDynamicConfig = newConfig.getBoolean(Const.Config.RemoteConfigFeatureFlag, true); + boolean useDynamicConfig = newConfig.getBoolean(Const.Config.RemoteConfigFeatureFlagProp, true); manager.updateConfigService(useDynamicConfig).onComplete(update -> { if (update.succeeded()) { LOGGER.info("Remote config feature flag toggled successfully"); diff --git a/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java b/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java index a04690e96..e967b5c29 100644 --- a/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java +++ b/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java @@ -6,7 +6,7 @@ import io.vertx.core.Vertx; import io.vertx.core.json.JsonObject; -import static com.uid2.operator.Const.Config.ConfigScanPeriodMs; +import static com.uid2.operator.Const.Config.ConfigScanPeriodMsProp; public class ConfigRetrieverFactory { public ConfigRetriever create(Vertx vertx, JsonObject bootstrapConfig, String operatorKey) { @@ -22,7 +22,7 @@ public ConfigRetriever create(Vertx vertx, JsonObject bootstrapConfig, String op .setConfig(storeConfig); ConfigRetrieverOptions retrieverOptions = new ConfigRetrieverOptions() - .setScanPeriod(bootstrapConfig.getLong(ConfigScanPeriodMs)) + .setScanPeriod(bootstrapConfig.getLong(ConfigScanPeriodMsProp)) .addStore(storeOptions); return ConfigRetriever.create(vertx, retrieverOptions); diff --git a/src/main/java/com/uid2/operator/service/ConfigService.java b/src/main/java/com/uid2/operator/service/ConfigService.java index 4326fa1ad..4ac779a0a 100644 --- a/src/main/java/com/uid2/operator/service/ConfigService.java +++ b/src/main/java/com/uid2/operator/service/ConfigService.java @@ -50,11 +50,14 @@ private JsonObject configValidationHandler(JsonObject config) { Integer refreshExpiresAfter = config.getInteger(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS); Integer refreshIdentityAfter = config.getInteger(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS); Integer maxBidstreamLifetimeSeconds = config.getInteger(Const.Config.MaxBidstreamLifetimeSecondsProp, identityExpiresAfter); + Integer sharingTokenExpiry = config.getInteger(Const.Config.SharingTokenExpiryProp); isValid &= validateIdentityRefreshTokens(identityExpiresAfter, refreshExpiresAfter, refreshIdentityAfter); isValid &= validateBidstreamLifetime(maxBidstreamLifetimeSeconds, identityExpiresAfter); + isValid &= validateSharingTokenExpiry(sharingTokenExpiry); + if (!isValid) { logger.error("Failed to update config"); JsonObject lastConfig = this.getConfig(); diff --git a/src/main/java/com/uid2/operator/service/ConfigValidatorUtil.java b/src/main/java/com/uid2/operator/service/ConfigValidatorUtil.java index 1c2ae471d..2fbf874e4 100644 --- a/src/main/java/com/uid2/operator/service/ConfigValidatorUtil.java +++ b/src/main/java/com/uid2/operator/service/ConfigValidatorUtil.java @@ -4,41 +4,50 @@ import org.slf4j.LoggerFactory; import static com.uid2.operator.Const.Config.MaxBidstreamLifetimeSecondsProp; +import static com.uid2.operator.Const.Config.SharingTokenExpiryProp; import static com.uid2.operator.service.UIDOperatorService.*; public class ConfigValidatorUtil { private static final Logger logger = LoggerFactory.getLogger(ConfigValidatorUtil.class); - public static final String VALUES_ARE_NULL = "Required config values are null"; + public static final String VALUES_ARE_NULL = "One or more of the following required config values are null: "; public static Boolean validateIdentityRefreshTokens(Integer identityExpiresAfter, Integer refreshExpiresAfter, Integer refreshIdentityAfter) { boolean isValid = true; if (areValuesNull(identityExpiresAfter, refreshExpiresAfter, refreshIdentityAfter)) { - logger.error(VALUES_ARE_NULL); + logger.error(VALUES_ARE_NULL + IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS + ", " + REFRESH_TOKEN_EXPIRES_AFTER_SECONDS + ", " + REFRESH_IDENTITY_TOKEN_AFTER_SECONDS); return false; } if (refreshExpiresAfter < identityExpiresAfter) { - logger.error(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS + " must be >= " + IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS); + logger.error(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS + " ({}) < " + IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS + " ({})", refreshExpiresAfter, identityExpiresAfter); isValid = false; } if (identityExpiresAfter < refreshIdentityAfter) { - logger.error(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS + " must be >= " + REFRESH_IDENTITY_TOKEN_AFTER_SECONDS); + logger.error(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS + " ({}) < " + REFRESH_IDENTITY_TOKEN_AFTER_SECONDS + " ({})", identityExpiresAfter, refreshIdentityAfter); isValid = false; } if (refreshExpiresAfter < refreshIdentityAfter) { - logger.error(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS + " must be >= " + REFRESH_IDENTITY_TOKEN_AFTER_SECONDS); + logger.error(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS + " ({}) < " + REFRESH_IDENTITY_TOKEN_AFTER_SECONDS + " ({})", refreshExpiresAfter, refreshIdentityAfter); } return isValid; } public static Boolean validateBidstreamLifetime(Integer maxBidstreamLifetimeSeconds, Integer identityTokenExpiresAfterSeconds) { if (areValuesNull(maxBidstreamLifetimeSeconds, identityTokenExpiresAfterSeconds)) { - logger.error(VALUES_ARE_NULL); + logger.error(VALUES_ARE_NULL + MaxBidstreamLifetimeSecondsProp + ", " + IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS); return false; } if (maxBidstreamLifetimeSeconds < identityTokenExpiresAfterSeconds) { - logger.error(MaxBidstreamLifetimeSecondsProp + " must be >= " + IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS); + logger.error(MaxBidstreamLifetimeSecondsProp + " ({}) < " + IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS + " ({})", maxBidstreamLifetimeSeconds, identityTokenExpiresAfterSeconds); + return false; + } + return true; + } + + public static Boolean validateSharingTokenExpiry(Integer sharingTokenExpiry) { + if (areValuesNull(sharingTokenExpiry)) { + logger.error(VALUES_ARE_NULL + SharingTokenExpiryProp); return false; } return true; diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index 7422ad380..6643f924c 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -182,7 +182,7 @@ public UIDOperatorVerticle(IConfigService configService, this.optOutStatusApiEnabled = config.getBoolean(Const.Config.OptOutStatusApiEnabled, true); this.optOutStatusMaxRequestSize = config.getInteger(Const.Config.OptOutStatusMaxRequestSize, 5000); this.allowLegacyAPI = config.getBoolean(Const.Config.AllowLegacyAPIProp, true); - this.identityV3Enabled = config.getBoolean(IdentityV3, false); + this.identityV3Enabled = config.getBoolean(IdentityV3Prop, false); } @Override diff --git a/src/test/java/com/uid2/operator/ConfigServiceTest.java b/src/test/java/com/uid2/operator/ConfigServiceTest.java index 8c751e57e..473be6492 100644 --- a/src/test/java/com/uid2/operator/ConfigServiceTest.java +++ b/src/test/java/com/uid2/operator/ConfigServiceTest.java @@ -38,13 +38,14 @@ void setUp() { .put("host", "localhost") .put("port", 8088) .put("path", "/operator/config")) - .put(ConfigScanPeriodMs, 300000); + .put(ConfigScanPeriodMsProp, 300000); runtimeConfig = new JsonObject() .put(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, 3600) .put(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, 7200) .put(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS, 1800) - .put(MaxBidstreamLifetimeSecondsProp, 7200); + .put(MaxBidstreamLifetimeSecondsProp, 7200) + .put(SharingTokenExpiryProp, 3600); configRetrieverFactory = new ConfigRetrieverFactory(); @@ -108,7 +109,7 @@ void testInvalidConfigRevertsToPrevious(VertxTestContext testContext) { JsonObject jsonBootstrapConfig = new JsonObject() .put("type", "json") .put("config", invalidConfig) - .put(ConfigScanPeriodMs, -1); + .put(ConfigScanPeriodMsProp, -1); ConfigRetriever spyRetriever = spy(configRetrieverFactory.create(vertx, jsonBootstrapConfig, "")); when(spyRetriever.getCachedConfig()).thenReturn(lastConfig); ConfigService.create(spyRetriever) @@ -128,7 +129,7 @@ void testFirstInvalidConfigThrowsRuntimeException(VertxTestContext testContext) JsonObject jsonBootstrapConfig = new JsonObject() .put("type", "json") .put("config", invalidConfig) - .put(ConfigScanPeriodMs, -1); + .put(ConfigScanPeriodMsProp, -1); ConfigRetriever configRetriever = configRetrieverFactory.create(vertx, jsonBootstrapConfig, ""); ConfigService.create(configRetriever) .onComplete(testContext.failing(throwable -> { diff --git a/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java b/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java index a7b37a71a..1a0de54d1 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java @@ -22,7 +22,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import static com.uid2.operator.Const.Config.IdentityV3; +import static com.uid2.operator.Const.Config.IdentityV3Prop; import static org.junit.jupiter.api.Assertions.*; import org.junit.jupiter.params.ParameterizedTest; @@ -91,7 +91,7 @@ void setup() throws Exception { uid2Config.put(UIDOperatorService.IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS); uid2Config.put(UIDOperatorService.REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, REFRESH_TOKEN_EXPIRES_AFTER_SECONDS); uid2Config.put(UIDOperatorService.REFRESH_IDENTITY_TOKEN_AFTER_SECONDS, REFRESH_IDENTITY_TOKEN_AFTER_SECONDS); - uid2Config.put(IdentityV3, false); + uid2Config.put(IdentityV3Prop, false); uid2Service = new ExtendedUIDOperatorService( optOutStore, @@ -100,14 +100,14 @@ void setup() throws Exception { this.clock, IdentityScope.UID2, this.shutdownHandler::handleSaltRetrievalResponse, - uid2Config.getBoolean(IdentityV3) + uid2Config.getBoolean(IdentityV3Prop) ); euidConfig = new JsonObject(); euidConfig.put(UIDOperatorService.IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS); euidConfig.put(UIDOperatorService.REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, REFRESH_TOKEN_EXPIRES_AFTER_SECONDS); euidConfig.put(UIDOperatorService.REFRESH_IDENTITY_TOKEN_AFTER_SECONDS, REFRESH_IDENTITY_TOKEN_AFTER_SECONDS); - euidConfig.put(IdentityV3, true); + euidConfig.put(IdentityV3Prop, true); euidService = new ExtendedUIDOperatorService( optOutStore, @@ -116,7 +116,7 @@ void setup() throws Exception { this.clock, IdentityScope.EUID, this.shutdownHandler::handleSaltRetrievalResponse, - euidConfig.getBoolean(IdentityV3) + euidConfig.getBoolean(IdentityV3Prop) ); } @@ -783,7 +783,7 @@ void testExpiredSaltsNotifiesShutdownHandler(TestIdentityInputType type, String this.clock, IdentityScope.UID2, this.shutdownHandler::handleSaltRetrievalResponse, - uid2Config.getBoolean(IdentityV3) + uid2Config.getBoolean(IdentityV3Prop) ); UIDOperatorService euidService = new UIDOperatorService( @@ -793,7 +793,7 @@ void testExpiredSaltsNotifiesShutdownHandler(TestIdentityInputType type, String this.clock, IdentityScope.EUID, this.shutdownHandler::handleSaltRetrievalResponse, - euidConfig.getBoolean(IdentityV3) + euidConfig.getBoolean(IdentityV3Prop) ); when(this.optOutStore.getLatestEntry(any())).thenReturn(null); diff --git a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java index f064a0f75..3b8449715 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java @@ -157,7 +157,7 @@ private void setupConfig(JsonObject config) { config.put(Const.Config.SharingTokenExpiryProp, 60 * 60 * 24 * 30); config.put("identity_scope", getIdentityScope().toString()); - config.put(Const.Config.IdentityV3, useRawUidV3()); + config.put(Const.Config.IdentityV3Prop, useRawUidV3()); config.put("client_side_token_generate", true); config.put("key_sharing_endpoint_provide_app_names", true); config.put("client_side_token_generate_log_invalid_http_origins", true); diff --git a/src/test/java/com/uid2/operator/benchmark/BenchmarkCommon.java b/src/test/java/com/uid2/operator/benchmark/BenchmarkCommon.java index 111d17706..d5a2279bf 100644 --- a/src/test/java/com/uid2/operator/benchmark/BenchmarkCommon.java +++ b/src/test/java/com/uid2/operator/benchmark/BenchmarkCommon.java @@ -41,7 +41,7 @@ import java.util.List; import java.util.Random; -import static com.uid2.operator.Const.Config.IdentityV3; +import static com.uid2.operator.Const.Config.IdentityV3Prop; public class BenchmarkCommon { @@ -77,7 +77,7 @@ static IUIDOperatorService createUidOperatorService() throws Exception { Clock.systemUTC(), IdentityScope.UID2, null, - config.getBoolean(IdentityV3) + config.getBoolean(IdentityV3Prop) ); } From 3301aa4cf3d57016e4d1ea16b00ab7a2b980462f Mon Sep 17 00:00:00 2001 From: Xu Yang <58192524+clarkxuyang@users.noreply.github.com> Date: Thu, 23 Jan 2025 13:47:12 -0800 Subject: [PATCH 222/431] UID2-4719 change azure cc starting process (#1260) * UID2-4719 change azure cc starting process --- .../publish-azure-cc-enclave-docker.yaml | 1 + pom.xml | 2 +- scripts/aws/ec2.py | 40 ++-- scripts/azure-cc/Dockerfile | 37 +++- scripts/azure-cc/azureEntryPoint.py | 176 ++++++++++++++++++ scripts/confidential_compute.py | 46 +++-- scripts/gcp-oidc/gcp.py | 22 +-- 7 files changed, 263 insertions(+), 61 deletions(-) create mode 100644 scripts/azure-cc/azureEntryPoint.py diff --git a/.github/workflows/publish-azure-cc-enclave-docker.yaml b/.github/workflows/publish-azure-cc-enclave-docker.yaml index 0127a71f4..84e74f5cb 100644 --- a/.github/workflows/publish-azure-cc-enclave-docker.yaml +++ b/.github/workflows/publish-azure-cc-enclave-docker.yaml @@ -92,6 +92,7 @@ jobs: echo "jar_version=$(mvn help:evaluate -Dexpression=project.version | grep -e '^[1-9][^\[]')" >> $GITHUB_OUTPUT echo "git_commit=$(git show --format="%h" --no-patch)" >> $GITHUB_OUTPUT cp -r target ${{ env.DOCKER_CONTEXT_PATH }}/ + cp scripts/confidential_compute.py ${{ env.DOCKER_CONTEXT_PATH }}/ - name: Log in to the Docker container registry uses: docker/login-action@v3 diff --git a/pom.xml b/pom.xml index 740f874c6..c3f6067c4 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.45.7-alpha-169-SNAPSHOT + 5.45.8-alpha-172-SNAPSHOT UTF-8 diff --git a/scripts/aws/ec2.py b/scripts/aws/ec2.py index ee47f4556..31cc77461 100644 --- a/scripts/aws/ec2.py +++ b/scripts/aws/ec2.py @@ -16,7 +16,7 @@ import yaml sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from confidential_compute import ConfidentialCompute, ConfidentialComputeConfig, MissingInstanceProfile, ConfigNotFound, InvalidConfigValue, ConfidentialComputeStartupException +from confidential_compute import ConfidentialCompute, ConfidentialComputeConfig, MissingInstanceProfile, ApiTokenNotFound, InvalidConfigValue, ConfidentialComputeStartupException class AWSConfidentialComputeConfig(ConfidentialComputeConfig): enclave_memory_mb: int @@ -48,10 +48,10 @@ def get_meta_url(cls) -> str: return f"http://{cls.AWS_METADATA}/latest/dynamic/instance-identity/document" -class EC2(ConfidentialCompute): +class EC2EntryPoint(ConfidentialCompute): def __init__(self): - super().__init__() + self.configs: AWSConfidentialComputeConfig = {} def __get_aws_token(self) -> str: """Fetches a temporary AWS EC2 metadata token.""" @@ -74,38 +74,36 @@ def __get_current_region(self) -> str: except requests.RequestException as e: raise RuntimeError(f"Failed to fetch region: {e}") - def __validate_aws_specific_config(self, secret): - if "enclave_memory_mb" in secret or "enclave_cpu_count" in secret: + def __validate_aws_specific_config(self): + if "enclave_memory_mb" in self.configs or "enclave_cpu_count" in self.configs: max_capacity = self.__get_max_capacity() min_capacity = {"enclave_memory_mb": 11000, "enclave_cpu_count" : 2 } for key in ["enclave_memory_mb", "enclave_cpu_count"]: - if int(secret.get(key, 0)) > max_capacity.get(key): - raise ValueError(f"{key} value ({secret.get(key, 0)}) exceeds the maximum allowed ({max_capacity.get(key)}).") - if min_capacity.get(key) > int(secret.get(key, 10**9)): - raise ValueError(f"{key} value ({secret.get(key, 0)}) needs to be higher than the minimum required ({min_capacity.get(key)}).") + if int(self.configs.get(key, 0)) > max_capacity.get(key): + raise ValueError(f"{key} value ({self.configs.get(key, 0)}) exceeds the maximum allowed ({max_capacity.get(key)}).") + if min_capacity.get(key) > int(self.configs.get(key, 10**9)): + raise ValueError(f"{key} value ({self.configs.get(key, 0)}) needs to be higher than the minimum required ({min_capacity.get(key)}).") - def _get_secret(self, secret_identifier: str) -> AWSConfidentialComputeConfig: + def _set_confidential_config(self, secret_identifier: str) -> None: """Fetches a secret value from AWS Secrets Manager and adds defaults""" - def add_defaults(configs: Dict[str, any]) -> AWSConfidentialComputeConfig: + def add_defaults(configs: Dict[str, any]) -> None: """Adds default values to configuration if missing.""" default_capacity = self.__get_max_capacity() configs.setdefault("enclave_memory_mb", default_capacity["enclave_memory_mb"]) configs.setdefault("enclave_cpu_count", default_capacity["enclave_cpu_count"]) configs.setdefault("debug_mode", False) - return configs region = self.__get_current_region() print(f"Running in {region}") client = boto3.client("secretsmanager", region_name=region) try: - secret = add_defaults(json.loads(client.get_secret_value(SecretId=secret_identifier)["SecretString"])) - self.__validate_aws_specific_config(secret) - return secret + add_defaults(json.loads(client.get_secret_value(SecretId=secret_identifier)["SecretString"])) + self.__validate_aws_specific_config() except NoCredentialsError as _: raise MissingInstanceProfile(self.__class__.__name__) except ClientError as _: - raise ConfigNotFound(self.__class__.__name__, f"Secret Manager {secret_identifier} in {region}") + raise ApiTokenNotFound(self.__class__.__name__, f"Secret Manager {secret_identifier} in {region}") @staticmethod def __get_max_capacity(): @@ -137,7 +135,7 @@ def __run_config_server(self) -> None: json.dump(self.configs, config_file) os.chdir("/opt/uid2operator/config-server") command = ["./bin/flask", "run", "--host", AuxiliaryConfig.LOCALHOST, "--port", AuxiliaryConfig.FLASK_PORT] - self.run_command(command, seperate_process=True) + self.run_command(command, separate_process=True) def __run_socks_proxy(self) -> None: """ @@ -205,12 +203,12 @@ def __run_nitro_enclave(self): if self.configs.get('debug_mode', False): print("Running in debug_mode") command += ["--debug-mode", "--attach-console"] - self.run_command(command, seperate_process=True) + self.run_command(command, separate_process=True) def run_compute(self) -> None: """Main execution flow for confidential compute.""" secret_manager_key = self.__get_secret_name_from_userdata() - self.configs = self._get_secret(secret_manager_key) + self._set_confidential_config(secret_manager_key) print(f"Fetched configs from {secret_manager_key}") if not self.configs.get("skip_validations"): self.validate_configuration() @@ -246,7 +244,7 @@ def __kill_auxiliaries(self) -> None: parser.add_argument("-o", "--operation", choices=["stop", "start"], default="start", help="Operation to perform.") args = parser.parse_args() try: - ec2 = EC2() + ec2 = EC2EntryPoint() if args.operation == "stop": ec2.cleanup() else: @@ -254,5 +252,5 @@ def __kill_auxiliaries(self) -> None: except ConfidentialComputeStartupException as e: print("Failed starting up Confidential Compute. Please checks the logs for errors and retry \n", e) except Exception as e: - print("Unexpected failure while starting up Confidential Compute. Please contact UID support team with this log \n ", e) + print("Unexpected failure while starting up Confidential Compute. Please contact UID support team with this log \n ", e) \ No newline at end of file diff --git a/scripts/azure-cc/Dockerfile b/scripts/azure-cc/Dockerfile index bb0c96b70..d4a65c507 100644 --- a/scripts/azure-cc/Dockerfile +++ b/scripts/azure-cc/Dockerfile @@ -1,13 +1,24 @@ -# sha from https://hub.docker.com/layers/amd64/eclipse-temurin/21.0.4_7-jre-alpine/images/sha256-8179ddc8a6c5ac9af935020628763b9a5a671e0914976715d2b61b21881cefca +# Use Alpine-based JRE image FROM eclipse-temurin@sha256:8179ddc8a6c5ac9af935020628763b9a5a671e0914976715d2b61b21881cefca -# Install Packages -RUN apk update && apk add jq +# Install necessary packages and set up virtual environment +RUN apk update && apk add --no-cache jq python3 py3-pip && \ + python3 -m venv /venv && \ + . /venv/bin/activate && \ + pip install --no-cache-dir requests azure-identity azure-keyvault-secrets && \ + rm -rf /var/cache/apk/* +# Set virtual environment path +ENV PATH="/venv/bin:$PATH" + +# Working directory WORKDIR /app + +# Expose necessary ports EXPOSE 8080 EXPOSE 9080 +# ARG and ENV variables ARG JAR_NAME=uid2-operator ARG JAR_VERSION=1.0.0-SNAPSHOT ARG IMAGE_VERSION=1.0.0.unknownhash @@ -17,18 +28,28 @@ ENV IMAGE_VERSION=${IMAGE_VERSION} ENV REGION=default ENV LOKI_HOSTNAME=loki +# Copy application files COPY ./target/${JAR_NAME}-${JAR_VERSION}-jar-with-dependencies.jar /app/${JAR_NAME}-${JAR_VERSION}.jar COPY ./target/${JAR_NAME}-${JAR_VERSION}-sources.jar /app COPY ./target/${JAR_NAME}-${JAR_VERSION}-static.tar.gz /app/static.tar.gz COPY ./conf/*.json /app/conf/ COPY ./conf/*.xml /app/conf/ -RUN tar xzvf /app/static.tar.gz --no-same-owner --no-same-permissions && rm -f /app/static.tar.gz +# Extract and clean up tar.gz +RUN tar xzvf /app/static.tar.gz --no-same-owner --no-same-permissions && \ + rm -f /app/static.tar.gz + +COPY ./azureEntryPoint.py /app +COPY ./confidential_compute.py /app +RUN chmod a+x /app/*.py -COPY ./entrypoint.sh /app/ -RUN chmod a+x /app/entrypoint.sh +# Create and configure non-root user +RUN adduser -D uid2-operator && \ + mkdir -p /opt/uid2 && chmod 777 -R /opt/uid2 && \ + chmod 705 -R /app && mkdir -p /app/file-uploads && chmod 777 -R /app/file-uploads -RUN adduser -D uid2-operator && mkdir -p /opt/uid2 && chmod 777 -R /opt/uid2 && mkdir -p /app && chmod 705 -R /app && mkdir -p /app/file-uploads && chmod 777 -R /app/file-uploads +# Switch to non-root user USER uid2-operator -CMD ["/app/entrypoint.sh"] +# Run the Python entry point +CMD python3 /app/azureEntryPoint.py \ No newline at end of file diff --git a/scripts/azure-cc/azureEntryPoint.py b/scripts/azure-cc/azureEntryPoint.py new file mode 100644 index 000000000..9332dec83 --- /dev/null +++ b/scripts/azure-cc/azureEntryPoint.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 + +import json +import os +import time +from typing import Dict +import sys +import shutil +import requests +import logging + +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from confidential_compute import ConfidentialCompute, MissingConfig, MissingInstanceProfile, AuxiliariesException, SecretAccessDenied, ApiTokenNotFound, ConfidentialComputeStartupException +from azure.keyvault.secrets import SecretClient +from azure.identity import DefaultAzureCredential, CredentialUnavailableError +from azure.core.exceptions import ResourceNotFoundError, ClientAuthenticationError + +class AzureEntryPoint(ConfidentialCompute): + + kv_name = os.getenv("VAULT_NAME") + secret_name = os.getenv("OPERATOR_KEY_SECRET_NAME") + env_name = os.getenv("DEPLOYMENT_ENVIRONMENT") + jar_name = os.getenv("JAR_NAME", "default-jar-name") + jar_version = os.getenv("JAR_VERSION", "default-jar-version") + + FINAL_CONFIG = "/tmp/final-config.json" + + def __init__(self): + super().__init__() + + def __check_env_variables(self): + # Check essential env variables + if AzureEntryPoint.kv_name is None: + raise MissingConfig(self.__class__.__name__, ["VAULT_NAME"]) + if AzureEntryPoint.secret_name is None: + raise MissingConfig(self.__class__.__name__, ["OPERATOR_KEY_SECRET_NAME"]) + if AzureEntryPoint.env_name is None: + raise MissingConfig(self.__class__.__name__, ["DEPLOYMENT_ENVIRONMENT"]) + logging.info("Environment variables validation success") + + def __create_final_config(self): + TARGET_CONFIG = f"/app/conf/{AzureEntryPoint.env_name}-uid2-config.json" + if not os.path.isfile(TARGET_CONFIG): + logging.error(f"Unrecognized config {TARGET_CONFIG}") + sys.exit(1) + + logging.info(f"-- copying {TARGET_CONFIG} to {AzureEntryPoint.FINAL_CONFIG}") + try: + shutil.copy(TARGET_CONFIG, AzureEntryPoint.FINAL_CONFIG) + except IOError as e: + logging.error(f"Failed to create {AzureEntryPoint.FINAL_CONFIG} with error: {e}") + sys.exit(1) + + CORE_BASE_URL = os.getenv("CORE_BASE_URL") + OPTOUT_BASE_URL = os.getenv("OPTOUT_BASE_URL") + if CORE_BASE_URL and OPTOUT_BASE_URL and AzureEntryPoint.env_name != 'prod': + logging.info(f"-- replacing URLs by {CORE_BASE_URL} and {OPTOUT_BASE_URL}") + with open(AzureEntryPoint.FINAL_CONFIG, "r") as file: + config = file.read() + + config = config.replace("https://core-integ.uidapi.com", CORE_BASE_URL) + config = config.replace("https://optout-integ.uidapi.com", OPTOUT_BASE_URL) + + with open(AzureEntryPoint.FINAL_CONFIG, "w") as file: + file.write(config) + + with open(AzureEntryPoint.FINAL_CONFIG, "r") as file: + logging.info(file.read()) + + def __set_base_urls(self): + with open(AzureEntryPoint.FINAL_CONFIG, "r") as file: + jdata = json.load(file) + self.configs["core_base_url"] = jdata["core_attest_url"] + self.configs["optout_base_url"] = jdata["optout_api_uri"] + + def __set_api_token(self): + try: + credential = DefaultAzureCredential() + kv_URL = f"https://{AzureEntryPoint.kv_name}.vault.azure.net" + secret_client = SecretClient(vault_url=kv_URL, credential=credential) + secret = secret_client.get_secret(AzureEntryPoint.secret_name) + # print(f"Secret Value: {secret.value}") + self.configs["api_token"] = secret.value + + except (CredentialUnavailableError, ClientAuthenticationError) as auth_error: + logging.error(f"Read operator key, authentication error: {auth_error}") + raise SecretAccessDenied(self.__class__.__name__, str(auth_error)) + except ResourceNotFoundError as not_found_error: + logging.error(f"Read operator key, secret not found: {AzureEntryPoint.secret_name}. Error: {not_found_error}") + raise ApiTokenNotFound(self.__class__.__name__, str(not_found_error)) + + + def _set_confidential_config(self, secret_identifier: str = None): + self.configs["skip_validations"] = os.getenv("SKIP_VALIDATIONS", "false").lower() == "true" + self.configs["debug_mode"] = os.getenv("DEBUG_MODE", "false").lower() == "true" + self.configs["environment"] = AzureEntryPoint.env_name + + # set self.configs["api_token"] + self.__set_api_token() + # set base urls from final config file + self.__set_base_urls() + + def __run_operator(self): + + # Start the operator + os.environ["azure_vault_name"] = AzureEntryPoint.kv_name + os.environ["azure_secret_name"] = AzureEntryPoint.secret_name + + java_command = [ + "java", + "-XX:MaxRAMPercentage=95", "-XX:-UseCompressedOops", "-XX:+PrintFlagsFinal", + "-Djava.security.egd=file:/dev/./urandom", + "-Dvertx.logger-delegate-factory-class-name=io.vertx.core.logging.SLF4JLogDelegateFactory", + "-Dlogback.configurationFile=/app/conf/logback.xml", + f"-Dvertx-config-path={AzureEntryPoint.FINAL_CONFIG}", + "-jar", + f"{AzureEntryPoint.jar_name}-{AzureEntryPoint.jar_version}.jar" + ] + logging.info("-- starting java operator application") + self.run_command(java_command, separate_process=False) + + def _validate_auxiliaries(self): + logging.info("Waiting for sidecar ...") + + MAX_RETRIES = 15 + PING_URL = "http://169.254.169.254/ping" + delay = 1 + + for attempt in range(1, MAX_RETRIES + 1): + try: + response = requests.get(PING_URL, timeout=5) + if response.status_code in [200, 204]: + logging.info("Sidecar started successfully.") + return + else: + logging.warning( + f"Attempt {attempt}: Unexpected status code {response.status_code}. Response: {response.text}" + ) + except Exception as e: + logging.info(f"Attempt {attempt}: Error during request - {e}") + + if attempt == MAX_RETRIES: + logging.error( + f"Sidecar failed to start after {MAX_RETRIES} attempts. Exiting." + ) + raise AuxiliariesException(self.__class__.__name__) + + logging.info(f"Retrying in {delay} seconds... (Attempt {attempt}/{MAX_RETRIES})") + time.sleep(delay) + delay += 1 + + def run_compute(self) -> None: + """Main execution flow for confidential compute.""" + self.__check_env_variables() + self.__create_final_config() + self._set_confidential_config() + if not self.configs.get("skip_validations"): + self.validate_configuration() + self._setup_auxiliaries() + self.__run_operator() + + def _setup_auxiliaries(self) -> None: + """ setup auxiliary services are running.""" + pass + +if __name__ == "__main__": + + logging.basicConfig(level=logging.INFO) + logging.info("Start AzureEntryPoint") + try: + operator = AzureEntryPoint() + operator.run_compute() + except ConfidentialComputeStartupException as e: + logging.error(f"Failed starting up Azure Confidential Compute. Please checks the logs for errors and retry {e}", exc_info=True) + except Exception as e: + logging.error(f"Unexpected failure while starting up Azure Confidential Compute. Please contact UID support team with this log {e}", exc_info=True) \ No newline at end of file diff --git a/scripts/confidential_compute.py b/scripts/confidential_compute.py index 827ab8f6a..9872abe62 100644 --- a/scripts/confidential_compute.py +++ b/scripts/confidential_compute.py @@ -5,6 +5,7 @@ from abc import ABC, abstractmethod from typing import TypedDict, NotRequired, get_type_hints import subprocess +import logging class ConfidentialComputeConfig(TypedDict): api_token: str @@ -17,9 +18,9 @@ class ConfidentialComputeConfig(TypedDict): class ConfidentialComputeStartupException(Exception): def __init__(self, error_name, provider, extra_message=None): urls = { - "EC2": "https://unifiedid.com/docs/guides/operator-guide-aws-marketplace#uid2-operator-error-codes", - "Azure": "https://unifiedid.com/docs/guides/operator-guide-azure-enclave#uid2-operator-error-codes", - "GCPEntrypoint": "https://unifiedid.com/docs/guides/operator-private-gcp-confidential-space#uid2-operator-error-codes", + "EC2EntryPoint": "https://unifiedid.com/docs/guides/operator-guide-aws-marketplace#uid2-operator-error-codes", + "AzureEntryPoint": "https://unifiedid.com/docs/guides/operator-guide-azure-enclave#uid2-operator-error-codes", + "GCPEntryPoint": "https://unifiedid.com/docs/guides/operator-private-gcp-confidential-space#uid2-operator-error-codes", } url = urls.get(provider) super().__init__(f"{error_name}\n" + (extra_message if extra_message else "") + f"\nVisit {url} for more details") @@ -28,7 +29,7 @@ class MissingInstanceProfile(ConfidentialComputeStartupException): def __init__(self, cls, message = None): super().__init__(error_name=f"E01: {self.__class__.__name__}", provider=cls, extra_message=message) -class ConfigNotFound(ConfidentialComputeStartupException): +class ApiTokenNotFound(ConfidentialComputeStartupException): def __init__(self, cls, message = None): super().__init__(error_name=f"E02: {self.__class__.__name__}", provider=cls, extra_message=message) @@ -47,7 +48,15 @@ def __init__(self, cls): class UID2ServicesUnreachable(ConfidentialComputeStartupException): def __init__(self, cls, ip=None): super().__init__(error_name=f"E06: {self.__class__.__name__}", provider=cls, extra_message=ip) - + +class AuxiliariesException(ConfidentialComputeStartupException): + def __init__(self, cls, inner_message = None): + super().__init__(error_name=f"E07: {self.__class__.__name__}", provider=cls, extra_message=inner_message) + +class SecretAccessDenied(ConfidentialComputeStartupException): + def __init__(self, cls, message = None): + super().__init__(error_name=f"E08: {self.__class__.__name__}", provider=cls, extra_message=message) + class ConfidentialCompute(ABC): def __init__(self): @@ -55,7 +64,7 @@ def __init__(self): def validate_configuration(self): """ Validates the paramters specified through configs/secret manager .""" - print("Validating configurations provided") + logging.info("Validating configurations provided") def validate_operator_key(): """ Validates the operator key format and its environment alignment.""" operator_key = self.configs.get("api_token") @@ -66,9 +75,9 @@ def validate_operator_key(): expected_env = "I" if debug_mode or env == "integ" else "P" if operator_key.split("-")[2] != expected_env: raise InvalidOperatorKey(self.__class__.__name__) - print("Validated operator key matches environment") + logging.info("Validated operator key matches environment") else: - print("Skipping operator key validation") + logging.info("Skipping operator key validation") def validate_url(url_key, environment): """URL should include environment except in prod""" @@ -77,7 +86,7 @@ def validate_url(url_key, environment): parsed_url = urlparse(self.configs[url_key]) if parsed_url.scheme != 'https' and parsed_url.path: raise InvalidConfigValue(self.__class__.__name__, url_key) - print(f"Validated {self.configs[url_key]} matches other config parameters") + logging.info(f"Validated {self.configs[url_key]} matches other config parameters") def validate_connectivity() -> None: """ Validates that the core URL is accessible.""" @@ -85,7 +94,7 @@ def validate_connectivity() -> None: core_url = self.configs["core_base_url"] core_ip = socket.gethostbyname(urlparse(core_url).netloc) requests.get(core_url, timeout=5) - print(f"Validated connectivity to {core_url}") + logging.info(f"Validated connectivity to {core_url}") except (requests.ConnectionError, requests.Timeout) as e: raise UID2ServicesUnreachable(self.__class__.__name__, core_ip) except Exception as e: @@ -108,15 +117,12 @@ def validate_connectivity() -> None: validate_url("optout_base_url", environment) validate_operator_key() validate_connectivity() - print("Completed static validation of confidential compute config values") + logging.info("Completed static validation of confidential compute config values") @abstractmethod - def _get_secret(self, secret_identifier: str) -> ConfidentialComputeConfig: + def _set_confidential_config(self, secret_identifier: str) -> None: """ - Fetches the secret from a secret store. - - Raises: - SecretNotFoundException: If the secret is not found. + Set ConfidentialComputeConfig """ pass @@ -136,13 +142,13 @@ def run_compute(self) -> None: pass @staticmethod - def run_command(command, seperate_process=False): - print(f"Running command: {' '.join(command)}") + def run_command(command, separate_process=False): + logging.info(f"Running command: {' '.join(command)}") try: - if seperate_process: + if separate_process: subprocess.Popen(command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) else: subprocess.run(command,check=True) except Exception as e: - print(f"Failed to run command: {str(e)}") + logging.error(f"Failed to run command: {e}", exc_info=True) raise RuntimeError (f"Failed to start {' '.join(command)} ") \ No newline at end of file diff --git a/scripts/gcp-oidc/gcp.py b/scripts/gcp-oidc/gcp.py index a2a4a5013..a74becadf 100644 --- a/scripts/gcp-oidc/gcp.py +++ b/scripts/gcp-oidc/gcp.py @@ -10,14 +10,15 @@ from google.api_core.exceptions import PermissionDenied, NotFound sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from confidential_compute import ConfidentialCompute, ConfidentialComputeConfig, MissingConfig, ConfigNotFound, MissingInstanceProfile, ConfidentialComputeStartupException +from confidential_compute import ConfidentialCompute, ConfidentialComputeConfig, MissingConfig, ApiTokenNotFound, SecretAccessDenied, ConfidentialComputeStartupException -class GCPEntrypoint(ConfidentialCompute): +class GCPEntryPoint(ConfidentialCompute): def __init__(self): super().__init__() - def _get_secret(self, secret_identifier=None) -> ConfidentialComputeConfig: + def _set_confidential_config(self, secret_identifier=None) -> None: + keys_mapping = { "core_base_url": "CORE_BASE_URL", "optout_base_url": "OPTOUT_BASE_URL", @@ -25,7 +26,7 @@ def _get_secret(self, secret_identifier=None) -> ConfidentialComputeConfig: "skip_validations": "SKIP_VALIDATIONS", "debug_mode": "DEBUG_MODE", } - config: ConfidentialComputeConfig = { + self.config = { key: (os.environ[env_var].lower() == "true" if key in ["skip_validations", "debug_mode"] else os.environ[env_var]) for key, env_var in keys_mapping.items() if env_var in os.environ } @@ -38,11 +39,10 @@ def _get_secret(self, secret_identifier=None) -> ConfidentialComputeConfig: response = client.access_secret_version(name=secret_version_name) secret_value = response.payload.data.decode("UTF-8") except (PermissionDenied, DefaultCredentialsError) as e: - raise MissingInstanceProfile(self.__class__.__name__, str(e)) + raise SecretAccessDenied(self.__class__.__name__, str(e)) except NotFound: - raise ConfigNotFound(self.__class__.__name__, f"Secret Manager {os.getenv("API_TOKEN_SECRET_NAME")}") - config["api_token"] = secret_value - return config + raise ApiTokenNotFound(self.__class__.__name__, f"Secret Manager {os.getenv("API_TOKEN_SECRET_NAME")}") + self.config["api_token"] = secret_value def __populate_operator_config(self, destination): target_config = f"/app/conf/{self.configs["environment"].lower()}-config.json" @@ -63,7 +63,7 @@ def _validate_auxiliaries(self) -> None: pass def run_compute(self) -> None: - self.configs = self._get_secret('read_from_env_vars') + self._set_confidential_config() print(f"Fetched configs") if not self.configs.get("skip_validations"): self.validate_configuration() @@ -86,10 +86,10 @@ def run_compute(self) -> None: if __name__ == "__main__": try: - gcp = GCP() + gcp = GCPEntryPoint() gcp.run_compute() except ConfidentialComputeStartupException as e: print("Failed starting up Confidential Compute. Please checks the logs for errors and retry \n", e) except Exception as e: - print("Unexpected failure while starting up Confidential Compute. Please contact UID support team with this log \n ", e) + print("Unexpected failure while starting up Confidential Compute. Please contact UID support team with this log \n ", e) From c690f90d828ad4caae70f93a4f5c67aae0e080b3 Mon Sep 17 00:00:00 2001 From: Xu Yang <58192524+clarkxuyang@users.noreply.github.com> Date: Fri, 24 Jan 2025 15:20:05 -0800 Subject: [PATCH 223/431] xuy-UID2-4719 use operator-key as name (#1318) --- scripts/aws/ec2.py | 4 ++-- scripts/azure-cc/azureEntryPoint.py | 14 +++++++------- scripts/confidential_compute.py | 8 ++++---- scripts/gcp-oidc/gcp.py | 8 ++++---- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/scripts/aws/ec2.py b/scripts/aws/ec2.py index 31cc77461..14eb998a1 100644 --- a/scripts/aws/ec2.py +++ b/scripts/aws/ec2.py @@ -16,7 +16,7 @@ import yaml sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from confidential_compute import ConfidentialCompute, ConfidentialComputeConfig, MissingInstanceProfile, ApiTokenNotFound, InvalidConfigValue, ConfidentialComputeStartupException +from confidential_compute import ConfidentialCompute, ConfidentialComputeConfig, MissingInstanceProfile, OperatorKeyNotFound, InvalidConfigValue, ConfidentialComputeStartupException class AWSConfidentialComputeConfig(ConfidentialComputeConfig): enclave_memory_mb: int @@ -103,7 +103,7 @@ def add_defaults(configs: Dict[str, any]) -> None: except NoCredentialsError as _: raise MissingInstanceProfile(self.__class__.__name__) except ClientError as _: - raise ApiTokenNotFound(self.__class__.__name__, f"Secret Manager {secret_identifier} in {region}") + raise OperatorKeyNotFound(self.__class__.__name__, f"Secret Manager {secret_identifier} in {region}") @staticmethod def __get_max_capacity(): diff --git a/scripts/azure-cc/azureEntryPoint.py b/scripts/azure-cc/azureEntryPoint.py index 9332dec83..534525d86 100644 --- a/scripts/azure-cc/azureEntryPoint.py +++ b/scripts/azure-cc/azureEntryPoint.py @@ -10,7 +10,7 @@ import logging sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from confidential_compute import ConfidentialCompute, MissingConfig, MissingInstanceProfile, AuxiliariesException, SecretAccessDenied, ApiTokenNotFound, ConfidentialComputeStartupException +from confidential_compute import ConfidentialCompute, MissingConfig, MissingInstanceProfile, AuxiliariesException, OperatorKeyAccessDenied, OperatorKeyNotFound, ConfidentialComputeStartupException from azure.keyvault.secrets import SecretClient from azure.identity import DefaultAzureCredential, CredentialUnavailableError from azure.core.exceptions import ResourceNotFoundError, ClientAuthenticationError @@ -73,21 +73,21 @@ def __set_base_urls(self): self.configs["core_base_url"] = jdata["core_attest_url"] self.configs["optout_base_url"] = jdata["optout_api_uri"] - def __set_api_token(self): + def __set_operator_key(self): try: credential = DefaultAzureCredential() kv_URL = f"https://{AzureEntryPoint.kv_name}.vault.azure.net" secret_client = SecretClient(vault_url=kv_URL, credential=credential) secret = secret_client.get_secret(AzureEntryPoint.secret_name) # print(f"Secret Value: {secret.value}") - self.configs["api_token"] = secret.value + self.configs["operator_key"] = secret.value except (CredentialUnavailableError, ClientAuthenticationError) as auth_error: logging.error(f"Read operator key, authentication error: {auth_error}") - raise SecretAccessDenied(self.__class__.__name__, str(auth_error)) + raise OperatorKeyAccessDenied(self.__class__.__name__, str(auth_error)) except ResourceNotFoundError as not_found_error: logging.error(f"Read operator key, secret not found: {AzureEntryPoint.secret_name}. Error: {not_found_error}") - raise ApiTokenNotFound(self.__class__.__name__, str(not_found_error)) + raise OperatorKeyNotFound(self.__class__.__name__, str(not_found_error)) def _set_confidential_config(self, secret_identifier: str = None): @@ -95,8 +95,8 @@ def _set_confidential_config(self, secret_identifier: str = None): self.configs["debug_mode"] = os.getenv("DEBUG_MODE", "false").lower() == "true" self.configs["environment"] = AzureEntryPoint.env_name - # set self.configs["api_token"] - self.__set_api_token() + # set self.configs["operator_key"] + self.__set_operator_key() # set base urls from final config file self.__set_base_urls() diff --git a/scripts/confidential_compute.py b/scripts/confidential_compute.py index 9872abe62..e156758ee 100644 --- a/scripts/confidential_compute.py +++ b/scripts/confidential_compute.py @@ -8,7 +8,7 @@ import logging class ConfidentialComputeConfig(TypedDict): - api_token: str + operator_key: str core_base_url: str optout_base_url: str environment: str @@ -29,7 +29,7 @@ class MissingInstanceProfile(ConfidentialComputeStartupException): def __init__(self, cls, message = None): super().__init__(error_name=f"E01: {self.__class__.__name__}", provider=cls, extra_message=message) -class ApiTokenNotFound(ConfidentialComputeStartupException): +class OperatorKeyNotFound(ConfidentialComputeStartupException): def __init__(self, cls, message = None): super().__init__(error_name=f"E02: {self.__class__.__name__}", provider=cls, extra_message=message) @@ -53,7 +53,7 @@ class AuxiliariesException(ConfidentialComputeStartupException): def __init__(self, cls, inner_message = None): super().__init__(error_name=f"E07: {self.__class__.__name__}", provider=cls, extra_message=inner_message) -class SecretAccessDenied(ConfidentialComputeStartupException): +class OperatorKeyAccessDenied(ConfidentialComputeStartupException): def __init__(self, cls, message = None): super().__init__(error_name=f"E08: {self.__class__.__name__}", provider=cls, extra_message=message) @@ -67,7 +67,7 @@ def validate_configuration(self): logging.info("Validating configurations provided") def validate_operator_key(): """ Validates the operator key format and its environment alignment.""" - operator_key = self.configs.get("api_token") + operator_key = self.configs.get("operator_key") pattern = r"^(UID2|EUID)-.\-(I|P|L)-\d+-.*$" if re.match(pattern, operator_key): env = self.configs.get("environment", "").lower() diff --git a/scripts/gcp-oidc/gcp.py b/scripts/gcp-oidc/gcp.py index a74becadf..8e76e9c4f 100644 --- a/scripts/gcp-oidc/gcp.py +++ b/scripts/gcp-oidc/gcp.py @@ -10,7 +10,7 @@ from google.api_core.exceptions import PermissionDenied, NotFound sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from confidential_compute import ConfidentialCompute, ConfidentialComputeConfig, MissingConfig, ApiTokenNotFound, SecretAccessDenied, ConfidentialComputeStartupException +from confidential_compute import ConfidentialCompute, ConfidentialComputeConfig, MissingConfig, OperatorKeyNotFound, OperatorKeyAccessDenied, ConfidentialComputeStartupException class GCPEntryPoint(ConfidentialCompute): @@ -39,10 +39,10 @@ def _set_confidential_config(self, secret_identifier=None) -> None: response = client.access_secret_version(name=secret_version_name) secret_value = response.payload.data.decode("UTF-8") except (PermissionDenied, DefaultCredentialsError) as e: - raise SecretAccessDenied(self.__class__.__name__, str(e)) + raise OperatorKeyAccessDenied(self.__class__.__name__, str(e)) except NotFound: - raise ApiTokenNotFound(self.__class__.__name__, f"Secret Manager {os.getenv("API_TOKEN_SECRET_NAME")}") - self.config["api_token"] = secret_value + raise OperatorKeyNotFound(self.__class__.__name__, f"Secret Manager {os.getenv("API_TOKEN_SECRET_NAME")}") + self.config["operator_key"] = secret_value def __populate_operator_config(self, destination): target_config = f"/app/conf/{self.configs["environment"].lower()}-config.json" From e00fc4d8466aeaa7e23f47a1354c2e6627b164f6 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Mon, 27 Jan 2025 17:21:00 +0000 Subject: [PATCH 224/431] [CI Pipeline] Released Patch version: 5.45.7 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c3f6067c4..a85367e0d 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.45.8-alpha-172-SNAPSHOT + 5.45.7 UTF-8 From 03478769874a134136c8de3c42fe2097f5ffd856 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Mon, 27 Jan 2025 23:56:57 +0000 Subject: [PATCH 225/431] [CI Pipeline] Released Patch version: 5.45.8 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a85367e0d..7e5db7312 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.45.7 + 5.45.8 UTF-8 From 775bacf4ec27588dde8b6ed10f7b297efe2a478a Mon Sep 17 00:00:00 2001 From: abuabraham-ttd <130511456+abuabraham-ttd@users.noreply.github.com> Date: Thu, 30 Jan 2025 10:45:26 -0800 Subject: [PATCH 226/431] Remove and replace additional operations in enclave start (#1289) * Remove and replace additional operations in enclave start --- Makefile.eif | 31 ++-- scripts/aws/Dockerfile | 6 +- scripts/aws/EUID_CloudFormation.template.yml | 2 +- scripts/aws/UID_CloudFormation.template.yml | 2 +- ...uid-config.json => euid-integ-config.json} | 2 +- ...euid-config.json => euid-prod-config.json} | 2 +- ...id2-config.json => uid2-integ-config.json} | 2 +- ...uid2-config.json => uid2-prod-config.json} | 2 +- scripts/aws/config-server/app.py | 19 --- scripts/aws/ec2.py | 13 +- scripts/aws/entrypoint.sh | 152 +++++++++--------- scripts/aws/load_config.py | 41 ----- scripts/aws/make_config.py | 57 ------- scripts/gcp-oidc/gcp.py | 4 +- 14 files changed, 102 insertions(+), 233 deletions(-) rename scripts/aws/conf/{integ-euid-config.json => euid-integ-config.json} (99%) rename scripts/aws/conf/{prod-euid-config.json => euid-prod-config.json} (99%) rename scripts/aws/conf/{integ-uid2-config.json => uid2-integ-config.json} (99%) rename scripts/aws/conf/{prod-uid2-config.json => uid2-prod-config.json} (99%) delete mode 100644 scripts/aws/load_config.py delete mode 100644 scripts/aws/make_config.py diff --git a/Makefile.eif b/Makefile.eif index c50b59b57..38e47c13c 100644 --- a/Makefile.eif +++ b/Makefile.eif @@ -13,23 +13,14 @@ all: build_eif build_eif: uid2operator.eif euidoperator.eif -uid2operator.eif: build_artifacts build_configs build/proxies.nitro.yaml build/syslog-ng-client.conf build/syslog-ng-core_4.6.0-1_amd64.deb build/syslog-ng-ose-pub.asc build/entrypoint.sh build/vsockpx build/Dockerfile build/load_config.py build/make_config.py +uid2operator.eif: build_artifacts build_configs build/proxies.nitro.yaml build/syslog-ng-client.conf build/syslog-ng-core_4.6.0-1_amd64.deb build/syslog-ng-ose-pub.asc build/entrypoint.sh build/vsockpx build/Dockerfile cd build; docker build -t uid2operator . --build-arg JAR_VERSION=`cat package.version` --build-arg IMAGE_VERSION=`cat package.version`-`git show --format="%h" --no-patch`; docker save -o ./uid2operator.tar uid2operator; docker cp ./uid2operator.tar amazonlinux:/uid2operator.tar; rm -f ./uid2operator.tar docker exec amazonlinux bash aws_nitro_eif.sh uid2operator -euidoperator.eif: build_artifacts build_configs build/proxies.nitro.yaml build/syslog-ng-client.conf build/syslog-ng-core_4.6.0-1_amd64.deb build/syslog-ng-ose-pub.asc build/entrypoint.sh build/vsockpx build/Dockerfile build/load_config.py build/make_config.py +euidoperator.eif: build_artifacts build_configs build/proxies.nitro.yaml build/syslog-ng-client.conf build/syslog-ng-core_4.6.0-1_amd64.deb build/syslog-ng-ose-pub.asc build/entrypoint.sh build/vsockpx build/Dockerfile cd build; docker build -t euidoperator . --build-arg IDENTITY_SCOPE='EUID' --build-arg JAR_VERSION=`cat package.version` --build-arg IMAGE_VERSION=`cat package.version`-`git show --format="%h" --no-patch`; docker save -o ./euidoperator.tar euidoperator; docker cp ./euidoperator.tar amazonlinux:/euidoperator.tar; rm -f ./euidoperator.tar docker exec amazonlinux bash aws_nitro_eif.sh euidoperator -################################################################################################################################################################## - -# Config scripts - -build/load_config.py: ./scripts/aws/load_config.py - cp ./scripts/aws/load_config.py ./build/ - -build/make_config.py: ./scripts/aws/make_config.py - cp ./scripts/aws/make_config.py ./build/ ################################################################################################################################################################## @@ -37,22 +28,22 @@ build/make_config.py: ./scripts/aws/make_config.py .PHONY: build_configs -build_configs: build/conf/default-config.json build/conf/prod-uid2-config.json build/conf/integ-uid2-config.json build/conf/prod-euid-config.json build/conf/integ-euid-config.json build/conf/logback.xml build/conf/logback-debug.xml +build_configs: build/conf/default-config.json build/conf/euid-integ-config.json build/conf/euid-prod-config.json build/conf/uid2-integ-config.json build/conf/uid2-prod-config.json build/conf/logback.xml build/conf/logback-debug.xml build/conf/default-config.json: build_artifacts ./scripts/aws/conf/default-config.json cp ./scripts/aws/conf/default-config.json ./build/conf/ -build/conf/prod-uid2-config.json: build_artifacts ./scripts/aws/conf/prod-uid2-config.json - cp ./scripts/aws/conf/prod-uid2-config.json ./build/conf/ +build/conf/euid-integ-config.json: build_artifacts ./scripts/aws/conf/euid-integ-config.json + cp ./scripts/aws/conf/euid-integ-config.json ./build/conf/ -build/conf/prod-euid-config.json: build_artifacts ./scripts/aws/conf/prod-euid-config.json - cp ./scripts/aws/conf/prod-euid-config.json ./build/conf/ +build/conf/euid-prod-config.json: build_artifacts ./scripts/aws/conf/euid-prod-config.json + cp ./scripts/aws/conf/euid-prod-config.json ./build/conf/ -build/conf/integ-uid2-config.json: build_artifacts ./scripts/aws/conf/integ-uid2-config.json - cp ./scripts/aws/conf/integ-uid2-config.json ./build/conf/ +build/conf/uid2-integ-config.json: build_artifacts ./scripts/aws/conf/uid2-integ-config.json + cp ./scripts/aws/conf/uid2-integ-config.json ./build/conf/ -build/conf/integ-euid-config.json: build_artifacts ./scripts/aws/conf/integ-euid-config.json - cp ./scripts/aws/conf/integ-euid-config.json ./build/conf/ +build/conf/uid2-prod-config.json: build_artifacts ./scripts/aws/conf/uid2-prod-config.json + cp ./scripts/aws/conf/uid2-prod-config.json ./build/conf/ build/conf/logback.xml: build_artifacts ./scripts/aws/conf/logback.xml cp ./scripts/aws/conf/logback.xml ./build/conf/ diff --git a/scripts/aws/Dockerfile b/scripts/aws/Dockerfile index e210001c3..340f13e2d 100644 --- a/scripts/aws/Dockerfile +++ b/scripts/aws/Dockerfile @@ -31,14 +31,10 @@ COPY ./target/${JAR_NAME}-${JAR_VERSION}-jar-with-dependencies.jar /app/${JAR_NA COPY ./static /app/static COPY ./libjnsm.so /app/lib/ COPY ./vsockpx /app/ -COPY ./make_config.py /app/ COPY ./entrypoint.sh /app/ COPY ./proxies.nitro.yaml /app/ COPY ./conf/default-config.json /app/conf/ -COPY ./conf/prod-uid2-config.json /app/conf/ -COPY ./conf/integ-uid2-config.json /app/conf/ -COPY ./conf/prod-euid-config.json /app/conf/ -COPY ./conf/integ-euid-config.json /app/conf/ +COPY ./conf/*.json /app/conf/ COPY ./conf/*.xml /app/conf/ COPY ./syslog-ng-client.conf /etc/syslog-ng/syslog-ng.conf diff --git a/scripts/aws/EUID_CloudFormation.template.yml b/scripts/aws/EUID_CloudFormation.template.yml index 3bb7958ad..2b9a6c5bb 100644 --- a/scripts/aws/EUID_CloudFormation.template.yml +++ b/scripts/aws/EUID_CloudFormation.template.yml @@ -165,7 +165,7 @@ Resources: - !If [IsIntegEnvironment, 'https://core.integ.euid.eu', 'https://core.prod.euid.eu'] - ', "optout_base_url": ' - !If [IsIntegEnvironment, 'https://optout.integ.euid.eu', 'https://optout.prod.euid.eu'] - - ', "api_token": "' + - ', "operator_key": "' - Ref: APIToken - '"' - ', "service_instances": 6' diff --git a/scripts/aws/UID_CloudFormation.template.yml b/scripts/aws/UID_CloudFormation.template.yml index 179ec47a1..0c0e9b359 100644 --- a/scripts/aws/UID_CloudFormation.template.yml +++ b/scripts/aws/UID_CloudFormation.template.yml @@ -193,7 +193,7 @@ Resources: - !If [IsIntegEnvironment, 'https://core-integ.uidapi.com', 'https://core.uidapi.com'] - ', "optout_base_url": ' - !If [IsIntegEnvironment, 'https://optout-integ.uidapi.com', 'https://optout.uidapi.com'] - - ', "api_token": "' + - ', "operator_key": "' - Ref: APIToken - '"' - ', "service_instances": 6' diff --git a/scripts/aws/conf/integ-euid-config.json b/scripts/aws/conf/euid-integ-config.json similarity index 99% rename from scripts/aws/conf/integ-euid-config.json rename to scripts/aws/conf/euid-integ-config.json index 45d3dbe94..9e82cee9d 100644 --- a/scripts/aws/conf/integ-euid-config.json +++ b/scripts/aws/conf/euid-integ-config.json @@ -12,4 +12,4 @@ "optout_api_uri": "https://optout.integ.euid.eu/optout/replicate", "optout_s3_folder": "optout/", "allow_legacy_api": false -} +} \ No newline at end of file diff --git a/scripts/aws/conf/prod-euid-config.json b/scripts/aws/conf/euid-prod-config.json similarity index 99% rename from scripts/aws/conf/prod-euid-config.json rename to scripts/aws/conf/euid-prod-config.json index 0fbf5d69c..3426c03a3 100644 --- a/scripts/aws/conf/prod-euid-config.json +++ b/scripts/aws/conf/euid-prod-config.json @@ -28,4 +28,4 @@ "enable_phone_support": true, "enable_v1_phone_support": false, "enable_v2_encryption": true -} +} \ No newline at end of file diff --git a/scripts/aws/conf/integ-uid2-config.json b/scripts/aws/conf/uid2-integ-config.json similarity index 99% rename from scripts/aws/conf/integ-uid2-config.json rename to scripts/aws/conf/uid2-integ-config.json index a7272a26a..2b3280d38 100644 --- a/scripts/aws/conf/integ-uid2-config.json +++ b/scripts/aws/conf/uid2-integ-config.json @@ -12,4 +12,4 @@ "optout_api_uri": "https://optout-integ.uidapi.com/optout/replicate", "optout_s3_folder": "uid-optout-integ/", "allow_legacy_api": false -} +} \ No newline at end of file diff --git a/scripts/aws/conf/prod-uid2-config.json b/scripts/aws/conf/uid2-prod-config.json similarity index 99% rename from scripts/aws/conf/prod-uid2-config.json rename to scripts/aws/conf/uid2-prod-config.json index 5da450033..381fc8021 100644 --- a/scripts/aws/conf/prod-uid2-config.json +++ b/scripts/aws/conf/uid2-prod-config.json @@ -23,4 +23,4 @@ "refresh_token_expires_after_seconds": 2592000, "refresh_identity_token_after_seconds": 3600, "allow_legacy_api": false -} +} \ No newline at end of file diff --git a/scripts/aws/config-server/app.py b/scripts/aws/config-server/app.py index edb80e4d5..c0c94fc63 100644 --- a/scripts/aws/config-server/app.py +++ b/scripts/aws/config-server/app.py @@ -10,25 +10,6 @@ def get_config(): with open('/etc/secret/secret-value/config', 'r') as secret_file: secret_value = secret_file.read().strip() secret_value_json = json.loads(secret_value) - secret_value_json["environment"] = secret_value_json["environment"].lower() - if "core_base_url" in secret_value_json: - secret_value_json["core_base_url"] = secret_value_json["core_base_url"].lower() - if "optout_base_url" in secret_value_json: - secret_value_json["optout_base_url"] = secret_value_json["optout_base_url"].lower() - if "operator_type" in secret_value_json and secret_value_json["operator_type"].lower() == "public": - mount_path = '/etc/config/config-values' - if os.path.exists(mount_path): - config_keys = [f for f in os.listdir(mount_path) if os.path.isfile(os.path.join(mount_path, f))] - config = {} - for k in config_keys: - with open(os.path.join(mount_path, k), 'r') as value: - config[k] = value.read() - try: - json.loads(config[k]) - config[k] = json.loads(config[k]) - except Exception: - pass - secret_value_json.update(config) return json.dumps(secret_value_json) except Exception as e: return str(e), 500 diff --git a/scripts/aws/ec2.py b/scripts/aws/ec2.py index 14eb998a1..f9c4feb28 100644 --- a/scripts/aws/ec2.py +++ b/scripts/aws/ec2.py @@ -21,6 +21,8 @@ class AWSConfidentialComputeConfig(ConfidentialComputeConfig): enclave_memory_mb: int enclave_cpu_count: int + core_api_token: str + optout_api_token: str class AuxiliaryConfig: FLASK_PORT: str = "27015" @@ -51,7 +53,7 @@ def get_meta_url(cls) -> str: class EC2EntryPoint(ConfidentialCompute): def __init__(self): - self.configs: AWSConfidentialComputeConfig = {} + super().__init__() def __get_aws_token(self) -> str: """Fetches a temporary AWS EC2 metadata token.""" @@ -87,18 +89,21 @@ def __validate_aws_specific_config(self): def _set_confidential_config(self, secret_identifier: str) -> None: """Fetches a secret value from AWS Secrets Manager and adds defaults""" - def add_defaults(configs: Dict[str, any]) -> None: + def add_defaults(configs: Dict[str, any]) -> AWSConfidentialComputeConfig: """Adds default values to configuration if missing.""" default_capacity = self.__get_max_capacity() configs.setdefault("enclave_memory_mb", default_capacity["enclave_memory_mb"]) configs.setdefault("enclave_cpu_count", default_capacity["enclave_cpu_count"]) configs.setdefault("debug_mode", False) + configs.setdefault("core_api_token", configs.get("api_token", "")) + configs.setdefault("optout_api_token", configs.get("api_token", "")) + return configs region = self.__get_current_region() print(f"Running in {region}") client = boto3.client("secretsmanager", region_name=region) try: - add_defaults(json.loads(client.get_secret_value(SecretId=secret_identifier)["SecretString"])) + self.configs = add_defaults(json.loads(client.get_secret_value(SecretId=secret_identifier)["SecretString"])) self.__validate_aws_specific_config() except NoCredentialsError as _: raise MissingInstanceProfile(self.__class__.__name__) @@ -203,7 +208,7 @@ def __run_nitro_enclave(self): if self.configs.get('debug_mode', False): print("Running in debug_mode") command += ["--debug-mode", "--attach-console"] - self.run_command(command, separate_process=True) + self.run_command(command, separate_process=False) def run_compute(self) -> None: """Main execution flow for confidential compute.""" diff --git a/scripts/aws/entrypoint.sh b/scripts/aws/entrypoint.sh index c7f3676f9..93a9662ba 100755 --- a/scripts/aws/entrypoint.sh +++ b/scripts/aws/entrypoint.sh @@ -7,93 +7,87 @@ LOG_FILE="/home/start.txt" set -x exec &> >(tee -a "$LOG_FILE") -set -o pipefail -ulimit -n 65536 - -# -- setup loopback device -echo "Setting up loopback device..." -ifconfig lo 127.0.0.1 - -# -- start vsock proxy -echo "Starting vsock proxy..." -/app/vsockpx --config /app/proxies.nitro.yaml --daemon --workers $(( ( $(nproc) + 3 ) / 4 )) --log-level 3 - -# -- load config from identity service -echo "Loading config from identity service via proxy..." - -#wait for config service, then download config -OVERRIDES_CONFIG="/app/conf/config-overrides.json" - -RETRY_COUNT=0 -MAX_RETRY=20 -until curl -s -f -o "${OVERRIDES_CONFIG}" -x socks5h://127.0.0.1:3305 http://127.0.0.1:27015/getConfig -do - echo "Waiting for config service to be available" - RETRY_COUNT=$(( RETRY_COUNT + 1)) - if [ $RETRY_COUNT -gt $MAX_RETRY ]; then - echo "Config Server did not return a response. Exiting" - exit 1 - fi - sleep 2 -done +PARAMETERIZED_CONFIG="/app/conf/config-overrides.json" +OPERATOR_CONFIG="/tmp/final-config.json" -DEBUG_MODE=$(jq -r ".debug_mode" < "${OVERRIDES_CONFIG}") +setup_auxiliaries() { + set -o pipefail + ulimit -n 65536 + + # -- setup loopback device + echo "Setting up loopback device..." + ifconfig lo 127.0.0.1 + + # -- start vsock proxy + echo "Starting vsock proxy..." + /app/vsockpx --config /app/proxies.nitro.yaml --daemon --workers $(( ( $(nproc) + 3 ) / 4 )) --log-level 3 -if [[ "$DEBUG_MODE" == "true" ]]; then - LOGBACK_CONF="./conf/logback-debug.xml" -else - LOGBACK_CONF="./conf/logback.xml" - # -- setup syslog-ng - echo "Starting syslog-ng..." /usr/sbin/syslog-ng --verbose -fi +} -# check the config is valid. Querying for a known missing element (empty) makes jq parse the file, but does not echo the results -if jq empty "${OVERRIDES_CONFIG}"; then - echo "Identity service returned valid config" -else - echo "Failed to get a valid config from identity service" - exit 1 -fi -export DEPLOYMENT_ENVIRONMENT=$(jq -r ".environment" < "${OVERRIDES_CONFIG}") -export CORE_BASE_URL=$(jq -r ".core_base_url" < "${OVERRIDES_CONFIG}") -export OPTOUT_BASE_URL=$(jq -r ".optout_base_url" < "${OVERRIDES_CONFIG}") -echo "DEPLOYMENT_ENVIRONMENT=${DEPLOYMENT_ENVIRONMENT}" -if [ -z "${DEPLOYMENT_ENVIRONMENT}" ]; then - echo "DEPLOYMENT_ENVIRONMENT cannot be empty" - exit 1 -fi -if [ "${DEPLOYMENT_ENVIRONMENT}" != "prod" ] && [ "${DEPLOYMENT_ENVIRONMENT}" != "integ" ]; then - echo "Unrecognized DEPLOYMENT_ENVIRONMENT ${DEPLOYMENT_ENVIRONMENT}" - exit 1 -fi +build_parameterized_config() { + curl -s -f -o "${PARAMETERIZED_CONFIG}" -x socks5h://127.0.0.1:3305 http://127.0.0.1:27015/getConfig + REQUIRED_KEYS=("optout_base_url" "core_base_url" "api_token" "environment") + for key in "${REQUIRED_KEYS[@]}"; do + if ! jq -e "has(\"${key}\")" "${PARAMETERIZED_CONFIG}" > /dev/null; then + echo "Error: Key '${key}' is missing. Please add it to flask config server" + exit 1 + fi + done + FILTER=$(printf '. | {') + for key in "${REQUIRED_KEYS[@]}"; do + FILTER+="$key: .${key}, " + done + FILTER+="debug_mode: .debug_mode, " + FILTER=${FILTER%, }'}' + jq "${FILTER}" "${PARAMETERIZED_CONFIG}" > "${PARAMETERIZED_CONFIG}.tmp" && mv "${PARAMETERIZED_CONFIG}.tmp" "${PARAMETERIZED_CONFIG}" +} + +build_operator_config() { + CORE_BASE_URL=$(jq -r ".core_base_url" < "${PARAMETERIZED_CONFIG}") + OPTOUT_BASE_URL=$(jq -r ".optout_base_url" < "${PARAMETERIZED_CONFIG}") + DEPLOYMENT_ENVIRONMENT=$(jq -r ".environment" < "${PARAMETERIZED_CONFIG}") + DEBUG_MODE=$(jq -r ".debug_mode" < "${PARAMETERIZED_CONFIG}") + + IDENTITY_SCOPE_LOWER=$(echo "${IDENTITY_SCOPE}" | tr '[:upper:]' '[:lower:]') + DEPLOYMENT_ENVIRONMENT_LOWER=$(echo "${DEPLOYMENT_ENVIRONMENT}" | tr '[:upper:]' '[:lower:]') + DEFAULT_CONFIG="/app/conf/${IDENTITY_SCOPE_LOWER}-${DEPLOYMENT_ENVIRONMENT_LOWER}-config.json" + + jq -s '.[0] * .[1]' "${DEFAULT_CONFIG}" "${PARAMETERIZED_CONFIG}" > "${OPERATOR_CONFIG}" + + if [[ "$DEPLOYMENT_ENVIRONMENT" == "prod" ]]; then + if [[ "$DEBUG_MODE" == "true" ]]; then + echo "Cannot run in DEBUG_MODE in production environment. Exiting." + exit 1 + fi + fi -echo "Loading config final..." -export FINAL_CONFIG="/app/conf/config-final.json" -if [ "${IDENTITY_SCOPE}" = "UID2" ]; then - python3 /app/make_config.py /app/conf/prod-uid2-config.json /app/conf/integ-uid2-config.json ${OVERRIDES_CONFIG} "$(nproc)" > ${FINAL_CONFIG} -elif [ "${IDENTITY_SCOPE}" = "EUID" ]; then - python3 /app/make_config.py /app/conf/prod-euid-config.json /app/conf/integ-euid-config.json ${OVERRIDES_CONFIG} "$(nproc)" > ${FINAL_CONFIG} -else - echo "Unrecognized IDENTITY_SCOPE ${IDENTITY_SCOPE}" - exit 1 -fi + #TODO: Remove below logic after remote config management is implemented + + if [[ "$DEPLOYMENT_ENVIRONMENT" != "prod" ]]; then + #Allow override of base URL in non-prod environments + CORE_PATTERN="https://core.*uidapi.com" + OPTOUT_PATTERN="https://optout.*uidapi.com" + if [[ "$DEPLOYMENT_ENVIRONMENT" == "euid" ]]; then + CORE_PATTERN="https://core.*euid.eu" + OPTOUT_PATTERN="https://optout.*euid.eu" + fi + sed -i "s#${CORE_PATTERN}#${CORE_BASE_URL}#g" "${OPERATOR_CONFIG}" + sed -i "s#${OPTOUT_PATTERN}#${OPTOUT_BASE_URL}#g" "${OPERATOR_CONFIG}" + fi + +} -# -- replace base URLs if both CORE_BASE_URL and OPTOUT_BASE_URL are provided -# -- using hardcoded domains is fine because they should not be changed frequently -if [ -n "${CORE_BASE_URL}" ] && [ "${CORE_BASE_URL}" != "null" ] && [ -n "${OPTOUT_BASE_URL}" ] && [ "${OPTOUT_BASE_URL}" != "null" ] && [ "${DEPLOYMENT_ENVIRONMENT}" != "prod" ]; then - echo "Replacing core and optout URLs by ${CORE_BASE_URL} and ${OPTOUT_BASE_URL}..." +setup_auxiliaries +build_parameterized_config +build_operator_config - sed -i "s#https://core-integ.uidapi.com#${CORE_BASE_URL}#g" "${FINAL_CONFIG}" - sed -i "s#https://core-prod.uidapi.com#${CORE_BASE_URL}#g" "${FINAL_CONFIG}" - sed -i "s#https://core.integ.euid.eu#${CORE_BASE_URL}#g" "${FINAL_CONFIG}" - sed -i "s#https://core.prod.euid.eu#${CORE_BASE_URL}#g" "${FINAL_CONFIG}" +DEBUG_MODE=$(jq -r ".debug_mode" < "${OPERATOR_CONFIG}") +LOGBACK_CONF="./conf/logback.xml" - sed -i "s#https://optout-integ.uidapi.com#${OPTOUT_BASE_URL}#g" "${FINAL_CONFIG}" - sed -i "s#https://optout-prod.uidapi.com#${OPTOUT_BASE_URL}#g" "${FINAL_CONFIG}" - sed -i "s#https://optout.integ.euid.eu#${OPTOUT_BASE_URL}#g" "${FINAL_CONFIG}" - sed -i "s#https://optout.prod.euid.eu#${OPTOUT_BASE_URL}#g" "${FINAL_CONFIG}" +if [[ "$DEBUG_MODE" == "true" ]]; then + LOGBACK_CONF="./conf/logback-debug.xml" fi # -- set pwd to /app so we can find default configs @@ -106,7 +100,7 @@ java \ -XX:MaxRAMPercentage=95 -XX:-UseCompressedOops -XX:+PrintFlagsFinal \ -Djava.security.egd=file:/dev/./urandom \ -Djava.library.path=/app/lib \ - -Dvertx-config-path="${FINAL_CONFIG}" \ + -Dvertx-config-path="${OPERATOR_CONFIG}" \ -Dvertx.logger-delegate-factory-class-name=io.vertx.core.logging.SLF4JLogDelegateFactory \ -Dlogback.configurationFile=${LOGBACK_CONF} \ -Dhttp_proxy=socks5://127.0.0.1:3305 \ diff --git a/scripts/aws/load_config.py b/scripts/aws/load_config.py deleted file mode 100644 index 9f0446a49..000000000 --- a/scripts/aws/load_config.py +++ /dev/null @@ -1,41 +0,0 @@ -import os -import boto3 -import base64 -import json -from botocore.exceptions import ClientError - -secret_name = os.environ['UID2_CONFIG_SECRET_KEY'] -region_name = os.environ['AWS_REGION_NAME'] -aws_access_key_id = os.environ['AWS_ACCESS_KEY_ID'] -secret_key = os.environ['AWS_SECRET_KEY'] -session_token = os.environ['AWS_SESSION_TOKEN'] - -def get_secret(): - session = boto3.session.Session() - client = session.client( - service_name='secretsmanager', - region_name=region_name, - aws_access_key_id = aws_access_key_id, - aws_secret_access_key = secret_key, - aws_session_token = session_token - ) - try: - get_secret_value_response = client.get_secret_value( - SecretId=secret_name - ) - except ClientError as e: - raise e - else: - if 'SecretString' in get_secret_value_response: - secret = get_secret_value_response['SecretString'] - else: - decoded_binary_secret = base64.b64decode(get_secret_value_response['SecretBinary']) - - return secret - -def get_config(): - result = get_secret() - conf = json.loads(result) - print(result) - -get_config() diff --git a/scripts/aws/make_config.py b/scripts/aws/make_config.py deleted file mode 100644 index 93439ef28..000000000 --- a/scripts/aws/make_config.py +++ /dev/null @@ -1,57 +0,0 @@ -import json -import sys - - -def load_json(path): - with open(path, 'r') as f: - return json.load(f) - - -def apply_override(config, overrides, key, type): - value = overrides.get(key) - if value is not None: - config[key] = type(value) - - -config_path = sys.argv[1] -integ_config_path = sys.argv[2] -overrides_path = sys.argv[3] -thread_count = int(sys.argv[4]) - -config = load_json(config_path) -overrides = load_json(overrides_path) - -# set API key -config['core_api_token'] = overrides['api_token'] -config['optout_api_token'] = overrides['api_token'] - -# number of threads -config['service_instances'] = int((thread_count + 1) * 2 / 3) - -# environment -if overrides.get('environment') == 'integ': - integ_config = load_json(integ_config_path) - apply_override(config, integ_config, 'sites_metadata_path', str) - apply_override(config, integ_config, 'clients_metadata_path', str) - apply_override(config, integ_config, 'keysets_metadata_path', str) - apply_override(config, integ_config, 'keyset_keys_metadata_path', str) - apply_override(config, integ_config, 'client_side_keypairs_metadata_path', str) - apply_override(config, integ_config, 'salts_metadata_path', str) - apply_override(config, integ_config, 'services_metadata_path', str) - apply_override(config, integ_config, 'service_links_metadata_path', str) - apply_override(config, integ_config, 'optout_metadata_path', str) - apply_override(config, integ_config, 'core_attest_url', str) - apply_override(config, integ_config, 'optout_api_uri', str) - apply_override(config, integ_config, 'optout_s3_folder', str) - - -apply_override(config, overrides, 'operator_type', str) -if 'operator_type' in config and config['operator_type'] == 'public': - config.update(overrides) -else: - # allowed overrides - apply_override(config, overrides, 'loki_enabled', bool) - apply_override(config, overrides, 'optout_synthetic_logs_enabled', bool) - apply_override(config, overrides, 'optout_synthetic_logs_count', int) - -print(json.dumps(config)) diff --git a/scripts/gcp-oidc/gcp.py b/scripts/gcp-oidc/gcp.py index 8e76e9c4f..e39e8c49b 100644 --- a/scripts/gcp-oidc/gcp.py +++ b/scripts/gcp-oidc/gcp.py @@ -26,7 +26,7 @@ def _set_confidential_config(self, secret_identifier=None) -> None: "skip_validations": "SKIP_VALIDATIONS", "debug_mode": "DEBUG_MODE", } - self.config = { + self.configs = { key: (os.environ[env_var].lower() == "true" if key in ["skip_validations", "debug_mode"] else os.environ[env_var]) for key, env_var in keys_mapping.items() if env_var in os.environ } @@ -42,7 +42,7 @@ def _set_confidential_config(self, secret_identifier=None) -> None: raise OperatorKeyAccessDenied(self.__class__.__name__, str(e)) except NotFound: raise OperatorKeyNotFound(self.__class__.__name__, f"Secret Manager {os.getenv("API_TOKEN_SECRET_NAME")}") - self.config["operator_key"] = secret_value + self.configs["operator_key"] = secret_value def __populate_operator_config(self, destination): target_config = f"/app/conf/{self.configs["environment"].lower()}-config.json" From 9850745a7e6eec4caefd87386fbda1580f62c236 Mon Sep 17 00:00:00 2001 From: abuabraham-ttd <130511456+abuabraham-ttd@users.noreply.github.com> Date: Thu, 30 Jan 2025 13:52:34 -0800 Subject: [PATCH 227/431] Pass DEBUG_MODE to container to allow for better error handling (#1332) * Allow override for debug --- scripts/gcp-oidc/Dockerfile | 2 +- scripts/gcp-oidc/terraform/main.tf | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/gcp-oidc/Dockerfile b/scripts/gcp-oidc/Dockerfile index 5d03ee49b..d687cc500 100644 --- a/scripts/gcp-oidc/Dockerfile +++ b/scripts/gcp-oidc/Dockerfile @@ -1,7 +1,7 @@ # sha from https://hub.docker.com/layers/amd64/eclipse-temurin/21.0.4_7-jre-alpine/images/sha256-8179ddc8a6c5ac9af935020628763b9a5a671e0914976715d2b61b21881cefca FROM eclipse-temurin@sha256:8179ddc8a6c5ac9af935020628763b9a5a671e0914976715d2b61b21881cefca -LABEL "tee.launch_policy.allow_env_override"="API_TOKEN_SECRET_NAME,DEPLOYMENT_ENVIRONMENT,CORE_BASE_URL,OPTOUT_BASE_URL" +LABEL "tee.launch_policy.allow_env_override"="API_TOKEN_SECRET_NAME,DEPLOYMENT_ENVIRONMENT,CORE_BASE_URL,OPTOUT_BASE_URL,DEBUG_MODE,SKIP_VALIDATIONS" LABEL "tee.launch_policy.log_redirect"="always" # Install Packages diff --git a/scripts/gcp-oidc/terraform/main.tf b/scripts/gcp-oidc/terraform/main.tf index aefb68362..3a600e26e 100644 --- a/scripts/gcp-oidc/terraform/main.tf +++ b/scripts/gcp-oidc/terraform/main.tf @@ -106,6 +106,7 @@ resource "google_compute_instance_template" "uid_operator" { tee-image-reference = var.uid_operator_image tee-container-log-redirect = true tee-restart-policy = "Never" + tee-env-DEBUG_MODE = var.debug_mode tee-env-DEPLOYMENT_ENVIRONMENT = var.uid_deployment_env tee-env-API_TOKEN_SECRET_NAME = module.secret-manager.secret_versions[0] tee-env-CORE_BASE_URL = var.uid_deployment_env == "integ" ? "https://core-integ.uidapi.com" : "https://core-prod.uidapi.com" From d153181bb9734eb627a7048374204fad29f41072 Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Fri, 31 Jan 2025 14:01:51 +1100 Subject: [PATCH 228/431] Address Review Comments - update config scan period for local-config.json - remove unnecessary config values - make ConfigRetrieverFactory create method static - extract runtime config creation to helper method in Main - Added token config validation to UIDOperatorService --- conf/default-config.json | 1 - conf/local-config.json | 6 +- conf/local-e2e-docker-public-config.json | 3 - src/main/java/com/uid2/operator/Main.java | 68 +++++++++++++------ .../service/ConfigRetrieverFactory.java | 2 +- .../uid2/operator/service/ConfigService.java | 1 + .../service/DelegatingConfigService.java | 2 +- .../operator/service/UIDOperatorService.java | 16 ++++- .../operator/vertx/UIDOperatorVerticle.java | 4 +- .../com/uid2/operator/ConfigServiceTest.java | 9 +-- 10 files changed, 73 insertions(+), 39 deletions(-) diff --git a/conf/default-config.json b/conf/default-config.json index a57739a7f..5a74087fe 100644 --- a/conf/default-config.json +++ b/conf/default-config.json @@ -34,6 +34,5 @@ "optout_inmem_cache": false, "enclave_platform": null, "failure_shutdown_wait_hours": 120, - "sharing_token_expiry_seconds": 2592000, "operator_type": "public" } diff --git a/conf/local-config.json b/conf/local-config.json index f1abb67f7..8de6fbd91 100644 --- a/conf/local-config.json +++ b/conf/local-config.json @@ -9,14 +9,10 @@ "salts_metadata_path": "/com.uid2.core/test/salts/metadata.json", "services_metadata_path": "/com.uid2.core/test/services/metadata.json", "service_links_metadata_path": "/com.uid2.core/test/service_links/metadata.json", - "identity_token_expires_after_seconds": 3600, - "refresh_token_expires_after_seconds": 86400, - "refresh_identity_token_after_seconds": 900, "refresh_token_v3": false, "identity_v3": false, "identity_scope": "uid2", "enable_v2_encryption": false, - "sharing_token_expiry_seconds": 2592000, "cloud_download_threads": 8, "cloud_upload_threads": 2, "cloud_refresh_interval": 60, @@ -43,6 +39,6 @@ "path": "conf/runtime-config-defaults.json", "format": "json" }, - "config_scan_period_ms": -1 + "config_scan_period_ms": 5000 } } diff --git a/conf/local-e2e-docker-public-config.json b/conf/local-e2e-docker-public-config.json index 3357a2b85..756453681 100644 --- a/conf/local-e2e-docker-public-config.json +++ b/conf/local-e2e-docker-public-config.json @@ -13,9 +13,6 @@ "salts_metadata_path": "http://core:8088/salt/refresh", "services_metadata_path": "http://core:8088/services/refresh", "service_links_metadata_path": "http://core:8088/service_links/refresh", - "identity_token_expires_after_seconds": 3600, - "refresh_token_expires_after_seconds": 86400, - "refresh_identity_token_after_seconds": 900, "refresh_token_v3": true, "identity_v3": false, "identity_scope": "uid2", diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index 57560ac6e..1ae72c5e5 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -267,43 +267,73 @@ private ICloudStorage wrapCloudStorageForOptOut(ICloudStorage cloudStorage) { } } - private void run() throws Exception { - this.createVertxInstancesMetric(); - this.createVertxEventLoopsMetric(); + private Future initialiseConfigService() throws Exception { + Promise promise = Promise.promise(); - ConfigRetrieverFactory configRetrieverFactory = new ConfigRetrieverFactory(); - ConfigRetriever dynamicConfigRetriever = configRetrieverFactory.create(vertx, config.getJsonObject("runtime_config_store"), this.createOperatorKeyRetriever().retrieve()); + ConfigRetriever dynamicConfigRetriever = ConfigRetrieverFactory.create( + vertx, + config.getJsonObject("runtime_config_store"), + this.createOperatorKeyRetriever().retrieve() + ); Future dynamicConfigFuture = ConfigService.create(dynamicConfigRetriever); - ConfigRetriever featureFlagConfigRetriever = configRetrieverFactory.create( + ConfigRetriever featureFlagConfigRetriever = ConfigRetrieverFactory.create( vertx, new JsonObject() .put("type", "file") .put("config", new JsonObject() .put("path", "conf/feat-flag/feat-flag.json") .put("format", "json")) - .put(ConfigScanPeriodMsProp, 60000), + .put(ConfigScanPeriodMsProp, 10000), "" ); - featureFlagConfigRetriever.getConfig().compose(featureFlagConfig -> { + featureFlagConfigRetriever.getConfig() + .compose(featureFlagConfig -> { + if (featureFlagConfig == null) { + return Future.failedFuture(new RuntimeException("Feature flag config is null")); + } + JsonObject remoteConfigJson = featureFlagConfig.getJsonObject("remote_config"); JsonObject featureFlagBootstrapConfig = remoteConfigJson.getJsonObject("runtime_config_store"); - ConfigRetriever staticConfigRetriever = configRetrieverFactory.create(vertx, featureFlagBootstrapConfig, ""); - return Future.all(dynamicConfigFuture, ConfigService.create(staticConfigRetriever)); - }) - .compose(configServiceManagerCompositeFuture -> { - ConfigService dynamicConfigService = configServiceManagerCompositeFuture.resultAt(0); - ConfigService staticConfigService = configServiceManagerCompositeFuture.resultAt(1); - boolean featureFlag = featureFlagConfigRetriever.getCachedConfig().getJsonObject("remote_config").getBoolean(Const.Config.RemoteConfigFeatureFlagProp, false); + ConfigRetriever staticConfigRetriever = ConfigRetrieverFactory.create(vertx, featureFlagBootstrapConfig, ""); + Future staticConfigFuture = ConfigService.create(staticConfigRetriever); - ConfigServiceManager configServiceManager = new ConfigServiceManager( - vertx, dynamicConfigService, staticConfigService, featureFlag); + return Future.all(dynamicConfigFuture, staticConfigFuture); + }) + .onComplete(ar -> { + if (ar.succeeded()) { + CompositeFuture configServiceManagerCompositeFuture = ar.result(); + IConfigService dynamicConfigService = configServiceManagerCompositeFuture.resultAt(0); + IConfigService staticConfigService = configServiceManagerCompositeFuture.resultAt(1); + + boolean remoteConfigFeatureFlag = featureFlagConfigRetriever.getCachedConfig() + .getJsonObject("remote_config") + .getBoolean(Const.Config.RemoteConfigFeatureFlagProp, false); + + ConfigServiceManager configServiceManager = new ConfigServiceManager( + vertx, dynamicConfigService, staticConfigService, remoteConfigFeatureFlag); + + setupFeatureFlagListener(configServiceManager, featureFlagConfigRetriever); + + IConfigService configService = configServiceManager.getDelegatingConfigService(); + promise.complete(configService); + } else { + LOGGER.error("Failed to initialise ConfigService: ", ar.cause()); + promise.fail(ar.cause()); + } + }); + return promise.future(); + } + + private void run() throws Exception { + this.createVertxInstancesMetric(); + this.createVertxEventLoopsMetric(); - setupFeatureFlagListener(configServiceManager, featureFlagConfigRetriever); + this.initialiseConfigService() + .compose(configService -> { - IConfigService configService = configServiceManager.getDelegatingConfigService(); Supplier operatorVerticleSupplier = () -> { UIDOperatorVerticle verticle = new UIDOperatorVerticle(configService, config, this.clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, getKeyManager(), saltProvider, optOutStore, Clock.systemUTC(), _statsCollectorQueue, new SecureLinkValidatorService(this.serviceLinkProvider, this.serviceProvider), this.shutdownHandler::handleSaltRetrievalResponse); return verticle; diff --git a/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java b/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java index e967b5c29..0df88955b 100644 --- a/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java +++ b/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java @@ -9,7 +9,7 @@ import static com.uid2.operator.Const.Config.ConfigScanPeriodMsProp; public class ConfigRetrieverFactory { - public ConfigRetriever create(Vertx vertx, JsonObject bootstrapConfig, String operatorKey) { + public static ConfigRetriever create(Vertx vertx, JsonObject bootstrapConfig, String operatorKey) { String type = bootstrapConfig.getString("type"); JsonObject storeConfig = bootstrapConfig.getJsonObject("config"); if (type.equals("http")) { diff --git a/src/main/java/com/uid2/operator/service/ConfigService.java b/src/main/java/com/uid2/operator/service/ConfigService.java index 4ac779a0a..98b73c97d 100644 --- a/src/main/java/com/uid2/operator/service/ConfigService.java +++ b/src/main/java/com/uid2/operator/service/ConfigService.java @@ -26,6 +26,7 @@ public static Future create(ConfigRetriever configRetriever) { ConfigService instance = new ConfigService(configRetriever); + // Prevent dependent classes from attempting to access configuration before it has been retrieved. configRetriever.getConfig(ar -> { if (ar.succeeded()) { logger.info("Successfully loaded config"); diff --git a/src/main/java/com/uid2/operator/service/DelegatingConfigService.java b/src/main/java/com/uid2/operator/service/DelegatingConfigService.java index a34d8b1c7..caf5016a4 100644 --- a/src/main/java/com/uid2/operator/service/DelegatingConfigService.java +++ b/src/main/java/com/uid2/operator/service/DelegatingConfigService.java @@ -4,7 +4,7 @@ import java.util.concurrent.atomic.AtomicReference; -public class DelegatingConfigService implements IConfigService{ +public class DelegatingConfigService implements IConfigService { private final AtomicReference activeConfigService; public DelegatingConfigService(IConfigService initialConfigService) { diff --git a/src/main/java/com/uid2/operator/service/UIDOperatorService.java b/src/main/java/com/uid2/operator/service/UIDOperatorService.java index c7e5b813c..d85d4d4ed 100644 --- a/src/main/java/com/uid2/operator/service/UIDOperatorService.java +++ b/src/main/java/com/uid2/operator/service/UIDOperatorService.java @@ -49,7 +49,7 @@ public class UIDOperatorService implements IUIDOperatorService { private final Handler saltRetrievalResponseHandler; public UIDOperatorService(IOptOutStore optOutStore, ISaltProvider saltProvider, ITokenEncoder encoder, Clock clock, - IdentityScope identityScope, Handler saltRetrievalResponseHandler, Boolean identityV3Enabled) { + IdentityScope identityScope, Handler saltRetrievalResponseHandler, boolean identityV3Enabled) { this.saltProvider = saltProvider; this.encoder = encoder; this.optOutStore = optOutStore; @@ -76,8 +76,21 @@ public UIDOperatorService(IOptOutStore optOutStore, ISaltProvider saltProvider, this.rawUidV3Enabled = identityV3Enabled; } + private void validateTokenDurations(Duration refreshIdentityAfter, Duration refreshExpiresAfter, Duration identityExpiresAfter) { + if (identityExpiresAfter.compareTo(refreshExpiresAfter) > 0) { + throw new IllegalStateException(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS + " must be >= " + IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS); + } + if (refreshIdentityAfter.compareTo(identityExpiresAfter) > 0) { + throw new IllegalStateException(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS + " must be >= " + REFRESH_IDENTITY_TOKEN_AFTER_SECONDS); + } + if (refreshIdentityAfter.compareTo(refreshExpiresAfter) > 0) { + throw new IllegalStateException(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS + " must be >= " + REFRESH_IDENTITY_TOKEN_AFTER_SECONDS); + } + } + @Override public IdentityTokens generateIdentity(IdentityRequest request, Duration refreshIdentityAfter, Duration refreshExpiresAfter, Duration identityExpiresAfter) { + this.validateTokenDurations(refreshIdentityAfter, refreshExpiresAfter, identityExpiresAfter); final Instant now = EncodingUtils.NowUTCMillis(this.clock); final byte[] firstLevelHash = getFirstLevelHash(request.userIdentity.id, now); final UserIdentity firstLevelHashIdentity = new UserIdentity( @@ -93,6 +106,7 @@ public IdentityTokens generateIdentity(IdentityRequest request, Duration refresh @Override public RefreshResponse refreshIdentity(RefreshToken token, Duration refreshIdentityAfter, Duration refreshExpiresAfter, Duration identityExpiresAfter) { + this.validateTokenDurations(refreshIdentityAfter, refreshExpiresAfter, identityExpiresAfter); // should not be possible as different scopes should be using different keys, but just in case if (token.userIdentity.identityScope != this.identityScope) { return RefreshResponse.Invalid; diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index 6643f924c..166a077ad 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -96,8 +96,8 @@ public class UIDOperatorVerticle extends AbstractVerticle { private final IOptOutStore optOutStore; private final IClientKeyProvider clientKeyProvider; private final Clock clock; - private final Boolean allowLegacyAPI; - private final Boolean identityV3Enabled; + private final boolean allowLegacyAPI; + private final boolean identityV3Enabled; protected IUIDOperatorService idService; private final Map _identityMapMetricSummaries = new HashMap<>(); private final Map, DistributionSummary> _refreshDurationMetricSummaries = new HashMap<>(); diff --git a/src/test/java/com/uid2/operator/ConfigServiceTest.java b/src/test/java/com/uid2/operator/ConfigServiceTest.java index 473be6492..b1b448b02 100644 --- a/src/test/java/com/uid2/operator/ConfigServiceTest.java +++ b/src/test/java/com/uid2/operator/ConfigServiceTest.java @@ -27,7 +27,6 @@ class ConfigServiceTest { private JsonObject bootstrapConfig; private JsonObject runtimeConfig; private HttpServer server; - private ConfigRetrieverFactory configRetrieverFactory; @BeforeEach void setUp() { @@ -47,8 +46,6 @@ void setUp() { .put(MaxBidstreamLifetimeSecondsProp, 7200) .put(SharingTokenExpiryProp, 3600); - configRetrieverFactory = new ConfigRetrieverFactory(); - } @AfterEach @@ -87,7 +84,7 @@ private Future startMockServer(JsonObject config) { @Test void testGetConfig(VertxTestContext testContext) { - ConfigRetriever configRetriever = configRetrieverFactory.create(vertx, bootstrapConfig, ""); + ConfigRetriever configRetriever = ConfigRetrieverFactory.create(vertx, bootstrapConfig, ""); JsonObject httpStoreConfig = runtimeConfig; startMockServer(httpStoreConfig) .compose(v -> ConfigService.create(configRetriever)) @@ -110,7 +107,7 @@ void testInvalidConfigRevertsToPrevious(VertxTestContext testContext) { .put("type", "json") .put("config", invalidConfig) .put(ConfigScanPeriodMsProp, -1); - ConfigRetriever spyRetriever = spy(configRetrieverFactory.create(vertx, jsonBootstrapConfig, "")); + ConfigRetriever spyRetriever = spy(ConfigRetrieverFactory.create(vertx, jsonBootstrapConfig, "")); when(spyRetriever.getCachedConfig()).thenReturn(lastConfig); ConfigService.create(spyRetriever) .compose(configService -> { @@ -130,7 +127,7 @@ void testFirstInvalidConfigThrowsRuntimeException(VertxTestContext testContext) .put("type", "json") .put("config", invalidConfig) .put(ConfigScanPeriodMsProp, -1); - ConfigRetriever configRetriever = configRetrieverFactory.create(vertx, jsonBootstrapConfig, ""); + ConfigRetriever configRetriever = ConfigRetrieverFactory.create(vertx, jsonBootstrapConfig, ""); ConfigService.create(configRetriever) .onComplete(testContext.failing(throwable -> { assertThrows(RuntimeException.class, () -> { From 45efc3215aad06589de4618fdffc80d21ecac972 Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Fri, 31 Jan 2025 16:07:38 +1100 Subject: [PATCH 229/431] Address Review Comments - fix type inconsistencies - minor code cleanup - replace getConfig() method from BenchmarkCommon with final static fields --- .../operator/vertx/UIDOperatorVerticle.java | 18 ++++----- .../operator/ConfigServiceManagerTest.java | 2 +- .../uid2/operator/UIDOperatorServiceTest.java | 2 +- .../operator/UIDOperatorVerticleTest.java | 37 ++++++++---------- .../operator/benchmark/BenchmarkCommon.java | 34 ++++++----------- .../benchmark/TokenEndecBenchmark.java | 38 +++++++++---------- 6 files changed, 55 insertions(+), 76 deletions(-) diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index 166a077ad..068196015 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -135,7 +135,7 @@ public class UIDOperatorVerticle extends AbstractVerticle { private static final String ERROR_INVALID_INPUT_WITH_PHONE_SUPPORT = "Required Parameter Missing: exactly one of [email, email_hash, phone, phone_hash] must be specified"; private static final String ERROR_INVALID_INPUT_EMAIL_MISSING = "Required Parameter Missing: exactly one of email or email_hash must be specified"; private static final String ERROR_INVALID_INPUT_EMAIL_TWICE = "Only one of email or email_hash can be specified"; - public static final String Config = "config"; + private static final String RC_CONFIG_KEY = "remote-config"; public final static String ORIGIN_HEADER = "Origin"; public UIDOperatorVerticle(IConfigService configService, @@ -237,7 +237,7 @@ private Router createRoutesSetup() throws IOException { router.route("/static/*").handler(StaticHandler.create("static")); router.route().handler(ctx -> { JsonObject curConfig = configService.getConfig(); - ctx.put(Config, curConfig); + ctx.put(RC_CONFIG_KEY, curConfig); ctx.next(); }); router.route().failureHandler(new GenericFailureHandler()); @@ -318,7 +318,7 @@ private void handleClientSideTokenGenerate(RoutingContext rc) { } private JsonObject getConfigFromRc(RoutingContext rc) { - return rc.get(Config); + return rc.get(RC_CONFIG_KEY); } private Set getDomainNameListForClientSideTokenGenerate(ClientSideKeypair keypair) { @@ -623,8 +623,8 @@ public void handleKeysRequest(RoutingContext rc) { public void handleKeysSharing(RoutingContext rc) { JsonObject config = this.getConfigFromRc(rc); - Integer maxSharingLifetimeSeconds = config.getInteger(Const.Config.MaxSharingLifetimeProp, config.getInteger(Const.Config.SharingTokenExpiryProp)); - String sharingTokenExpirySeconds = config.getString(Const.Config.SharingTokenExpiryProp); + int sharingTokenExpirySeconds = config.getInteger(Const.Config.SharingTokenExpiryProp); + int maxSharingLifetimeSeconds = config.getInteger(Const.Config.MaxSharingLifetimeProp, sharingTokenExpirySeconds); try { final ClientKey clientKey = AuthMiddleware.getAuthClient(ClientKey.class, rc); @@ -681,7 +681,7 @@ public void handleKeysBidstream(RoutingContext rc) { JsonObject config = this.getConfigFromRc(rc); Integer identityTokenExpiresAfterSeconds = config.getInteger(UIDOperatorService.IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS); - Integer maxBidstreamLifetimeSeconds = config.getInteger(Const.Config.MaxBidstreamLifetimeSecondsProp, identityTokenExpiresAfterSeconds); + int maxBidstreamLifetimeSeconds = config.getInteger(Const.Config.MaxBidstreamLifetimeSecondsProp, identityTokenExpiresAfterSeconds); addBidstreamHeaderFields(resp, maxBidstreamLifetimeSeconds); @@ -691,7 +691,7 @@ public void handleKeysBidstream(RoutingContext rc) { ResponseUtil.SuccessV2(rc, resp); } - private void addBidstreamHeaderFields(JsonObject resp, Integer maxBidstreamLifetimeSeconds) { + private void addBidstreamHeaderFields(JsonObject resp, int maxBidstreamLifetimeSeconds) { resp.put("max_bidstream_lifetime_seconds", maxBidstreamLifetimeSeconds + TOKEN_LIFETIME_TOLERANCE.toSeconds()); addIdentityScopeField(resp); addAllowClockSkewSecondsField(resp); @@ -729,7 +729,7 @@ private void addSites(JsonObject resp, List keys, Map saltRetrievalResponseHandler, Boolean identityV3Enabled) { + public ExtendedUIDOperatorService(IOptOutStore optOutStore, ISaltProvider saltProvider, ITokenEncoder encoder, Clock clock, IdentityScope identityScope, Handler saltRetrievalResponseHandler, boolean identityV3Enabled) { super(optOutStore, saltProvider, encoder, clock, identityScope, saltRetrievalResponseHandler, identityV3Enabled); } } diff --git a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java index 3b8449715..e226b29a6 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java @@ -5099,7 +5099,6 @@ void tokenGenerateRespectsConfigValues(Vertx vertx, VertxTestContext testContext setupSalts(); setupKeys(); - String v1Param = "email=" + emailAddress; JsonObject v2Payload = new JsonObject(); v2Payload.put("email", emailAddress); @@ -5107,27 +5106,22 @@ void tokenGenerateRespectsConfigValues(Vertx vertx, VertxTestContext testContext Duration newRefreshExpiresAfter = Duration.ofMinutes(30); Duration newRefreshIdentityAfter = Duration.ofMinutes(10); - config.put(UIDOperatorService.IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, newIdentityExpiresAfter.toMillis() / 1000); - config.put(UIDOperatorService.REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, newRefreshExpiresAfter.toMillis() / 1000); - config.put(UIDOperatorService.REFRESH_IDENTITY_TOKEN_AFTER_SECONDS, newRefreshIdentityAfter.toMillis() / 1000); - when(configService.getConfig()).thenReturn(config); + config.put(UIDOperatorService.IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, newIdentityExpiresAfter.toSeconds()); + config.put(UIDOperatorService.REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, newRefreshExpiresAfter.toSeconds()); + config.put(UIDOperatorService.REFRESH_IDENTITY_TOKEN_AFTER_SECONDS, newRefreshIdentityAfter.toSeconds()); - try { - sendTokenGenerate("v2", vertx, - v1Param, v2Payload, 200, - respJson -> { + sendTokenGenerate("v2", vertx, + null, v2Payload, 200, + respJson -> { + testContext.verify(() -> { JsonObject body = respJson.getJsonObject("body"); - testContext.verify(() -> { - assertNotNull(body); - assertEquals(now.plusMillis(newIdentityExpiresAfter.toMillis()).toEpochMilli(), body.getLong("identity_expires")); - assertEquals(now.plusMillis(newRefreshExpiresAfter.toMillis()).toEpochMilli(), body.getLong("refresh_expires")); - assertEquals(now.plusMillis(newRefreshIdentityAfter.toMillis()).toEpochMilli(), body.getLong("refresh_from")); - }); - testContext.completeNow(); + assertNotNull(body); + assertEquals(now.plusMillis(newIdentityExpiresAfter.toMillis()).toEpochMilli(), body.getLong("identity_expires")); + assertEquals(now.plusMillis(newRefreshExpiresAfter.toMillis()).toEpochMilli(), body.getLong("refresh_expires")); + assertEquals(now.plusMillis(newRefreshIdentityAfter.toMillis()).toEpochMilli(), body.getLong("refresh_from")); }); - } catch (Exception e) { - testContext.failNow(e); - } + testContext.completeNow(); + }); } @Test @@ -5152,9 +5146,8 @@ void keySharingRespectsConfigValues(Vertx vertx, VertxTestContext testContext) { MultipleKeysetsTests test = new MultipleKeysetsTests(Arrays.asList(keysets), Arrays.asList(encryptionKeys)); setupSiteDomainAndAppNameMock(true, false, 101, 102, 103, 104, 105); send(apiVersion, vertx, apiVersion + "/key/sharing", true, null, null, 200, respJson -> { - - JsonObject body = respJson.getJsonObject("body"); testContext.verify(() -> { + JsonObject body = respJson.getJsonObject("body"); assertNotNull(body); assertEquals(newSharingTokenExpiry, Integer.parseInt(body.getString("token_expiry_seconds"))); assertEquals(newMaxSharingLifetimeSeconds + TOKEN_LIFETIME_TOLERANCE.toSeconds(), body.getLong(Const.Config.MaxSharingLifetimeProp)); @@ -5178,8 +5171,8 @@ void keyBidstreamRespectsConfigValues(Vertx vertx, VertxTestContext testContext) new MultipleKeysetsTests(); send(apiVersion, vertx, apiVersion + endpoint.getPath(), true, null, null, 200, respJson -> { - JsonObject body = respJson.getJsonObject("body"); testContext.verify(() -> { + JsonObject body = respJson.getJsonObject("body"); assertNotNull(body); assertEquals(newMaxBidstreamLifetimeSeconds + TOKEN_LIFETIME_TOLERANCE.toSeconds(), body.getLong(Const.Config.MaxBidstreamLifetimeSecondsProp)); }); diff --git a/src/test/java/com/uid2/operator/benchmark/BenchmarkCommon.java b/src/test/java/com/uid2/operator/benchmark/BenchmarkCommon.java index d5a2279bf..aea998a94 100644 --- a/src/test/java/com/uid2/operator/benchmark/BenchmarkCommon.java +++ b/src/test/java/com/uid2/operator/benchmark/BenchmarkCommon.java @@ -41,29 +41,29 @@ import java.util.List; import java.util.Random; -import static com.uid2.operator.Const.Config.IdentityV3Prop; - public class BenchmarkCommon { - static IUIDOperatorService createUidOperatorService() throws Exception { + final static int IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS = 600; + final static int REFRESH_TOKEN_EXPIRES_AFTER_SECONDS = 900; + final static int REFRESH_IDENTITY_TOKEN_AFTER_SECONDS = 300; + + static IUIDOperatorService createUidOperatorService() throws Exception { RotatingKeysetKeyStore keysetKeyStore = new RotatingKeysetKeyStore( new EmbeddedResourceStorage(Main.class), new GlobalScope(new CloudPath("/com.uid2.core/test/keyset_keys/metadata.json"))); keysetKeyStore.loadContent(); - RotatingKeysetProvider keysetProvider = new RotatingKeysetProvider( - new EmbeddedResourceStorage(Main.class), - new GlobalScope(new CloudPath("/com.uid2.core/test/keysets/metadata.json"))); - keysetProvider.loadContent(); + RotatingKeysetProvider keysetProvider = new RotatingKeysetProvider( + new EmbeddedResourceStorage(Main.class), + new GlobalScope(new CloudPath("/com.uid2.core/test/keysets/metadata.json"))); + keysetProvider.loadContent(); RotatingSaltProvider saltProvider = new RotatingSaltProvider( new EmbeddedResourceStorage(Main.class), "/com.uid2.core/test/salts/metadata.json"); saltProvider.loadContent(); - final JsonObject config = getConfig(); - - final EncryptedTokenEncoder tokenEncoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); + final EncryptedTokenEncoder tokenEncoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); final List optOutPartitionFiles = new ArrayList<>(); final ICloudStorage optOutLocalStorage = make1mOptOutEntryStorage( saltProvider.getSnapshot(Instant.now()).getFirstLevelSalt(), @@ -77,22 +77,10 @@ static IUIDOperatorService createUidOperatorService() throws Exception { Clock.systemUTC(), IdentityScope.UID2, null, - config.getBoolean(IdentityV3Prop) + false ); } - public static JsonObject getConfig() { - final int IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS = 600; - final int REFRESH_TOKEN_EXPIRES_AFTER_SECONDS = 900; - final int REFRESH_IDENTITY_TOKEN_AFTER_SECONDS = 300; - - final JsonObject config = new JsonObject(); - config.put(UIDOperatorService.IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS); - config.put(UIDOperatorService.REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, REFRESH_TOKEN_EXPIRES_AFTER_SECONDS); - config.put(UIDOperatorService.REFRESH_IDENTITY_TOKEN_AFTER_SECONDS, REFRESH_IDENTITY_TOKEN_AFTER_SECONDS); - return config; - } - static EncryptedTokenEncoder createTokenEncoder() throws Exception { RotatingKeysetKeyStore keysetKeyStore = new RotatingKeysetKeyStore( new EmbeddedResourceStorage(Main.class), diff --git a/src/test/java/com/uid2/operator/benchmark/TokenEndecBenchmark.java b/src/test/java/com/uid2/operator/benchmark/TokenEndecBenchmark.java index d1fba74be..1dfeeb95f 100644 --- a/src/test/java/com/uid2/operator/benchmark/TokenEndecBenchmark.java +++ b/src/test/java/com/uid2/operator/benchmark/TokenEndecBenchmark.java @@ -3,7 +3,6 @@ import com.uid2.operator.model.*; import com.uid2.operator.service.EncryptedTokenEncoder; import com.uid2.operator.service.IUIDOperatorService; -import io.vertx.core.json.JsonObject; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; import org.openjdk.jmh.annotations.Mode; @@ -12,8 +11,6 @@ import java.util.ArrayList; import java.util.List; -import static com.uid2.operator.service.UIDOperatorService.*; - public class TokenEndecBenchmark { private static final IUIDOperatorService uidService; @@ -22,7 +19,6 @@ public class TokenEndecBenchmark { private static final EncryptedTokenEncoder encoder; private static final IdentityTokens[] generatedTokens; private static int idx = 0; - private static final JsonObject config; static { try { @@ -34,7 +30,6 @@ public class TokenEndecBenchmark { if (generatedTokens.length < 65536 || userIdentities.length < 65536) { throw new IllegalStateException("must create more than 65535 test candidates."); } - config = BenchmarkCommon.getConfig(); } catch (Exception e) { throw new RuntimeException(e); } @@ -46,12 +41,13 @@ static IdentityTokens[] createAdvertisingTokens() { tokens.add( uidService.generateIdentity( new IdentityRequest( - publisher, - userIdentities[i], - OptoutCheckPolicy.DoNotRespect), - Duration.ofSeconds(config.getInteger(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS)), - Duration.ofSeconds(config.getInteger(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS)), - Duration.ofSeconds(config.getInteger(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)))); + publisher, + userIdentities[i], + OptoutCheckPolicy.DoNotRespect), + Duration.ofSeconds(BenchmarkCommon.REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(BenchmarkCommon.REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(BenchmarkCommon.IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)) + ); } return tokens.toArray(new IdentityTokens[tokens.size()]); } @@ -60,12 +56,13 @@ static IdentityTokens[] createAdvertisingTokens() { @BenchmarkMode(Mode.Throughput) public IdentityTokens TokenGenerationBenchmark() { return uidService.generateIdentity(new IdentityRequest( - publisher, - userIdentities[(idx++) & 65535], - OptoutCheckPolicy.DoNotRespect), - Duration.ofSeconds(config.getInteger(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS)), - Duration.ofSeconds(config.getInteger(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS)), - Duration.ofSeconds(config.getInteger(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS))); + publisher, + userIdentities[(idx++) & 65535], + OptoutCheckPolicy.DoNotRespect), + Duration.ofSeconds(BenchmarkCommon.REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(BenchmarkCommon.REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(BenchmarkCommon.IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS) + ); } @Benchmark @@ -74,8 +71,9 @@ public RefreshResponse TokenRefreshBenchmark() { return uidService.refreshIdentity( encoder.decodeRefreshToken( generatedTokens[(idx++) & 65535].getRefreshToken()), - Duration.ofSeconds(config.getInteger(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS)), - Duration.ofSeconds(config.getInteger(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS)), - Duration.ofSeconds(config.getInteger(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS))); + Duration.ofSeconds(BenchmarkCommon.REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(BenchmarkCommon.REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(BenchmarkCommon.IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS) + ); } } From 292641f67741ded7cc426d45fe43ed00f868a1c8 Mon Sep 17 00:00:00 2001 From: abuabraham-ttd <130511456+abuabraham-ttd@users.noreply.github.com> Date: Fri, 31 Jan 2025 10:53:16 -0800 Subject: [PATCH 230/431] use operator_key (#1333) * use api_key if operator_key not present --- scripts/aws/ec2.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/scripts/aws/ec2.py b/scripts/aws/ec2.py index f9c4feb28..3e56c820d 100644 --- a/scripts/aws/ec2.py +++ b/scripts/aws/ec2.py @@ -90,13 +90,14 @@ def _set_confidential_config(self, secret_identifier: str) -> None: """Fetches a secret value from AWS Secrets Manager and adds defaults""" def add_defaults(configs: Dict[str, any]) -> AWSConfidentialComputeConfig: - """Adds default values to configuration if missing.""" + """Adds default values to configuration if missing. Sets operator_key if only api_token is specified for backward compatibility """ default_capacity = self.__get_max_capacity() + configs.setdefault("operator_key", configs.get("api_token")) configs.setdefault("enclave_memory_mb", default_capacity["enclave_memory_mb"]) configs.setdefault("enclave_cpu_count", default_capacity["enclave_cpu_count"]) configs.setdefault("debug_mode", False) - configs.setdefault("core_api_token", configs.get("api_token", "")) - configs.setdefault("optout_api_token", configs.get("api_token", "")) + configs.setdefault("core_api_token", configs.get("operator_key")) + configs.setdefault("optout_api_token", configs.get("operator_key")) return configs region = self.__get_current_region() @@ -258,4 +259,4 @@ def __kill_auxiliaries(self) -> None: print("Failed starting up Confidential Compute. Please checks the logs for errors and retry \n", e) except Exception as e: print("Unexpected failure while starting up Confidential Compute. Please contact UID support team with this log \n ", e) - \ No newline at end of file + From 34dbbd96bc94dfa080231c96ba898a7d05442129 Mon Sep 17 00:00:00 2001 From: abuabraham-ttd <130511456+abuabraham-ttd@users.noreply.github.com> Date: Fri, 31 Jan 2025 14:30:43 -0800 Subject: [PATCH 231/431] Update to extract core_api_token (#1334) --- scripts/aws/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/aws/entrypoint.sh b/scripts/aws/entrypoint.sh index 93a9662ba..15138b60f 100755 --- a/scripts/aws/entrypoint.sh +++ b/scripts/aws/entrypoint.sh @@ -28,7 +28,7 @@ setup_auxiliaries() { build_parameterized_config() { curl -s -f -o "${PARAMETERIZED_CONFIG}" -x socks5h://127.0.0.1:3305 http://127.0.0.1:27015/getConfig - REQUIRED_KEYS=("optout_base_url" "core_base_url" "api_token" "environment") + REQUIRED_KEYS=("optout_base_url" "core_base_url" "core_api_token" "optout_api_token" "environment") for key in "${REQUIRED_KEYS[@]}"; do if ! jq -e "has(\"${key}\")" "${PARAMETERIZED_CONFIG}" > /dev/null; then echo "Error: Key '${key}' is missing. Please add it to flask config server" From 8b66537a0ed9f5abecf366d89db32388875af802 Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Mon, 3 Feb 2025 09:56:37 +1100 Subject: [PATCH 232/431] Update config files for private operators - Copy feat-flag.json file in dockerfiles - Remove unnecessary config values - Add runtime_config_store config value --- Dockerfile | 2 ++ conf/docker-config.json | 14 +++++++++----- conf/local-e2e-docker-private-config.json | 14 ++++++++++---- conf/local-e2e-private-config.json | 16 ++++++++++------ conf/local-e2e-public-config.json | 16 ++++++++++------ ...alidator-latest-e2e-docker-public-config.json | 15 ++++++++++----- scripts/aws/Dockerfile | 1 + scripts/aws/conf/default-config.json | 1 - scripts/aws/conf/feat-flag/feat-flag.json | 15 +++++++++++++++ scripts/aws/conf/integ-euid-config.json | 12 +++++++++++- scripts/aws/conf/integ-uid2-config.json | 12 +++++++++++- scripts/aws/conf/prod-euid-config.json | 15 +++++++++++---- scripts/aws/conf/prod-uid2-config.json | 15 +++++++++++---- scripts/azure-cc/Dockerfile | 1 + scripts/azure-cc/conf/default-config.json | 4 ---- scripts/azure-cc/conf/feat-flag/feat-flag.json | 15 +++++++++++++++ scripts/azure-cc/conf/integ-uid2-config.json | 12 +++++++++++- scripts/azure-cc/conf/prod-uid2-config.json | 11 ++++++++++- scripts/gcp-oidc/Dockerfile | 1 + scripts/gcp-oidc/conf/default-config.json | 4 ---- scripts/gcp-oidc/conf/feat-flag/feat-flag.json | 15 +++++++++++++++ scripts/gcp-oidc/conf/integ-uid2-config.json | 12 +++++++++++- scripts/gcp-oidc/conf/prod-uid2-config.json | 11 ++++++++++- 23 files changed, 185 insertions(+), 49 deletions(-) create mode 100644 scripts/aws/conf/feat-flag/feat-flag.json create mode 100644 scripts/azure-cc/conf/feat-flag/feat-flag.json create mode 100644 scripts/gcp-oidc/conf/feat-flag/feat-flag.json diff --git a/Dockerfile b/Dockerfile index c698202c2..fb448af6e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,6 +20,8 @@ COPY ./target/${JAR_NAME}-${JAR_VERSION}-sources.jar /app COPY ./target/${JAR_NAME}-${JAR_VERSION}-static.tar.gz /app/static.tar.gz COPY ./conf/default-config.json ${EXTRA_CONFIG} /app/conf/ COPY ./conf/*.xml /app/conf/ +COPY ./conf/runtime-config-defaults.json /app/conf/ +COPY ./conf/feat-flag/feat-flag.json /app/conf/feat-flag/ RUN tar xzvf /app/static.tar.gz --no-same-owner --no-same-permissions && rm -f /app/static.tar.gz diff --git a/conf/docker-config.json b/conf/docker-config.json index 25f38e6ae..82367734c 100644 --- a/conf/docker-config.json +++ b/conf/docker-config.json @@ -2,8 +2,6 @@ "service_verbose": true, "service_instances": 1, "storage_mock": true, - "refresh_token_expires_after_seconds": 86400, - "refresh_identity_token_after_seconds": 900, "refresh_token_v3": false, "identity_v3": false, "identity_scope": "uid2", @@ -31,12 +29,18 @@ "salts_metadata_path": "/com.uid2.core/test/salts/metadata.json", "services_metadata_path": "/com.uid2.core/test/services/metadata.json", "service_links_metadata_path": "/com.uid2.core/test/service_links/metadata.json", - "identity_token_expires_after_seconds": 3600, "optout_metadata_path": null, "optout_inmem_cache": false, "enclave_platform": null, "failure_shutdown_wait_hours": 120, "salts_expired_shutdown_hours": 12, - "operator_type": "public" - + "operator_type": "public", + "runtime_config_store": { + "type": "file", + "config" : { + "path": "conf/runtime-config-defaults.json", + "format": "json" + }, + "config_scan_period_ms": 5000 + } } diff --git a/conf/local-e2e-docker-private-config.json b/conf/local-e2e-docker-private-config.json index 8637e6da3..dceeaf7c4 100644 --- a/conf/local-e2e-docker-private-config.json +++ b/conf/local-e2e-docker-private-config.json @@ -11,9 +11,6 @@ "keysets_metadata_path": "http://core:8088/key/keyset/refresh", "keyset_keys_metadata_path": "http://core:8088/key/keyset-keys/refresh", "salts_metadata_path": "http://core:8088/salt/refresh", - "identity_token_expires_after_seconds": 3600, - "refresh_token_expires_after_seconds": 86400, - "refresh_identity_token_after_seconds": 900, "refresh_token_v3": true, "identity_v3": false, "identity_scope": "uid2", @@ -27,5 +24,14 @@ "optout_delta_rotate_interval": 60, "cloud_refresh_interval": 30, "salts_expired_shutdown_hours": 12, - "operator_type": "private" + "operator_type": "private", + "runtime_config_store": { + "type": "http", + "config" : { + "host": "core", + "port": 8088, + "path": "/operator/config" + }, + "config_scan_period_ms": 300000 + } } diff --git a/conf/local-e2e-private-config.json b/conf/local-e2e-private-config.json index 4ab52330f..130624d1e 100644 --- a/conf/local-e2e-private-config.json +++ b/conf/local-e2e-private-config.json @@ -13,14 +13,10 @@ "salts_metadata_path": "http://localhost:8088/salt/refresh", "services_metadata_path": "http://localhost:8088/services/refresh", "service_links_metadata_path": "http://localhost:8088/service_links/refresh", - "identity_token_expires_after_seconds": 3600, - "refresh_token_expires_after_seconds": 86400, - "refresh_identity_token_after_seconds": 900, "refresh_token_v3": true, "identity_v3": false, "identity_scope": "uid2", "enable_v2_encryption": true, - "sharing_token_expiry_seconds": 2592000, "cloud_download_threads": 8, "cloud_upload_threads": 2, "cloud_refresh_interval": 60, @@ -38,6 +34,14 @@ "client_side_token_generate_domain_name_check_enabled": false, "client_side_token_generate_log_invalid_http_origins": true, "salts_expired_shutdown_hours": 12, - "operator_type": "private" - + "operator_type": "private", + "runtime_config_store": { + "type": "http", + "config" : { + "host": "localhost", + "port": 8088, + "path": "/operator/config" + }, + "config_scan_period_ms": 300000 + } } diff --git a/conf/local-e2e-public-config.json b/conf/local-e2e-public-config.json index bfdc8e394..d09491b2f 100644 --- a/conf/local-e2e-public-config.json +++ b/conf/local-e2e-public-config.json @@ -13,14 +13,10 @@ "salts_metadata_path": "http://localhost:8088/salt/refresh", "services_metadata_path": "http://localhost:8088/services/refresh", "service_links_metadata_path": "http://localhost:8088/service_links/refresh", - "identity_token_expires_after_seconds": 3600, - "refresh_token_expires_after_seconds": 86400, - "refresh_identity_token_after_seconds": 900, "refresh_token_v3": true, "identity_v3": false, "identity_scope": "uid2", "enable_v2_encryption": true, - "sharing_token_expiry_seconds": 2592000, "cloud_download_threads": 8, "cloud_upload_threads": 2, "cloud_refresh_interval": 60, @@ -39,6 +35,14 @@ "key_sharing_endpoint_provide_app_names": true, "client_side_token_generate_log_invalid_http_origins": true, "salts_expired_shutdown_hours": 12, - "operator_type": "public" - + "operator_type": "public", + "runtime_config_store": { + "type": "http", + "config" : { + "host": "localhost", + "port": 8088, + "path": "/operator/config" + }, + "config_scan_period_ms": 300000 + } } diff --git a/conf/validator-latest-e2e-docker-public-config.json b/conf/validator-latest-e2e-docker-public-config.json index 8f82b01a4..d04e70109 100644 --- a/conf/validator-latest-e2e-docker-public-config.json +++ b/conf/validator-latest-e2e-docker-public-config.json @@ -14,9 +14,6 @@ "salts_metadata_path": "http://core:8088/salt/refresh", "services_metadata_path": "http://core:8088/services/refresh", "service_links_metadata_path": "http://core:8088/service_links/refresh", - "identity_token_expires_after_seconds": 3600, - "refresh_token_expires_after_seconds": 86400, - "refresh_identity_token_after_seconds": 900, "refresh_token_v3": true, "identity_v3": false, "identity_scope": "uid2", @@ -32,6 +29,14 @@ "optout_api_uri": "http://optout:8081/optout/replicate", "optout_delta_rotate_interval": 60, "cloud_refresh_interval": 30, - "operator_type": "public" - + "operator_type": "public", + "runtime_config_store": { + "type": "http", + "config" : { + "host": "core", + "port": 8088, + "path": "/operator/config" + }, + "config_scan_period_ms": 300000 + } } diff --git a/scripts/aws/Dockerfile b/scripts/aws/Dockerfile index e210001c3..6ecb329b9 100644 --- a/scripts/aws/Dockerfile +++ b/scripts/aws/Dockerfile @@ -41,6 +41,7 @@ COPY ./conf/prod-euid-config.json /app/conf/ COPY ./conf/integ-euid-config.json /app/conf/ COPY ./conf/*.xml /app/conf/ COPY ./syslog-ng-client.conf /etc/syslog-ng/syslog-ng.conf +COPY ./conf/feat-flag/feat-flag.json /app/conf/feat-flag/ RUN chmod +x /app/vsockpx && chmod +x /app/entrypoint.sh diff --git a/scripts/aws/conf/default-config.json b/scripts/aws/conf/default-config.json index 35c3be58c..cdd71b32d 100644 --- a/scripts/aws/conf/default-config.json +++ b/scripts/aws/conf/default-config.json @@ -32,7 +32,6 @@ "optout_inmem_cache": false, "enclave_platform": null, "failure_shutdown_wait_hours": 120, - "sharing_token_expiry_seconds": 2592000, "validate_service_links": false, "operator_type": "private" } diff --git a/scripts/aws/conf/feat-flag/feat-flag.json b/scripts/aws/conf/feat-flag/feat-flag.json new file mode 100644 index 000000000..85fac0e10 --- /dev/null +++ b/scripts/aws/conf/feat-flag/feat-flag.json @@ -0,0 +1,15 @@ +{ + "remote_config": { + "enabled": false, + "runtime_config_store": { + "type": "json", + "config": { + "identity_token_expires_after_seconds": 3600, + "refresh_token_expires_after_seconds": 86400, + "refresh_identity_token_after_seconds": 900, + "sharing_token_expiry_seconds": 2592000 + }, + "config_scan_period_ms": -1 + } + } +} \ No newline at end of file diff --git a/scripts/aws/conf/integ-euid-config.json b/scripts/aws/conf/integ-euid-config.json index 45d3dbe94..14dcbcafb 100644 --- a/scripts/aws/conf/integ-euid-config.json +++ b/scripts/aws/conf/integ-euid-config.json @@ -11,5 +11,15 @@ "core_attest_url": "https://core.integ.euid.eu/attest", "optout_api_uri": "https://optout.integ.euid.eu/optout/replicate", "optout_s3_folder": "optout/", - "allow_legacy_api": false + "allow_legacy_api": false, + "runtime_config_store": { + "type": "http", + "config" : { + "host": "core.integ.euid.eu", + "port": 443, + "ssl": true, + "path": "/operator/config" + }, + "config_scan_period_ms": 300000 + } } diff --git a/scripts/aws/conf/integ-uid2-config.json b/scripts/aws/conf/integ-uid2-config.json index a7272a26a..b3c48ab81 100644 --- a/scripts/aws/conf/integ-uid2-config.json +++ b/scripts/aws/conf/integ-uid2-config.json @@ -11,5 +11,15 @@ "core_attest_url": "https://core-integ.uidapi.com/attest", "optout_api_uri": "https://optout-integ.uidapi.com/optout/replicate", "optout_s3_folder": "uid-optout-integ/", - "allow_legacy_api": false + "allow_legacy_api": false, + "runtime_config_store": { + "type": "http", + "config" : { + "host": "core-integ.uidapi.com", + "port": 443, + "ssl": true, + "path": "/operator/config" + }, + "config_scan_period_ms": 300000 + } } diff --git a/scripts/aws/conf/prod-euid-config.json b/scripts/aws/conf/prod-euid-config.json index 0fbf5d69c..73183d639 100644 --- a/scripts/aws/conf/prod-euid-config.json +++ b/scripts/aws/conf/prod-euid-config.json @@ -19,13 +19,20 @@ "optout_synthetic_logs_count": 0, "optout_inmem_cache": true, "optout_s3_folder": "optout/", - "identity_token_expires_after_seconds": 259200, - "refresh_token_expires_after_seconds": 2592000, - "refresh_identity_token_after_seconds": 3600, "allow_legacy_api": false, "identity_scope": "euid", "refresh_token_v3": true, "enable_phone_support": true, "enable_v1_phone_support": false, - "enable_v2_encryption": true + "enable_v2_encryption": true, + "runtime_config_store": { + "type": "http", + "config" : { + "host": "core.prod.euid.eu", + "port": 443, + "ssl": true, + "path": "/operator/config" + }, + "config_scan_period_ms": 300000 + } } diff --git a/scripts/aws/conf/prod-uid2-config.json b/scripts/aws/conf/prod-uid2-config.json index 5da450033..cad9056d7 100644 --- a/scripts/aws/conf/prod-uid2-config.json +++ b/scripts/aws/conf/prod-uid2-config.json @@ -19,8 +19,15 @@ "optout_synthetic_logs_count": 0, "optout_inmem_cache": true, "optout_s3_folder": "optout-v2/", - "identity_token_expires_after_seconds": 259200, - "refresh_token_expires_after_seconds": 2592000, - "refresh_identity_token_after_seconds": 3600, - "allow_legacy_api": false + "allow_legacy_api": false, + "runtime_config_store": { + "type": "http", + "config" : { + "host": "core-prod.uidapi.com", + "port": 443, + "ssl": true, + "path": "/operator/config" + }, + "config_scan_period_ms": 300000 + } } diff --git a/scripts/azure-cc/Dockerfile b/scripts/azure-cc/Dockerfile index bb0c96b70..eebb4eda1 100644 --- a/scripts/azure-cc/Dockerfile +++ b/scripts/azure-cc/Dockerfile @@ -22,6 +22,7 @@ COPY ./target/${JAR_NAME}-${JAR_VERSION}-sources.jar /app COPY ./target/${JAR_NAME}-${JAR_VERSION}-static.tar.gz /app/static.tar.gz COPY ./conf/*.json /app/conf/ COPY ./conf/*.xml /app/conf/ +COPY ./conf/feat-flag/feat-flag.json /app/conf/feat-flag/ RUN tar xzvf /app/static.tar.gz --no-same-owner --no-same-permissions && rm -f /app/static.tar.gz diff --git a/scripts/azure-cc/conf/default-config.json b/scripts/azure-cc/conf/default-config.json index 4870b2fda..25c7cb200 100644 --- a/scripts/azure-cc/conf/default-config.json +++ b/scripts/azure-cc/conf/default-config.json @@ -31,12 +31,8 @@ "optout_metadata_path": null, "enclave_platform": "azure-cc", "optout_inmem_cache": true, - "identity_token_expires_after_seconds": 86400, - "refresh_token_expires_after_seconds": 2592000, - "refresh_identity_token_after_seconds": 3600, "allow_legacy_api": false, "failure_shutdown_wait_hours": 120, - "sharing_token_expiry_seconds": 2592000, "validate_service_links": false, "operator_type": "private" } diff --git a/scripts/azure-cc/conf/feat-flag/feat-flag.json b/scripts/azure-cc/conf/feat-flag/feat-flag.json new file mode 100644 index 000000000..85fac0e10 --- /dev/null +++ b/scripts/azure-cc/conf/feat-flag/feat-flag.json @@ -0,0 +1,15 @@ +{ + "remote_config": { + "enabled": false, + "runtime_config_store": { + "type": "json", + "config": { + "identity_token_expires_after_seconds": 3600, + "refresh_token_expires_after_seconds": 86400, + "refresh_identity_token_after_seconds": 900, + "sharing_token_expiry_seconds": 2592000 + }, + "config_scan_period_ms": -1 + } + } +} \ No newline at end of file diff --git a/scripts/azure-cc/conf/integ-uid2-config.json b/scripts/azure-cc/conf/integ-uid2-config.json index 2cd4be5c3..d1b697ad7 100644 --- a/scripts/azure-cc/conf/integ-uid2-config.json +++ b/scripts/azure-cc/conf/integ-uid2-config.json @@ -10,5 +10,15 @@ "optout_metadata_path": "https://optout-integ.uidapi.com/optout/refresh", "core_attest_url": "https://core-integ.uidapi.com/attest", "optout_api_uri": "https://optout-integ.uidapi.com/optout/replicate", - "optout_s3_folder": "uid-optout-integ/" + "optout_s3_folder": "uid-optout-integ/", + "runtime_config_store": { + "type": "http", + "config" : { + "host": "core-integ.uidapi.com", + "port": 443, + "ssl": true, + "path": "/operator/config" + }, + "config_scan_period_ms": 300000 + } } diff --git a/scripts/azure-cc/conf/prod-uid2-config.json b/scripts/azure-cc/conf/prod-uid2-config.json index 02e2cde20..397eecc64 100644 --- a/scripts/azure-cc/conf/prod-uid2-config.json +++ b/scripts/azure-cc/conf/prod-uid2-config.json @@ -11,5 +11,14 @@ "core_attest_url": "https://core-prod.uidapi.com/attest", "optout_api_uri": "https://optout-prod.uidapi.com/optout/replicate", "optout_s3_folder": "optout-v2/", - "identity_token_expires_after_seconds": 259200 + "runtime_config_store": { + "type": "http", + "config" : { + "host": "core-prod.uidapi.com", + "port": 443, + "ssl": true, + "path": "/operator/config" + }, + "config_scan_period_ms": 300000 + } } diff --git a/scripts/gcp-oidc/Dockerfile b/scripts/gcp-oidc/Dockerfile index 76b302e30..c7ac4a2ae 100644 --- a/scripts/gcp-oidc/Dockerfile +++ b/scripts/gcp-oidc/Dockerfile @@ -25,6 +25,7 @@ COPY ./target/${JAR_NAME}-${JAR_VERSION}-sources.jar /app COPY ./target/${JAR_NAME}-${JAR_VERSION}-static.tar.gz /app/static.tar.gz COPY ./conf/*.json /app/conf/ COPY ./conf/*.xml /app/conf/ +COPY ./conf/feat-flag/feat-flag.json /app/conf/feat-flag/ RUN tar xzvf /app/static.tar.gz --no-same-owner --no-same-permissions && rm -f /app/static.tar.gz diff --git a/scripts/gcp-oidc/conf/default-config.json b/scripts/gcp-oidc/conf/default-config.json index 6a65ee2d0..af8455a79 100644 --- a/scripts/gcp-oidc/conf/default-config.json +++ b/scripts/gcp-oidc/conf/default-config.json @@ -31,12 +31,8 @@ "optout_metadata_path": null, "enclave_platform": "gcp-oidc", "optout_inmem_cache": true, - "identity_token_expires_after_seconds": 86400, - "refresh_token_expires_after_seconds": 2592000, - "refresh_identity_token_after_seconds": 3600, "allow_legacy_api": false, "failure_shutdown_wait_hours": 120, - "sharing_token_expiry_seconds": 2592000, "validate_service_links": false, "operator_type": "private" } diff --git a/scripts/gcp-oidc/conf/feat-flag/feat-flag.json b/scripts/gcp-oidc/conf/feat-flag/feat-flag.json new file mode 100644 index 000000000..85fac0e10 --- /dev/null +++ b/scripts/gcp-oidc/conf/feat-flag/feat-flag.json @@ -0,0 +1,15 @@ +{ + "remote_config": { + "enabled": false, + "runtime_config_store": { + "type": "json", + "config": { + "identity_token_expires_after_seconds": 3600, + "refresh_token_expires_after_seconds": 86400, + "refresh_identity_token_after_seconds": 900, + "sharing_token_expiry_seconds": 2592000 + }, + "config_scan_period_ms": -1 + } + } +} \ No newline at end of file diff --git a/scripts/gcp-oidc/conf/integ-uid2-config.json b/scripts/gcp-oidc/conf/integ-uid2-config.json index 935514b5a..b4b4f95bf 100644 --- a/scripts/gcp-oidc/conf/integ-uid2-config.json +++ b/scripts/gcp-oidc/conf/integ-uid2-config.json @@ -10,5 +10,15 @@ "optout_metadata_path": "https://optout.uidapi.com/optout/refresh", "core_attest_url": "https://core.uidapi.com/attest", "optout_api_uri": "https://optout.uidapi.com/optout/replicate", - "optout_s3_folder": "uid-optout-integ/" + "optout_s3_folder": "uid-optout-integ/", + "runtime_config_store": { + "type": "http", + "config" : { + "host": "core.uidapi.com", + "port": 443, + "ssl": true, + "path": "/operator/config" + }, + "config_scan_period_ms": 300000 + } } diff --git a/scripts/gcp-oidc/conf/prod-uid2-config.json b/scripts/gcp-oidc/conf/prod-uid2-config.json index f5445a9ec..fda859658 100644 --- a/scripts/gcp-oidc/conf/prod-uid2-config.json +++ b/scripts/gcp-oidc/conf/prod-uid2-config.json @@ -11,5 +11,14 @@ "core_attest_url": "https://core.uidapi.com/attest", "optout_api_uri": "https://optout.uidapi.com/optout/replicate", "optout_s3_folder": "optout-v2/", - "identity_token_expires_after_seconds": 259200 + "runtime_config_store": { + "type": "http", + "config" : { + "host": "core.uidapi.com", + "port": 443, + "ssl": true, + "path": "/operator/config" + }, + "config_scan_period_ms": 300000 + } } From 0a59cb63cb9b7da4fc66c92e171744ef5dc074d0 Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Mon, 3 Feb 2025 16:10:27 +1100 Subject: [PATCH 233/431] Undo run config changes --- .../uid2-operator_standalone.xml | 26 ------------------- 1 file changed, 26 deletions(-) diff --git a/.idea/runConfigurations/uid2-operator_standalone.xml b/.idea/runConfigurations/uid2-operator_standalone.xml index 823760e90..9eaad01dd 100644 --- a/.idea/runConfigurations/uid2-operator_standalone.xml +++ b/.idea/runConfigurations/uid2-operator_standalone.xml @@ -19,32 +19,6 @@ From 6f87884bafcd9547e79ee92e86b293cc297c061d Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Mon, 3 Feb 2025 16:11:44 +1100 Subject: [PATCH 234/431] Undo run config changes --- .../uid2-operator_standalone.xml | 39 ++++++++++++------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/.idea/runConfigurations/uid2-operator_standalone.xml b/.idea/runConfigurations/uid2-operator_standalone.xml index 9eaad01dd..511bb9905 100644 --- a/.idea/runConfigurations/uid2-operator_standalone.xml +++ b/.idea/runConfigurations/uid2-operator_standalone.xml @@ -2,23 +2,34 @@ - + From b47557ebe9a25cf5390fe5e4fc0337d6f0f1212e Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Mon, 3 Feb 2025 16:29:20 +1100 Subject: [PATCH 235/431] Undo removal of runtime config values from config files --- conf/default-config.json | 1 + conf/docker-config.json | 3 + conf/feat-flag/feat-flag.json | 12 +-- conf/local-config.json | 4 + conf/local-e2e-docker-private-config.json | 3 + conf/local-e2e-docker-public-config.json | 3 + conf/local-e2e-private-config.json | 4 + conf/local-e2e-public-config.json | 4 + ...dator-latest-e2e-docker-public-config.json | 3 + scripts/aws/conf/default-config.json | 1 + scripts/aws/conf/euid-prod-config.json | 3 + scripts/aws/conf/feat-flag/feat-flag.json | 12 +-- scripts/aws/conf/uid2-prod-config.json | 3 + scripts/azure-cc/conf/default-config.json | 4 + .../azure-cc/conf/feat-flag/feat-flag.json | 12 +-- scripts/azure-cc/conf/prod-uid2-config.json | 1 + scripts/gcp-oidc/conf/default-config.json | 76 ++++++++++--------- .../gcp-oidc/conf/feat-flag/feat-flag.json | 12 +-- scripts/gcp-oidc/conf/prod-config.json | 49 ++++++------ 19 files changed, 106 insertions(+), 104 deletions(-) diff --git a/conf/default-config.json b/conf/default-config.json index 5a74087fe..a57739a7f 100644 --- a/conf/default-config.json +++ b/conf/default-config.json @@ -34,5 +34,6 @@ "optout_inmem_cache": false, "enclave_platform": null, "failure_shutdown_wait_hours": 120, + "sharing_token_expiry_seconds": 2592000, "operator_type": "public" } diff --git a/conf/docker-config.json b/conf/docker-config.json index 82367734c..17266d2bd 100644 --- a/conf/docker-config.json +++ b/conf/docker-config.json @@ -2,6 +2,8 @@ "service_verbose": true, "service_instances": 1, "storage_mock": true, + "refresh_token_expires_after_seconds": 86400, + "refresh_identity_token_after_seconds": 900, "refresh_token_v3": false, "identity_v3": false, "identity_scope": "uid2", @@ -29,6 +31,7 @@ "salts_metadata_path": "/com.uid2.core/test/salts/metadata.json", "services_metadata_path": "/com.uid2.core/test/services/metadata.json", "service_links_metadata_path": "/com.uid2.core/test/service_links/metadata.json", + "identity_token_expires_after_seconds": 3600, "optout_metadata_path": null, "optout_inmem_cache": false, "enclave_platform": null, diff --git a/conf/feat-flag/feat-flag.json b/conf/feat-flag/feat-flag.json index 45e2f6d7c..6e816fb99 100644 --- a/conf/feat-flag/feat-flag.json +++ b/conf/feat-flag/feat-flag.json @@ -1,15 +1,5 @@ { "remote_config": { - "enabled": true, - "runtime_config_store": { - "type": "json", - "config": { - "identity_token_expires_after_seconds": 3600, - "refresh_token_expires_after_seconds": 86400, - "refresh_identity_token_after_seconds": 900, - "sharing_token_expiry_seconds": 2592000 - }, - "config_scan_period_ms": -1 - } + "enabled": true } } \ No newline at end of file diff --git a/conf/local-config.json b/conf/local-config.json index 8de6fbd91..6a91eecb2 100644 --- a/conf/local-config.json +++ b/conf/local-config.json @@ -9,10 +9,14 @@ "salts_metadata_path": "/com.uid2.core/test/salts/metadata.json", "services_metadata_path": "/com.uid2.core/test/services/metadata.json", "service_links_metadata_path": "/com.uid2.core/test/service_links/metadata.json", + "identity_token_expires_after_seconds": 3600, + "refresh_token_expires_after_seconds": 86400, + "refresh_identity_token_after_seconds": 900, "refresh_token_v3": false, "identity_v3": false, "identity_scope": "uid2", "enable_v2_encryption": false, + "sharing_token_expiry_seconds": 2592000, "cloud_download_threads": 8, "cloud_upload_threads": 2, "cloud_refresh_interval": 60, diff --git a/conf/local-e2e-docker-private-config.json b/conf/local-e2e-docker-private-config.json index dceeaf7c4..218f64e2c 100644 --- a/conf/local-e2e-docker-private-config.json +++ b/conf/local-e2e-docker-private-config.json @@ -11,6 +11,9 @@ "keysets_metadata_path": "http://core:8088/key/keyset/refresh", "keyset_keys_metadata_path": "http://core:8088/key/keyset-keys/refresh", "salts_metadata_path": "http://core:8088/salt/refresh", + "identity_token_expires_after_seconds": 3600, + "refresh_token_expires_after_seconds": 86400, + "refresh_identity_token_after_seconds": 900, "refresh_token_v3": true, "identity_v3": false, "identity_scope": "uid2", diff --git a/conf/local-e2e-docker-public-config.json b/conf/local-e2e-docker-public-config.json index 756453681..3357a2b85 100644 --- a/conf/local-e2e-docker-public-config.json +++ b/conf/local-e2e-docker-public-config.json @@ -13,6 +13,9 @@ "salts_metadata_path": "http://core:8088/salt/refresh", "services_metadata_path": "http://core:8088/services/refresh", "service_links_metadata_path": "http://core:8088/service_links/refresh", + "identity_token_expires_after_seconds": 3600, + "refresh_token_expires_after_seconds": 86400, + "refresh_identity_token_after_seconds": 900, "refresh_token_v3": true, "identity_v3": false, "identity_scope": "uid2", diff --git a/conf/local-e2e-private-config.json b/conf/local-e2e-private-config.json index 130624d1e..00d94d552 100644 --- a/conf/local-e2e-private-config.json +++ b/conf/local-e2e-private-config.json @@ -13,10 +13,14 @@ "salts_metadata_path": "http://localhost:8088/salt/refresh", "services_metadata_path": "http://localhost:8088/services/refresh", "service_links_metadata_path": "http://localhost:8088/service_links/refresh", + "identity_token_expires_after_seconds": 3600, + "refresh_token_expires_after_seconds": 86400, + "refresh_identity_token_after_seconds": 900, "refresh_token_v3": true, "identity_v3": false, "identity_scope": "uid2", "enable_v2_encryption": true, + "sharing_token_expiry_seconds": 2592000, "cloud_download_threads": 8, "cloud_upload_threads": 2, "cloud_refresh_interval": 60, diff --git a/conf/local-e2e-public-config.json b/conf/local-e2e-public-config.json index d09491b2f..edd7af5cb 100644 --- a/conf/local-e2e-public-config.json +++ b/conf/local-e2e-public-config.json @@ -13,10 +13,14 @@ "salts_metadata_path": "http://localhost:8088/salt/refresh", "services_metadata_path": "http://localhost:8088/services/refresh", "service_links_metadata_path": "http://localhost:8088/service_links/refresh", + "identity_token_expires_after_seconds": 3600, + "refresh_token_expires_after_seconds": 86400, + "refresh_identity_token_after_seconds": 900, "refresh_token_v3": true, "identity_v3": false, "identity_scope": "uid2", "enable_v2_encryption": true, + "sharing_token_expiry_seconds": 2592000, "cloud_download_threads": 8, "cloud_upload_threads": 2, "cloud_refresh_interval": 60, diff --git a/conf/validator-latest-e2e-docker-public-config.json b/conf/validator-latest-e2e-docker-public-config.json index d04e70109..f9b201ee8 100644 --- a/conf/validator-latest-e2e-docker-public-config.json +++ b/conf/validator-latest-e2e-docker-public-config.json @@ -14,6 +14,9 @@ "salts_metadata_path": "http://core:8088/salt/refresh", "services_metadata_path": "http://core:8088/services/refresh", "service_links_metadata_path": "http://core:8088/service_links/refresh", + "identity_token_expires_after_seconds": 3600, + "refresh_token_expires_after_seconds": 86400, + "refresh_identity_token_after_seconds": 900, "refresh_token_v3": true, "identity_v3": false, "identity_scope": "uid2", diff --git a/scripts/aws/conf/default-config.json b/scripts/aws/conf/default-config.json index cdd71b32d..35c3be58c 100644 --- a/scripts/aws/conf/default-config.json +++ b/scripts/aws/conf/default-config.json @@ -32,6 +32,7 @@ "optout_inmem_cache": false, "enclave_platform": null, "failure_shutdown_wait_hours": 120, + "sharing_token_expiry_seconds": 2592000, "validate_service_links": false, "operator_type": "private" } diff --git a/scripts/aws/conf/euid-prod-config.json b/scripts/aws/conf/euid-prod-config.json index efd774316..ac00370bc 100644 --- a/scripts/aws/conf/euid-prod-config.json +++ b/scripts/aws/conf/euid-prod-config.json @@ -19,6 +19,9 @@ "optout_synthetic_logs_count": 0, "optout_inmem_cache": true, "optout_s3_folder": "optout/", + "identity_token_expires_after_seconds": 259200, + "refresh_token_expires_after_seconds": 2592000, + "refresh_identity_token_after_seconds": 3600, "allow_legacy_api": false, "identity_scope": "euid", "refresh_token_v3": true, diff --git a/scripts/aws/conf/feat-flag/feat-flag.json b/scripts/aws/conf/feat-flag/feat-flag.json index 85fac0e10..a296546fb 100644 --- a/scripts/aws/conf/feat-flag/feat-flag.json +++ b/scripts/aws/conf/feat-flag/feat-flag.json @@ -1,15 +1,5 @@ { "remote_config": { - "enabled": false, - "runtime_config_store": { - "type": "json", - "config": { - "identity_token_expires_after_seconds": 3600, - "refresh_token_expires_after_seconds": 86400, - "refresh_identity_token_after_seconds": 900, - "sharing_token_expiry_seconds": 2592000 - }, - "config_scan_period_ms": -1 - } + "enabled": false } } \ No newline at end of file diff --git a/scripts/aws/conf/uid2-prod-config.json b/scripts/aws/conf/uid2-prod-config.json index f1e102010..a46670746 100644 --- a/scripts/aws/conf/uid2-prod-config.json +++ b/scripts/aws/conf/uid2-prod-config.json @@ -19,6 +19,9 @@ "optout_synthetic_logs_count": 0, "optout_inmem_cache": true, "optout_s3_folder": "optout-v2/", + "identity_token_expires_after_seconds": 259200, + "refresh_token_expires_after_seconds": 2592000, + "refresh_identity_token_after_seconds": 3600, "allow_legacy_api": false, "runtime_config_store": { "type": "http", diff --git a/scripts/azure-cc/conf/default-config.json b/scripts/azure-cc/conf/default-config.json index 25c7cb200..4870b2fda 100644 --- a/scripts/azure-cc/conf/default-config.json +++ b/scripts/azure-cc/conf/default-config.json @@ -31,8 +31,12 @@ "optout_metadata_path": null, "enclave_platform": "azure-cc", "optout_inmem_cache": true, + "identity_token_expires_after_seconds": 86400, + "refresh_token_expires_after_seconds": 2592000, + "refresh_identity_token_after_seconds": 3600, "allow_legacy_api": false, "failure_shutdown_wait_hours": 120, + "sharing_token_expiry_seconds": 2592000, "validate_service_links": false, "operator_type": "private" } diff --git a/scripts/azure-cc/conf/feat-flag/feat-flag.json b/scripts/azure-cc/conf/feat-flag/feat-flag.json index 85fac0e10..a296546fb 100644 --- a/scripts/azure-cc/conf/feat-flag/feat-flag.json +++ b/scripts/azure-cc/conf/feat-flag/feat-flag.json @@ -1,15 +1,5 @@ { "remote_config": { - "enabled": false, - "runtime_config_store": { - "type": "json", - "config": { - "identity_token_expires_after_seconds": 3600, - "refresh_token_expires_after_seconds": 86400, - "refresh_identity_token_after_seconds": 900, - "sharing_token_expiry_seconds": 2592000 - }, - "config_scan_period_ms": -1 - } + "enabled": false } } \ No newline at end of file diff --git a/scripts/azure-cc/conf/prod-uid2-config.json b/scripts/azure-cc/conf/prod-uid2-config.json index 397eecc64..1305fffcf 100644 --- a/scripts/azure-cc/conf/prod-uid2-config.json +++ b/scripts/azure-cc/conf/prod-uid2-config.json @@ -11,6 +11,7 @@ "core_attest_url": "https://core-prod.uidapi.com/attest", "optout_api_uri": "https://optout-prod.uidapi.com/optout/replicate", "optout_s3_folder": "optout-v2/", + "identity_token_expires_after_seconds": 259200, "runtime_config_store": { "type": "http", "config" : { diff --git a/scripts/gcp-oidc/conf/default-config.json b/scripts/gcp-oidc/conf/default-config.json index 49844e850..5d54e792d 100644 --- a/scripts/gcp-oidc/conf/default-config.json +++ b/scripts/gcp-oidc/conf/default-config.json @@ -1,38 +1,42 @@ { - "service_verbose": true, - "service_instances": 12, - "core_s3_bucket": null, - "core_attest_url": null, - "core_api_token": null, - "storage_mock": false, - "optout_s3_bucket": null, - "optout_s3_folder": "optout/", - "optout_s3_path_compat": false, - "optout_data_dir": "/opt/uid2/operator-optout/", - "optout_api_token": null, - "optout_api_uri": null, - "optout_bloom_filter_size": 8192, - "optout_delta_rotate_interval": 300, - "optout_delta_backtrack_in_days": 1, - "optout_partition_interval": 86400, - "optout_max_partitions": 30, - "optout_heap_default_capacity": 8192, - "cloud_download_threads": 8, - "cloud_upload_threads": 2, - "cloud_refresh_interval": 60, - "sites_metadata_path": "sites/metadata.json", - "clients_metadata_path": "clients/metadata.json", - "client_side_keypairs_metadata_path": "client_side_keypairs/metadata.json", - "keysets_metadata_path": "keysets/metadata.json", - "keyset_keys_metadata_path": "keyset_keys/metadata.json", - "salts_metadata_path": "salts/metadata.json", - "services_metadata_path": "services/metadata.json", - "service_links_metadata_path": "service_links/metadata.json", - "optout_metadata_path": null, - "enclave_platform": "gcp-oidc", - "optout_inmem_cache": true, - "allow_legacy_api": false, - "failure_shutdown_wait_hours": 120, - "validate_service_links": false, - "operator_type": "private" + "service_verbose": true, + "service_instances": 12, + "core_s3_bucket": null, + "core_attest_url": null, + "core_api_token": null, + "storage_mock": false, + "optout_s3_bucket": null, + "optout_s3_folder": "optout/", + "optout_s3_path_compat": false, + "optout_data_dir": "/opt/uid2/operator-optout/", + "optout_api_token": null, + "optout_api_uri": null, + "optout_bloom_filter_size": 8192, + "optout_delta_rotate_interval": 300, + "optout_delta_backtrack_in_days": 1, + "optout_partition_interval": 86400, + "optout_max_partitions": 30, + "optout_heap_default_capacity": 8192, + "cloud_download_threads": 8, + "cloud_upload_threads": 2, + "cloud_refresh_interval": 60, + "sites_metadata_path": "sites/metadata.json", + "clients_metadata_path": "clients/metadata.json", + "client_side_keypairs_metadata_path": "client_side_keypairs/metadata.json", + "keysets_metadata_path": "keysets/metadata.json", + "keyset_keys_metadata_path": "keyset_keys/metadata.json", + "salts_metadata_path": "salts/metadata.json", + "services_metadata_path": "services/metadata.json", + "service_links_metadata_path": "service_links/metadata.json", + "optout_metadata_path": null, + "enclave_platform": "gcp-oidc", + "optout_inmem_cache": true, + "identity_token_expires_after_seconds": 86400, + "refresh_token_expires_after_seconds": 2592000, + "refresh_identity_token_after_seconds": 3600, + "allow_legacy_api": false, + "failure_shutdown_wait_hours": 120, + "sharing_token_expiry_seconds": 2592000, + "validate_service_links": false, + "operator_type": "private" } \ No newline at end of file diff --git a/scripts/gcp-oidc/conf/feat-flag/feat-flag.json b/scripts/gcp-oidc/conf/feat-flag/feat-flag.json index 85fac0e10..a296546fb 100644 --- a/scripts/gcp-oidc/conf/feat-flag/feat-flag.json +++ b/scripts/gcp-oidc/conf/feat-flag/feat-flag.json @@ -1,15 +1,5 @@ { "remote_config": { - "enabled": false, - "runtime_config_store": { - "type": "json", - "config": { - "identity_token_expires_after_seconds": 3600, - "refresh_token_expires_after_seconds": 86400, - "refresh_identity_token_after_seconds": 900, - "sharing_token_expiry_seconds": 2592000 - }, - "config_scan_period_ms": -1 - } + "enabled": false } } \ No newline at end of file diff --git a/scripts/gcp-oidc/conf/prod-config.json b/scripts/gcp-oidc/conf/prod-config.json index fda859658..82c3a0d81 100644 --- a/scripts/gcp-oidc/conf/prod-config.json +++ b/scripts/gcp-oidc/conf/prod-config.json @@ -1,24 +1,25 @@ -{ - "sites_metadata_path": "https://core.uidapi.com/sites/refresh", - "clients_metadata_path": "https://core.uidapi.com/clients/refresh", - "keysets_metadata_path": "https://core.uidapi.com/key/keyset/refresh", - "keyset_keys_metadata_path": "https://core.uidapi.com/key/keyset-keys/refresh", - "client_side_keypairs_metadata_path": "https://core.uidapi.com/client_side_keypairs/refresh", - "salts_metadata_path": "https://core.uidapi.com/salt/refresh", - "services_metadata_path": "https://core.uidapi.com/services/refresh", - "service_links_metadata_path": "https://core.uidapi.com/service_links/refresh", - "optout_metadata_path": "https://optout.uidapi.com/optout/refresh", - "core_attest_url": "https://core.uidapi.com/attest", - "optout_api_uri": "https://optout.uidapi.com/optout/replicate", - "optout_s3_folder": "optout-v2/", - "runtime_config_store": { - "type": "http", - "config" : { - "host": "core.uidapi.com", - "port": 443, - "ssl": true, - "path": "/operator/config" - }, - "config_scan_period_ms": 300000 - } -} +{ + "sites_metadata_path": "https://core.uidapi.com/sites/refresh", + "clients_metadata_path": "https://core.uidapi.com/clients/refresh", + "keysets_metadata_path": "https://core.uidapi.com/key/keyset/refresh", + "keyset_keys_metadata_path": "https://core.uidapi.com/key/keyset-keys/refresh", + "client_side_keypairs_metadata_path": "https://core.uidapi.com/client_side_keypairs/refresh", + "salts_metadata_path": "https://core.uidapi.com/salt/refresh", + "services_metadata_path": "https://core.uidapi.com/services/refresh", + "service_links_metadata_path": "https://core.uidapi.com/service_links/refresh", + "optout_metadata_path": "https://optout.uidapi.com/optout/refresh", + "core_attest_url": "https://core.uidapi.com/attest", + "optout_api_uri": "https://optout.uidapi.com/optout/replicate", + "optout_s3_folder": "optout-v2/", + "identity_token_expires_after_seconds": 259200, + "runtime_config_store": { + "type": "http", + "config" : { + "host": "core.uidapi.com", + "port": 443, + "ssl": true, + "path": "/operator/config" + }, + "config_scan_period_ms": 300000 + } +} From e1b09e4d2b78b901c4f3ccb411cadcdfce42c4b4 Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Mon, 3 Feb 2025 16:33:07 +1100 Subject: [PATCH 236/431] Update static config service to use bootstrap config --- src/main/java/com/uid2/operator/Main.java | 27 ++++++++++++----------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index 1ae72c5e5..02ab4b952 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -277,6 +277,17 @@ private Future initialiseConfigService() throws Exception { ); Future dynamicConfigFuture = ConfigService.create(dynamicConfigRetriever); + ConfigRetriever staticConfigRetriever = ConfigRetrieverFactory.create( + vertx, + new JsonObject() + .put("type", "json") + .put("config", config) + .put(ConfigScanPeriodMsProp, -1), + "" + ); + + Future staticConfigFuture = ConfigService.create(staticConfigRetriever); + ConfigRetriever featureFlagConfigRetriever = ConfigRetrieverFactory.create( vertx, new JsonObject() @@ -288,27 +299,17 @@ private Future initialiseConfigService() throws Exception { "" ); - featureFlagConfigRetriever.getConfig() - .compose(featureFlagConfig -> { - if (featureFlagConfig == null) { - return Future.failedFuture(new RuntimeException("Feature flag config is null")); - } - - JsonObject remoteConfigJson = featureFlagConfig.getJsonObject("remote_config"); - JsonObject featureFlagBootstrapConfig = remoteConfigJson.getJsonObject("runtime_config_store"); - ConfigRetriever staticConfigRetriever = ConfigRetrieverFactory.create(vertx, featureFlagBootstrapConfig, ""); - Future staticConfigFuture = ConfigService.create(staticConfigRetriever); - return Future.all(dynamicConfigFuture, staticConfigFuture); - }) + Future.all(dynamicConfigFuture, staticConfigFuture, featureFlagConfigRetriever.getConfig()) .onComplete(ar -> { if (ar.succeeded()) { CompositeFuture configServiceManagerCompositeFuture = ar.result(); IConfigService dynamicConfigService = configServiceManagerCompositeFuture.resultAt(0); IConfigService staticConfigService = configServiceManagerCompositeFuture.resultAt(1); + JsonObject featureFlagConfig = configServiceManagerCompositeFuture.resultAt(2); - boolean remoteConfigFeatureFlag = featureFlagConfigRetriever.getCachedConfig() + boolean remoteConfigFeatureFlag = featureFlagConfig .getJsonObject("remote_config") .getBoolean(Const.Config.RemoteConfigFeatureFlagProp, false); From 63b0c4bafcc3086b9f4e3535aba9042c5db28c9f Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Mon, 3 Feb 2025 16:39:03 +1100 Subject: [PATCH 237/431] Update exception messages in validateTokenDurations method (UIDOperatorService) --- .../java/com/uid2/operator/service/UIDOperatorService.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/uid2/operator/service/UIDOperatorService.java b/src/main/java/com/uid2/operator/service/UIDOperatorService.java index d85d4d4ed..affbdde1c 100644 --- a/src/main/java/com/uid2/operator/service/UIDOperatorService.java +++ b/src/main/java/com/uid2/operator/service/UIDOperatorService.java @@ -78,13 +78,13 @@ public UIDOperatorService(IOptOutStore optOutStore, ISaltProvider saltProvider, private void validateTokenDurations(Duration refreshIdentityAfter, Duration refreshExpiresAfter, Duration identityExpiresAfter) { if (identityExpiresAfter.compareTo(refreshExpiresAfter) > 0) { - throw new IllegalStateException(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS + " must be >= " + IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS); + throw new IllegalStateException(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS + " (" + refreshExpiresAfter.toSeconds() + ") < " + IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS + " (" + identityExpiresAfter.toSeconds() + ")"); } if (refreshIdentityAfter.compareTo(identityExpiresAfter) > 0) { - throw new IllegalStateException(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS + " must be >= " + REFRESH_IDENTITY_TOKEN_AFTER_SECONDS); + throw new IllegalStateException(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS + " (" + identityExpiresAfter.toSeconds() + ") < " + REFRESH_IDENTITY_TOKEN_AFTER_SECONDS + " (" + refreshIdentityAfter.toSeconds() + ")"); } if (refreshIdentityAfter.compareTo(refreshExpiresAfter) > 0) { - throw new IllegalStateException(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS + " must be >= " + REFRESH_IDENTITY_TOKEN_AFTER_SECONDS); + throw new IllegalStateException(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS + " (" + refreshExpiresAfter.toSeconds() + ") < " + REFRESH_IDENTITY_TOKEN_AFTER_SECONDS + " (" + refreshIdentityAfter.toSeconds() + ")"); } } From 825a02aaad42b2e30e2ad8bc5b7b555f6494a6f3 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Mon, 3 Feb 2025 06:09:55 +0000 Subject: [PATCH 238/431] [CI Pipeline] Released Minor version: 5.46.0 --- pom.xml | 2 +- version.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 7e5db7312..69ee828da 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.45.8 + 5.46.0 UTF-8 diff --git a/version.json b/version.json index df99bb45c..54f0b5156 100644 --- a/version.json +++ b/version.json @@ -1 +1 @@ -{ "$schema": "https://raw.githubusercontent.com/dotnet/Nerdbank.GitVersioning/master/src/NerdBank.GitVersioning/version.schema.json", "version": "5.45", "publicReleaseRefSpec": [ "^refs/heads/master$", "^refs/heads/v\\d+(?:\\.\\d+)?$" ], "cloudBuild": { "setVersionVariables": true, "buildNumber": { "enabled": true, "includeCommitId": { "when": "always" } } } } +{ "$schema": "https://raw.githubusercontent.com/dotnet/Nerdbank.GitVersioning/master/src/NerdBank.GitVersioning/version.schema.json", "version": "5.46", "publicReleaseRefSpec": [ "^refs/heads/master$", "^refs/heads/v\\d+(?:\\.\\d+)?$" ], "cloudBuild": { "setVersionVariables": true, "buildNumber": { "enabled": true, "includeCommitId": { "when": "always" } } } } From 47bfe3b2bb52be50176a45b0cf59fa469241a184 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Mon, 3 Feb 2025 19:10:59 +0000 Subject: [PATCH 239/431] [CI Pipeline] Released Patch version: 5.46.2 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 69ee828da..a6c8a33b7 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.46.0 + 5.46.2 UTF-8 From 9d7c8b5d457bcefb36fc6816ef113e3147cb477a Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Tue, 4 Feb 2025 02:53:29 +0000 Subject: [PATCH 240/431] [CI Pipeline] Released Minor version: 5.47.0 --- pom.xml | 2 +- version.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index a6c8a33b7..2cb422883 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.46.2 + 5.47.0 UTF-8 diff --git a/version.json b/version.json index 54f0b5156..9090fa5b4 100644 --- a/version.json +++ b/version.json @@ -1 +1 @@ -{ "$schema": "https://raw.githubusercontent.com/dotnet/Nerdbank.GitVersioning/master/src/NerdBank.GitVersioning/version.schema.json", "version": "5.46", "publicReleaseRefSpec": [ "^refs/heads/master$", "^refs/heads/v\\d+(?:\\.\\d+)?$" ], "cloudBuild": { "setVersionVariables": true, "buildNumber": { "enabled": true, "includeCommitId": { "when": "always" } } } } +{ "$schema": "https://raw.githubusercontent.com/dotnet/Nerdbank.GitVersioning/master/src/NerdBank.GitVersioning/version.schema.json", "version": "5.47", "publicReleaseRefSpec": [ "^refs/heads/master$", "^refs/heads/v\\d+(?:\\.\\d+)?$" ], "cloudBuild": { "setVersionVariables": true, "buildNumber": { "enabled": true, "includeCommitId": { "when": "always" } } } } From c831e25ce9605e64659b2064bdd82aa827e114f4 Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Tue, 4 Feb 2025 18:28:36 +1100 Subject: [PATCH 241/431] Fix feature flag listener - retrieve `useDynamicConfig` toggle from correct location in feat-flag.json file --- src/main/java/com/uid2/operator/Main.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index 02ab4b952..7247d17a4 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -377,12 +377,12 @@ private void run() throws Exception { private void setupFeatureFlagListener(ConfigServiceManager manager, ConfigRetriever retriever) { retriever.listen(change -> { JsonObject newConfig = change.getNewConfiguration(); - boolean useDynamicConfig = newConfig.getBoolean(Const.Config.RemoteConfigFeatureFlagProp, true); + boolean useDynamicConfig = newConfig.getJsonObject("remote_config").getBoolean("enabled", true); manager.updateConfigService(useDynamicConfig).onComplete(update -> { if (update.succeeded()) { LOGGER.info("Remote config feature flag toggled successfully"); } else { - LOGGER.error("Failed to toggle remote config feature flag: " + update.cause()); + LOGGER.error("Failed to toggle remote config feature flag: ", update.cause()); } }); }); From dcbecdfdd004cde222ad01a79756213853247dff Mon Sep 17 00:00:00 2001 From: abuabraham-ttd <130511456+abuabraham-ttd@users.noreply.github.com> Date: Tue, 4 Feb 2025 14:30:18 -0800 Subject: [PATCH 242/431] Update to fix new config errors, change exception name (#1347) * Update to fix new config errors, change exception name --- scripts/aws/ec2.py | 20 ++--- scripts/aws/entrypoint.sh | 10 ++- scripts/azure-cc/azureEntryPoint.py | 28 +++--- scripts/azure-cc/deployment/operator.json | 14 +++ .../deployment/operator.parameters.json | 3 + scripts/azure-cc/entrypoint.sh | 87 ------------------- scripts/confidential_compute.py | 36 ++++---- scripts/gcp-oidc/gcp.py | 16 ++-- 8 files changed, 69 insertions(+), 145 deletions(-) delete mode 100644 scripts/azure-cc/entrypoint.sh diff --git a/scripts/aws/ec2.py b/scripts/aws/ec2.py index 3e56c820d..ab1775ff4 100644 --- a/scripts/aws/ec2.py +++ b/scripts/aws/ec2.py @@ -16,7 +16,7 @@ import yaml sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from confidential_compute import ConfidentialCompute, ConfidentialComputeConfig, MissingInstanceProfile, OperatorKeyNotFound, InvalidConfigValue, ConfidentialComputeStartupException +from confidential_compute import ConfidentialCompute, ConfidentialComputeConfig, InstanceProfileMissingError, OperatorKeyNotFoundError, ConfigurationValueError, ConfidentialComputeStartupError class AWSConfidentialComputeConfig(ConfidentialComputeConfig): enclave_memory_mb: int @@ -79,13 +79,11 @@ def __get_current_region(self) -> str: def __validate_aws_specific_config(self): if "enclave_memory_mb" in self.configs or "enclave_cpu_count" in self.configs: max_capacity = self.__get_max_capacity() - min_capacity = {"enclave_memory_mb": 11000, "enclave_cpu_count" : 2 } - for key in ["enclave_memory_mb", "enclave_cpu_count"]: - if int(self.configs.get(key, 0)) > max_capacity.get(key): - raise ValueError(f"{key} value ({self.configs.get(key, 0)}) exceeds the maximum allowed ({max_capacity.get(key)}).") - if min_capacity.get(key) > int(self.configs.get(key, 10**9)): - raise ValueError(f"{key} value ({self.configs.get(key, 0)}) needs to be higher than the minimum required ({min_capacity.get(key)}).") - + if self.configs.get('enclave_memory_mb') < 11000 or self.configs.get('enclave_memory_mb') > max_capacity.get('enclave_memory_mb'): + raise ConfigurationValueError(self.__class__.__name__, f"enclave_memory_mb must be in range 11000 and {max_capacity.get('enclave_memory_mb')}") + if self.configs.get('enclave_cpu_count') < 2 or self.configs.get('enclave_cpu_count') > max_capacity.get('enclave_cpu_count'): + raise ConfigurationValueError(self.__class__.__name__, f"enclave_cpu_count must be in range 2 and {max_capacity.get('enclave_cpu_count')}") + def _set_confidential_config(self, secret_identifier: str) -> None: """Fetches a secret value from AWS Secrets Manager and adds defaults""" @@ -107,9 +105,9 @@ def add_defaults(configs: Dict[str, any]) -> AWSConfidentialComputeConfig: self.configs = add_defaults(json.loads(client.get_secret_value(SecretId=secret_identifier)["SecretString"])) self.__validate_aws_specific_config() except NoCredentialsError as _: - raise MissingInstanceProfile(self.__class__.__name__) + raise InstanceProfileMissingError(self.__class__.__name__) except ClientError as _: - raise OperatorKeyNotFound(self.__class__.__name__, f"Secret Manager {secret_identifier} in {region}") + raise OperatorKeyNotFoundError(self.__class__.__name__, f"Secret Manager {secret_identifier} in {region}") @staticmethod def __get_max_capacity(): @@ -255,7 +253,7 @@ def __kill_auxiliaries(self) -> None: ec2.cleanup() else: ec2.run_compute() - except ConfidentialComputeStartupException as e: + except ConfidentialComputeStartupError as e: print("Failed starting up Confidential Compute. Please checks the logs for errors and retry \n", e) except Exception as e: print("Unexpected failure while starting up Confidential Compute. Please contact UID support team with this log \n ", e) diff --git a/scripts/aws/entrypoint.sh b/scripts/aws/entrypoint.sh index 15138b60f..5f324ce20 100755 --- a/scripts/aws/entrypoint.sh +++ b/scripts/aws/entrypoint.sh @@ -46,7 +46,9 @@ build_parameterized_config() { build_operator_config() { CORE_BASE_URL=$(jq -r ".core_base_url" < "${PARAMETERIZED_CONFIG}") + CORE_BASE_URL=$(echo "$CORE_BASE_URL" | sed -E 's#^(https?://)?([^/]+).*#\2#') OPTOUT_BASE_URL=$(jq -r ".optout_base_url" < "${PARAMETERIZED_CONFIG}") + OPTOUT_BASE_URL=$(echo "$OPTOUT_BASE_URL" | sed -E 's#^(https?://)?([^/]+).*#\2#') DEPLOYMENT_ENVIRONMENT=$(jq -r ".environment" < "${PARAMETERIZED_CONFIG}") DEBUG_MODE=$(jq -r ".debug_mode" < "${PARAMETERIZED_CONFIG}") @@ -67,11 +69,11 @@ build_operator_config() { if [[ "$DEPLOYMENT_ENVIRONMENT" != "prod" ]]; then #Allow override of base URL in non-prod environments - CORE_PATTERN="https://core.*uidapi.com" - OPTOUT_PATTERN="https://optout.*uidapi.com" + CORE_PATTERN="core.*uidapi.com" + OPTOUT_PATTERN="optout.*uidapi.com" if [[ "$DEPLOYMENT_ENVIRONMENT" == "euid" ]]; then - CORE_PATTERN="https://core.*euid.eu" - OPTOUT_PATTERN="https://optout.*euid.eu" + CORE_PATTERN="core.*euid.eu" + OPTOUT_PATTERN="optout.*euid.eu" fi sed -i "s#${CORE_PATTERN}#${CORE_BASE_URL}#g" "${OPERATOR_CONFIG}" sed -i "s#${OPTOUT_PATTERN}#${OPTOUT_BASE_URL}#g" "${OPERATOR_CONFIG}" diff --git a/scripts/azure-cc/azureEntryPoint.py b/scripts/azure-cc/azureEntryPoint.py index 534525d86..25348e9fd 100644 --- a/scripts/azure-cc/azureEntryPoint.py +++ b/scripts/azure-cc/azureEntryPoint.py @@ -8,12 +8,12 @@ import shutil import requests import logging - -sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from confidential_compute import ConfidentialCompute, MissingConfig, MissingInstanceProfile, AuxiliariesException, OperatorKeyAccessDenied, OperatorKeyNotFound, ConfidentialComputeStartupException +from urllib.parse import urlparse +from confidential_compute import ConfidentialCompute, ConfigurationMissingError, OperatorKeyPermissionError, OperatorKeyNotFoundError, ConfidentialComputeStartupError from azure.keyvault.secrets import SecretClient from azure.identity import DefaultAzureCredential, CredentialUnavailableError from azure.core.exceptions import ResourceNotFoundError, ClientAuthenticationError +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) class AzureEntryPoint(ConfidentialCompute): @@ -31,11 +31,11 @@ def __init__(self): def __check_env_variables(self): # Check essential env variables if AzureEntryPoint.kv_name is None: - raise MissingConfig(self.__class__.__name__, ["VAULT_NAME"]) + raise ConfigurationMissingError(self.__class__.__name__, ["VAULT_NAME"]) if AzureEntryPoint.secret_name is None: - raise MissingConfig(self.__class__.__name__, ["OPERATOR_KEY_SECRET_NAME"]) + raise ConfigurationMissingError(self.__class__.__name__, ["OPERATOR_KEY_SECRET_NAME"]) if AzureEntryPoint.env_name is None: - raise MissingConfig(self.__class__.__name__, ["DEPLOYMENT_ENVIRONMENT"]) + raise ConfigurationMissingError(self.__class__.__name__, ["DEPLOYMENT_ENVIRONMENT"]) logging.info("Environment variables validation success") def __create_final_config(self): @@ -53,13 +53,14 @@ def __create_final_config(self): CORE_BASE_URL = os.getenv("CORE_BASE_URL") OPTOUT_BASE_URL = os.getenv("OPTOUT_BASE_URL") + if CORE_BASE_URL and OPTOUT_BASE_URL and AzureEntryPoint.env_name != 'prod': logging.info(f"-- replacing URLs by {CORE_BASE_URL} and {OPTOUT_BASE_URL}") with open(AzureEntryPoint.FINAL_CONFIG, "r") as file: config = file.read() - config = config.replace("https://core-integ.uidapi.com", CORE_BASE_URL) - config = config.replace("https://optout-integ.uidapi.com", OPTOUT_BASE_URL) + config = config.replace("core-integ.uidapi.com", urlparse(CORE_BASE_URL).netloc) + config = config.replace("optout-integ.uidapi.com", urlparse(OPTOUT_BASE_URL).netloc) with open(AzureEntryPoint.FINAL_CONFIG, "w") as file: file.write(config) @@ -84,10 +85,10 @@ def __set_operator_key(self): except (CredentialUnavailableError, ClientAuthenticationError) as auth_error: logging.error(f"Read operator key, authentication error: {auth_error}") - raise OperatorKeyAccessDenied(self.__class__.__name__, str(auth_error)) + raise OperatorKeyPermissionError(self.__class__.__name__, str(auth_error)) except ResourceNotFoundError as not_found_error: logging.error(f"Read operator key, secret not found: {AzureEntryPoint.secret_name}. Error: {not_found_error}") - raise OperatorKeyNotFound(self.__class__.__name__, str(not_found_error)) + raise OperatorKeyNotFoundError(self.__class__.__name__, str(not_found_error)) def _set_confidential_config(self, secret_identifier: str = None): @@ -140,10 +141,7 @@ def _validate_auxiliaries(self): logging.info(f"Attempt {attempt}: Error during request - {e}") if attempt == MAX_RETRIES: - logging.error( - f"Sidecar failed to start after {MAX_RETRIES} attempts. Exiting." - ) - raise AuxiliariesException(self.__class__.__name__) + raise RuntimeError(f"Unable to detect sidecar running after {MAX_RETRIES} attempts. Exiting.") logging.info(f"Retrying in {delay} seconds... (Attempt {attempt}/{MAX_RETRIES})") time.sleep(delay) @@ -170,7 +168,7 @@ def _setup_auxiliaries(self) -> None: try: operator = AzureEntryPoint() operator.run_compute() - except ConfidentialComputeStartupException as e: + except ConfidentialComputeStartupError as e: logging.error(f"Failed starting up Azure Confidential Compute. Please checks the logs for errors and retry {e}", exc_info=True) except Exception as e: logging.error(f"Unexpected failure while starting up Azure Confidential Compute. Please contact UID support team with this log {e}", exc_info=True) \ No newline at end of file diff --git a/scripts/azure-cc/deployment/operator.json b/scripts/azure-cc/deployment/operator.json index b50ecced9..0348802de 100644 --- a/scripts/azure-cc/deployment/operator.json +++ b/scripts/azure-cc/deployment/operator.json @@ -54,6 +54,16 @@ "metadata": { "description": "Operator Key" } + }, + "skipValidations": { + "type": "string", + "metadata": { + "description": "Whether to skip pre-init validations" + }, + "allowedValues": [ + "true", + "false" + ] } }, "variables": { @@ -122,6 +132,10 @@ { "name": "DEPLOYMENT_ENVIRONMENT", "value": "[parameters('deploymentEnvironment')]" + }, + { + "name": "SKIP_VALIDATIONS", + "value": "[parameters('skipValidations')]" } ] } diff --git a/scripts/azure-cc/deployment/operator.parameters.json b/scripts/azure-cc/deployment/operator.parameters.json index 776690776..5095746ea 100644 --- a/scripts/azure-cc/deployment/operator.parameters.json +++ b/scripts/azure-cc/deployment/operator.parameters.json @@ -22,6 +22,9 @@ }, "deploymentEnvironment": { "value": "integ" + }, + "skipValidations": { + "value": "false" } } } diff --git a/scripts/azure-cc/entrypoint.sh b/scripts/azure-cc/entrypoint.sh deleted file mode 100644 index 14875c9bf..000000000 --- a/scripts/azure-cc/entrypoint.sh +++ /dev/null @@ -1,87 +0,0 @@ -#!/bin/sh -# -# This script must be compatible with Ash (provided in eclipse-temurin Docker image) and Bash - -function wait_for_sidecar() { - url="http://169.254.169.254/ping" - delay=1 - max_retries=15 - - while true; do - if wget -q --spider --tries=1 --timeout 5 "$url" > /dev/null; then - echo "side car started" - break - else - echo "side car not started. Retrying in $delay seconds..." - sleep $delay - if [ $delay -gt $max_retries ]; then - echo "side car failed to start" - break - fi - delay=$((delay + 1)) - fi - done -} - -TMP_FINAL_CONFIG="/tmp/final-config.tmp" - -if [ -z "${VAULT_NAME}" ]; then - echo "VAULT_NAME cannot be empty" - exit 1 -fi - -if [ -z "${OPERATOR_KEY_SECRET_NAME}" ]; then - echo "OPERATOR_KEY_SECRET_NAME cannot be empty" - exit 1 -fi - -export azure_vault_name="${VAULT_NAME}" -export azure_secret_name="${OPERATOR_KEY_SECRET_NAME}" - -# -- locate config file -if [ -z "${DEPLOYMENT_ENVIRONMENT}" ]; then - echo "DEPLOYMENT_ENVIRONMENT cannot be empty" - exit 1 -fi -if [ "${DEPLOYMENT_ENVIRONMENT}" != 'prod' -a "${DEPLOYMENT_ENVIRONMENT}" != 'integ' ]; then - echo "Unrecognized DEPLOYMENT_ENVIRONMENT ${DEPLOYMENT_ENVIRONMENT}" - exit 1 -fi - -TARGET_CONFIG="/app/conf/${DEPLOYMENT_ENVIRONMENT}-uid2-config.json" -if [ ! -f "${TARGET_CONFIG}" ]; then - echo "Unrecognized config ${TARGET_CONFIG}" - exit 1 -fi - -FINAL_CONFIG="/tmp/final-config.json" -echo "-- copying ${TARGET_CONFIG} to ${FINAL_CONFIG}" -cp ${TARGET_CONFIG} ${FINAL_CONFIG} -if [ $? -ne 0 ]; then - echo "Failed to create ${FINAL_CONFIG} with error code $?" - exit 1 -fi - -# -- replace base URLs if both CORE_BASE_URL and OPTOUT_BASE_URL are provided -# -- using hardcoded domains is fine because they should not be changed frequently -if [ -n "${CORE_BASE_URL}" -a -n "${OPTOUT_BASE_URL}" -a "${DEPLOYMENT_ENVIRONMENT}" != 'prod' ]; then - echo "-- replacing URLs by ${CORE_BASE_URL} and ${OPTOUT_BASE_URL}" - sed -i "s#https://core-integ.uidapi.com#${CORE_BASE_URL}#g" ${FINAL_CONFIG} - - sed -i "s#https://optout-integ.uidapi.com#${OPTOUT_BASE_URL}#g" ${FINAL_CONFIG} -fi - -cat $FINAL_CONFIG - -# delay the start of the operator until the side car has started correctly -wait_for_sidecar - -# -- start operator -echo "-- starting java application" -java \ - -XX:MaxRAMPercentage=95 -XX:-UseCompressedOops -XX:+PrintFlagsFinal \ - -Djava.security.egd=file:/dev/./urandom \ - -Dvertx.logger-delegate-factory-class-name=io.vertx.core.logging.SLF4JLogDelegateFactory \ - -Dlogback.configurationFile=/app/conf/logback.xml \ - -Dvertx-config-path=${FINAL_CONFIG} \ - -jar ${JAR_NAME}-${JAR_VERSION}.jar diff --git a/scripts/confidential_compute.py b/scripts/confidential_compute.py index e156758ee..dd8f2d5bc 100644 --- a/scripts/confidential_compute.py +++ b/scripts/confidential_compute.py @@ -15,7 +15,7 @@ class ConfidentialComputeConfig(TypedDict): skip_validations: NotRequired[bool] debug_mode: NotRequired[bool] -class ConfidentialComputeStartupException(Exception): +class ConfidentialComputeStartupError(Exception): def __init__(self, error_name, provider, extra_message=None): urls = { "EC2EntryPoint": "https://unifiedid.com/docs/guides/operator-guide-aws-marketplace#uid2-operator-error-codes", @@ -25,35 +25,31 @@ def __init__(self, error_name, provider, extra_message=None): url = urls.get(provider) super().__init__(f"{error_name}\n" + (extra_message if extra_message else "") + f"\nVisit {url} for more details") -class MissingInstanceProfile(ConfidentialComputeStartupException): +class InstanceProfileMissingError(ConfidentialComputeStartupError): def __init__(self, cls, message = None): super().__init__(error_name=f"E01: {self.__class__.__name__}", provider=cls, extra_message=message) -class OperatorKeyNotFound(ConfidentialComputeStartupException): +class OperatorKeyNotFoundError(ConfidentialComputeStartupError): def __init__(self, cls, message = None): super().__init__(error_name=f"E02: {self.__class__.__name__}", provider=cls, extra_message=message) -class MissingConfig(ConfidentialComputeStartupException): +class ConfigurationMissingError(ConfidentialComputeStartupError): def __init__(self, cls, missing_keys): super().__init__(error_name=f"E03: {self.__class__.__name__}", provider=cls, extra_message=', '.join(missing_keys)) -class InvalidConfigValue(ConfidentialComputeStartupException): +class ConfigurationValueError(ConfidentialComputeStartupError): def __init__(self, cls, config_key = None): super().__init__(error_name=f"E04: {self.__class__.__name__} " , provider=cls, extra_message=config_key) -class InvalidOperatorKey(ConfidentialComputeStartupException): +class OperatorKeyValidationError(ConfidentialComputeStartupError): def __init__(self, cls): super().__init__(error_name=f"E05: {self.__class__.__name__}", provider=cls) -class UID2ServicesUnreachable(ConfidentialComputeStartupException): +class UID2ServicesUnreachableError(ConfidentialComputeStartupError): def __init__(self, cls, ip=None): super().__init__(error_name=f"E06: {self.__class__.__name__}", provider=cls, extra_message=ip) -class AuxiliariesException(ConfidentialComputeStartupException): - def __init__(self, cls, inner_message = None): - super().__init__(error_name=f"E07: {self.__class__.__name__}", provider=cls, extra_message=inner_message) - -class OperatorKeyAccessDenied(ConfidentialComputeStartupException): +class OperatorKeyPermissionError(ConfidentialComputeStartupError): def __init__(self, cls, message = None): super().__init__(error_name=f"E08: {self.__class__.__name__}", provider=cls, extra_message=message) @@ -74,7 +70,7 @@ def validate_operator_key(): debug_mode = self.configs.get("debug_mode", False) expected_env = "I" if debug_mode or env == "integ" else "P" if operator_key.split("-")[2] != expected_env: - raise InvalidOperatorKey(self.__class__.__name__) + raise OperatorKeyValidationError(self.__class__.__name__) logging.info("Validated operator key matches environment") else: logging.info("Skipping operator key validation") @@ -82,10 +78,10 @@ def validate_operator_key(): def validate_url(url_key, environment): """URL should include environment except in prod""" if environment != "prod" and environment not in self.configs[url_key]: - raise InvalidConfigValue(self.__class__.__name__, url_key) + raise ConfigurationValueError(self.__class__.__name__, url_key) parsed_url = urlparse(self.configs[url_key]) if parsed_url.scheme != 'https' and parsed_url.path: - raise InvalidConfigValue(self.__class__.__name__, url_key) + raise ConfigurationValueError(self.__class__.__name__, url_key) logging.info(f"Validated {self.configs[url_key]} matches other config parameters") def validate_connectivity() -> None: @@ -96,22 +92,22 @@ def validate_connectivity() -> None: requests.get(core_url, timeout=5) logging.info(f"Validated connectivity to {core_url}") except (requests.ConnectionError, requests.Timeout) as e: - raise UID2ServicesUnreachable(self.__class__.__name__, core_ip) + raise UID2ServicesUnreachableError(self.__class__.__name__, core_ip) except Exception as e: - raise UID2ServicesUnreachable(self.__class__.__name__) + raise UID2ServicesUnreachableError(self.__class__.__name__) type_hints = get_type_hints(ConfidentialComputeConfig, include_extras=True) required_keys = [field for field, hint in type_hints.items() if "NotRequired" not in str(hint)] missing_keys = [key for key in required_keys if key not in self.configs] if missing_keys: - raise MissingConfig(self.__class__.__name__, missing_keys) + raise ConfigurationMissingError(self.__class__.__name__, missing_keys) environment = self.configs["environment"] if environment not in ["integ", "prod"]: - raise InvalidConfigValue(self.__class__.__name__, "environment") + raise ConfigurationValueError(self.__class__.__name__, "environment") if self.configs.get("debug_mode") and environment == "prod": - raise InvalidConfigValue(self.__class__.__name__, "debug_mode") + raise ConfigurationValueError(self.__class__.__name__, "debug_mode") validate_url("core_base_url", environment) validate_url("optout_base_url", environment) diff --git a/scripts/gcp-oidc/gcp.py b/scripts/gcp-oidc/gcp.py index e39e8c49b..dd135055a 100644 --- a/scripts/gcp-oidc/gcp.py +++ b/scripts/gcp-oidc/gcp.py @@ -8,9 +8,9 @@ from google.auth import default from google.auth.exceptions import DefaultCredentialsError from google.api_core.exceptions import PermissionDenied, NotFound - +from urllib.parse import urlparse sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from confidential_compute import ConfidentialCompute, ConfidentialComputeConfig, MissingConfig, OperatorKeyNotFound, OperatorKeyAccessDenied, ConfidentialComputeStartupException +from confidential_compute import ConfidentialCompute, ConfidentialComputeConfig, ConfigurationMissingError, OperatorKeyNotFoundError, OperatorKeyPermissionError, ConfidentialComputeStartupError class GCPEntryPoint(ConfidentialCompute): @@ -32,16 +32,16 @@ def _set_confidential_config(self, secret_identifier=None) -> None: } if not os.getenv("API_TOKEN_SECRET_NAME"): - raise MissingConfig(self.__class__.__name__, ["API_TOKEN_SECRET_NAME"]) + raise ConfigurationMissingError(self.__class__.__name__, ["API_TOKEN_SECRET_NAME"]) try: client = secretmanager.SecretManagerServiceClient() secret_version_name = f"{os.getenv("API_TOKEN_SECRET_NAME")}" response = client.access_secret_version(name=secret_version_name) secret_value = response.payload.data.decode("UTF-8") except (PermissionDenied, DefaultCredentialsError) as e: - raise OperatorKeyAccessDenied(self.__class__.__name__, str(e)) + raise OperatorKeyPermissionError(self.__class__.__name__, str(e)) except NotFound: - raise OperatorKeyNotFound(self.__class__.__name__, f"Secret Manager {os.getenv("API_TOKEN_SECRET_NAME")}") + raise OperatorKeyNotFoundError(self.__class__.__name__, f"Secret Manager {os.getenv("API_TOKEN_SECRET_NAME")}") self.configs["operator_key"] = secret_value def __populate_operator_config(self, destination): @@ -49,8 +49,8 @@ def __populate_operator_config(self, destination): shutil.copy(target_config, destination) with open(destination, 'r') as file: config = file.read() - config = config.replace("https://core.uidapi.com", self.configs.get("core_base_url")) - config = config.replace("https://optout.uidapi.com", self.configs.get("optout_base_url")) + config = config.replace("core.uidapi.com", urlparse(self.configs.get("core_base_url")).netloc) + config = config.replace("optout.uidapi.com", urlparse(self.configs.get("optout_base_url")).netloc) with open(destination, 'w') as file: file.write(config) @@ -88,7 +88,7 @@ def run_compute(self) -> None: try: gcp = GCPEntryPoint() gcp.run_compute() - except ConfidentialComputeStartupException as e: + except ConfidentialComputeStartupError as e: print("Failed starting up Confidential Compute. Please checks the logs for errors and retry \n", e) except Exception as e: print("Unexpected failure while starting up Confidential Compute. Please contact UID support team with this log \n ", e) From 48b0ac7c4daba87d24d56dbcebcd6d88cd49b99e Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Tue, 4 Feb 2025 23:04:39 +0000 Subject: [PATCH 243/431] [CI Pipeline] Released Patch version: 5.47.3 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2cb422883..265bcd60c 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.47.0 + 5.47.3 UTF-8 From 71224bfc78847a8107b1514d99729a66060bc66d Mon Sep 17 00:00:00 2001 From: Xu Yang <58192524+clarkxuyang@users.noreply.github.com> Date: Wed, 5 Feb 2025 09:39:00 -0800 Subject: [PATCH 244/431] xyang-UID2-4653 remove loki (#1336) --- Dockerfile | 1 - conf/logback.loki-local.xml | 30 ------------------------------ conf/logback.loki.xml | 30 ------------------------------ pom.xml | 5 ----- 4 files changed, 66 deletions(-) delete mode 100644 conf/logback.loki-local.xml delete mode 100644 conf/logback.loki.xml diff --git a/Dockerfile b/Dockerfile index fb448af6e..fa7d23d29 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,6 @@ ENV JAR_NAME=${JAR_NAME} ENV JAR_VERSION=${JAR_VERSION} ENV IMAGE_VERSION=${IMAGE_VERSION} ENV REGION=us-east-2 -ENV LOKI_HOSTNAME=loki ENV LOGBACK_CONF=${LOGBACK_CONF:-./conf/logback.xml} COPY ./target/${JAR_NAME}-${JAR_VERSION}-jar-with-dependencies.jar /app/${JAR_NAME}-${JAR_VERSION}.jar diff --git a/conf/logback.loki-local.xml b/conf/logback.loki-local.xml deleted file mode 100644 index ff0f0adb1..000000000 --- a/conf/logback.loki-local.xml +++ /dev/null @@ -1,30 +0,0 @@ - - - - http://localhost:3100/loki/api/v1/push - - - - - l=%level h=${HOSTNAME} po=${port_offset:-0} c=%logger{20} t=%thread | %msg %ex - - true - - - - - - - %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg %ex%n - - - - - - - - - \ No newline at end of file diff --git a/conf/logback.loki.xml b/conf/logback.loki.xml deleted file mode 100644 index d2358c272..000000000 --- a/conf/logback.loki.xml +++ /dev/null @@ -1,30 +0,0 @@ - - - - http://${LOKI_HOSTNAME}:3100/loki/api/v1/push - - - - - l=%level h=${HOSTNAME} po=${port_offset:-0} c=%logger{20} t=%thread | %msg %ex - - true - - - - - - - %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg %ex%n - - - - - - - - - diff --git a/pom.xml b/pom.xml index 265bcd60c..d5ad49fca 100644 --- a/pom.xml +++ b/pom.xml @@ -162,11 +162,6 @@ logback-classic 1.5.8 - - com.github.loki4j - loki-logback-appender - 1.5.2 - net.logstash.logback logstash-logback-encoder From dbbea23acbde93727eaa6cf704f1e0245fa4cf01 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Wed, 5 Feb 2025 17:46:12 +0000 Subject: [PATCH 245/431] [CI Pipeline] Released Patch version: 5.47.5 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d5ad49fca..f21af73f6 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.47.3 + 5.47.5 UTF-8 From 5170c953d21dbe9fec89a75ceb8294b03238c977 Mon Sep 17 00:00:00 2001 From: Xu Yang <58192524+clarkxuyang@users.noreply.github.com> Date: Wed, 5 Feb 2025 15:03:21 -0800 Subject: [PATCH 246/431] use python logging package instead (#1366) --- scripts/aws/ec2.py | 27 ++++++++++++++------------- scripts/gcp-oidc/gcp.py | 7 ++++--- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/scripts/aws/ec2.py b/scripts/aws/ec2.py index ab1775ff4..fc9243fc2 100644 --- a/scripts/aws/ec2.py +++ b/scripts/aws/ec2.py @@ -9,6 +9,7 @@ import requests import signal import argparse +import logging from botocore.exceptions import ClientError, NoCredentialsError from typing import Dict import sys @@ -99,7 +100,7 @@ def add_defaults(configs: Dict[str, any]) -> AWSConfidentialComputeConfig: return configs region = self.__get_current_region() - print(f"Running in {region}") + logging.info(f"Running in {region}") client = boto3.client("secretsmanager", region_name=region) try: self.configs = add_defaults(json.loads(client.get_secret_value(SecretId=secret_identifier)["SecretString"])) @@ -168,19 +169,19 @@ def _setup_auxiliaries(self) -> None: self.__setup_vsockproxy(log_level) self.__run_config_server() self.__run_socks_proxy() - print("Finished setting up all auxiliaries") + logging.info("Finished setting up all auxiliaries") def _validate_auxiliaries(self) -> None: """Validates connection to flask server direct and through socks proxy.""" - print("Validating auxiliaries") + logging.info("Validating auxiliaries") try: for attempt in range(10): try: response = requests.get(AuxiliaryConfig.get_config_url()) - print("Config server is reachable") + logging.info("Config server is reachable") break except requests.exceptions.ConnectionError as e: - print(f"Connecting to config server, attempt {attempt + 1} failed with ConnectionError: {e}") + logging.error(f"Connecting to config server, attempt {attempt + 1} failed with ConnectionError: {e}") time.sleep(1) else: raise RuntimeError(f"Config server unreachable") @@ -193,7 +194,7 @@ def _validate_auxiliaries(self) -> None: response.raise_for_status() except requests.RequestException as e: raise RuntimeError(f"Cannot connect to config server via SOCKS proxy: {e}") - print("Connectivity check to config server passes") + logging.info("Connectivity check to config server passes") def __run_nitro_enclave(self): command = [ @@ -205,7 +206,7 @@ def __run_nitro_enclave(self): "--enclave-name", "uid2operator" ] if self.configs.get('debug_mode', False): - print("Running in debug_mode") + logging.info("Running in debug_mode") command += ["--debug-mode", "--attach-console"] self.run_command(command, separate_process=False) @@ -213,7 +214,7 @@ def run_compute(self) -> None: """Main execution flow for confidential compute.""" secret_manager_key = self.__get_secret_name_from_userdata() self._set_confidential_config(secret_manager_key) - print(f"Fetched configs from {secret_manager_key}") + logging.info(f"Fetched configs from {secret_manager_key}") if not self.configs.get("skip_validations"): self.validate_configuration() self._setup_auxiliaries() @@ -236,11 +237,11 @@ def __kill_auxiliaries(self) -> None: if result.stdout.strip(): for pid in result.stdout.strip().split("\n"): os.kill(int(pid), signal.SIGKILL) - print(f"Killed process '{process_name}'.") + logging.info(f"Killed process '{process_name}'.") else: - print(f"No process named '{process_name}' found.") + logging.info(f"No process named '{process_name}' found.") except Exception as e: - print(f"Error killing process '{process_name}': {e}") + logging.error(f"Error killing process '{process_name}': {e}") if __name__ == "__main__": @@ -254,7 +255,7 @@ def __kill_auxiliaries(self) -> None: else: ec2.run_compute() except ConfidentialComputeStartupError as e: - print("Failed starting up Confidential Compute. Please checks the logs for errors and retry \n", e) + logging.error(f"Failed starting up Confidential Compute. Please checks the logs for errors and retry {e}") except Exception as e: - print("Unexpected failure while starting up Confidential Compute. Please contact UID support team with this log \n ", e) + logging.error(f"Unexpected failure while starting up Confidential Compute. Please contact UID support team with this log {e}") diff --git a/scripts/gcp-oidc/gcp.py b/scripts/gcp-oidc/gcp.py index dd135055a..c2704c7ed 100644 --- a/scripts/gcp-oidc/gcp.py +++ b/scripts/gcp-oidc/gcp.py @@ -4,6 +4,7 @@ import shutil from typing import Dict import sys +import logging from google.cloud import secretmanager from google.auth import default from google.auth.exceptions import DefaultCredentialsError @@ -64,7 +65,7 @@ def _validate_auxiliaries(self) -> None: def run_compute(self) -> None: self._set_confidential_config() - print(f"Fetched configs") + logging.info("Fetched configs") if not self.configs.get("skip_validations"): self.validate_configuration() config_locaton = "/tmp/final-config.json" @@ -89,7 +90,7 @@ def run_compute(self) -> None: gcp = GCPEntryPoint() gcp.run_compute() except ConfidentialComputeStartupError as e: - print("Failed starting up Confidential Compute. Please checks the logs for errors and retry \n", e) + logging.error(f"Failed starting up Confidential Compute. Please checks the logs for errors and retry {e}") except Exception as e: - print("Unexpected failure while starting up Confidential Compute. Please contact UID support team with this log \n ", e) + logging.error(f"Unexpected failure while starting up Confidential Compute. Please contact UID support team with this log {e}") From e26691525bb63b818bf7e58a55a96016d49949b2 Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Thu, 6 Feb 2025 14:34:11 +1100 Subject: [PATCH 247/431] Update Main "run" method to fail operator start up if ConfigService initialisation fails --- src/main/java/com/uid2/operator/Main.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index 02ab4b952..b1ac75ff9 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -350,7 +350,7 @@ private void run() throws Exception { try { fs.add(createStoreVerticles()); } catch (Exception e) { - throw new RuntimeException(e); + return Future.failedFuture(e); } CompositeFuture.all(fs).onComplete(ar -> { @@ -365,12 +365,12 @@ private void run() throws Exception { Promise promise = Promise.promise(); vertx.deployVerticle(operatorVerticleSupplier, options, promise); return promise.future(); - }) - .onFailure(t -> { - LOGGER.error("Failed to bootstrap operator: " + t.getMessage(), new Exception(t)); - vertx.close(); - System.exit(1); }); + }) + .onFailure(t -> { + LOGGER.error("Failed to bootstrap operator: " + t.getMessage(), new Exception(t)); + vertx.close(); + System.exit(1); }); } From bacb2eea2758cdf436c8855288322f7e4eec4bdf Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Thu, 6 Feb 2025 14:49:55 +1100 Subject: [PATCH 248/431] Change 'createStoreVerticles' try catch block to throw runtime exception --- src/main/java/com/uid2/operator/Main.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index b1ac75ff9..bf82f21e2 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -350,7 +350,7 @@ private void run() throws Exception { try { fs.add(createStoreVerticles()); } catch (Exception e) { - return Future.failedFuture(e); + throw new RuntimeException(e); } CompositeFuture.all(fs).onComplete(ar -> { From 31534a48f1b8ff3814c8c38ccf2eef2784773519 Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Thu, 6 Feb 2025 15:26:24 +1100 Subject: [PATCH 249/431] Change feature flag retriever store to be optional - change default to feat flag off --- src/main/java/com/uid2/operator/Main.java | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index 7247d17a4..df5067ece 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -38,8 +38,10 @@ import io.micrometer.prometheus.PrometheusMeterRegistry; import io.micrometer.prometheus.PrometheusRenameFilter; import io.vertx.config.ConfigRetriever; +import io.vertx.config.ConfigRetrieverOptions; import io.vertx.core.*; import io.vertx.core.http.HttpServerOptions; +import io.vertx.core.json.JsonArray; import io.vertx.core.json.JsonObject; import io.vertx.micrometer.*; import io.vertx.micrometer.backends.BackendRegistries; @@ -288,15 +290,18 @@ private Future initialiseConfigService() throws Exception { Future staticConfigFuture = ConfigService.create(staticConfigRetriever); - ConfigRetriever featureFlagConfigRetriever = ConfigRetrieverFactory.create( + ConfigRetriever featureFlagConfigRetriever = ConfigRetriever.create( vertx, - new JsonObject() - .put("type", "file") - .put("config", new JsonObject() - .put("path", "conf/feat-flag/feat-flag.json") - .put("format", "json")) - .put(ConfigScanPeriodMsProp, 10000), - "" + new ConfigRetrieverOptions( + new JsonObject() + .put("stores", new JsonArray() + .add(new JsonObject() + .put("type", "file") + .put("optional", true) + .put("config", new JsonObject() + .put("path", "conf/feat-flag/feat-flag.json") + .put("format", "json")))) + .put("scanPeriod", 10000)) ); @@ -377,7 +382,7 @@ private void run() throws Exception { private void setupFeatureFlagListener(ConfigServiceManager manager, ConfigRetriever retriever) { retriever.listen(change -> { JsonObject newConfig = change.getNewConfiguration(); - boolean useDynamicConfig = newConfig.getJsonObject("remote_config").getBoolean("enabled", true); + boolean useDynamicConfig = newConfig.getJsonObject("remote_config", new JsonObject()).getBoolean("enabled", false); manager.updateConfigService(useDynamicConfig).onComplete(update -> { if (update.succeeded()) { LOGGER.info("Remote config feature flag toggled successfully"); From b9d4a0ef173160b537d3e1babb71dfb5be78569e Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Thu, 6 Feb 2025 16:26:56 +1100 Subject: [PATCH 250/431] Address comments - remove unused const - add comment for json structure --- src/main/java/com/uid2/operator/Const.java | 1 - src/main/java/com/uid2/operator/Main.java | 10 +++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/uid2/operator/Const.java b/src/main/java/com/uid2/operator/Const.java index 4eb52198d..c7d7f3dda 100644 --- a/src/main/java/com/uid2/operator/Const.java +++ b/src/main/java/com/uid2/operator/Const.java @@ -32,6 +32,5 @@ public class Config extends com.uid2.shared.Const.Config { public static final String ConfigScanPeriodMsProp = "config_scan_period_ms"; public static final String IdentityV3Prop = "identity_v3"; - public static final String RemoteConfigFeatureFlagProp = "remote_config_feature_flag"; } } diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index df5067ece..b4a4d2245 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -290,6 +290,9 @@ private Future initialiseConfigService() throws Exception { Future staticConfigFuture = ConfigService.create(staticConfigRetriever); + // Create ConfigRetriever with options specified in JsonObject. + // Includes an optional file store and scan period of 10000 ms (10 sec). + // See https://vertx.io/docs/vertx-config/java/#_file for Vertx file store docs. ConfigRetriever featureFlagConfigRetriever = ConfigRetriever.create( vertx, new ConfigRetrieverOptions( @@ -299,13 +302,10 @@ private Future initialiseConfigService() throws Exception { .put("type", "file") .put("optional", true) .put("config", new JsonObject() - .put("path", "conf/feat-flag/feat-flag.json") - .put("format", "json")))) + .put("path", "conf/feat-flag/feat-flag.json")))) .put("scanPeriod", 10000)) ); - - Future.all(dynamicConfigFuture, staticConfigFuture, featureFlagConfigRetriever.getConfig()) .onComplete(ar -> { if (ar.succeeded()) { @@ -316,7 +316,7 @@ private Future initialiseConfigService() throws Exception { boolean remoteConfigFeatureFlag = featureFlagConfig .getJsonObject("remote_config") - .getBoolean(Const.Config.RemoteConfigFeatureFlagProp, false); + .getBoolean("enabled", false); ConfigServiceManager configServiceManager = new ConfigServiceManager( vertx, dynamicConfigService, staticConfigService, remoteConfigFeatureFlag); From c845db632a570cbf12a7232e1b7aae8717a7b2b9 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Thu, 6 Feb 2025 05:51:37 +0000 Subject: [PATCH 251/431] [CI Pipeline] Released Patch version: 5.47.10 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f21af73f6..ee2e6993d 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.47.5 + 5.47.10 UTF-8 From 39ffb142defafe8d1bb68fc7f33ca5f79b096aa1 Mon Sep 17 00:00:00 2001 From: Xu Yang <58192524+clarkxuyang@users.noreply.github.com> Date: Fri, 7 Feb 2025 09:42:51 -0800 Subject: [PATCH 252/431] xuy-UID2-4653-remove-loki (#1370) * xuy-UID2-4653-remove-loki --- Dockerfile | 3 +-- scripts/azure-cc/Dockerfile | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index fa7d23d29..38acf8ebd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,6 @@ ENV JAR_NAME=${JAR_NAME} ENV JAR_VERSION=${JAR_VERSION} ENV IMAGE_VERSION=${IMAGE_VERSION} ENV REGION=us-east-2 -ENV LOGBACK_CONF=${LOGBACK_CONF:-./conf/logback.xml} COPY ./target/${JAR_NAME}-${JAR_VERSION}-jar-with-dependencies.jar /app/${JAR_NAME}-${JAR_VERSION}.jar COPY ./target/${JAR_NAME}-${JAR_VERSION}-sources.jar /app @@ -31,5 +30,5 @@ CMD java \ -XX:MaxRAMPercentage=95 -XX:-UseCompressedOops -XX:+PrintFlagsFinal -XX:-OmitStackTraceInFastThrow \ -Djava.security.egd=file:/dev/./urandom \ -Dvertx.logger-delegate-factory-class-name=io.vertx.core.logging.SLF4JLogDelegateFactory \ - -Dlogback.configurationFile=${LOGBACK_CONF} \ + -Dlogback.configurationFile=/app/conf/logback.xml \ -jar ${JAR_NAME}-${JAR_VERSION}.jar diff --git a/scripts/azure-cc/Dockerfile b/scripts/azure-cc/Dockerfile index a14b451a9..a3845ac94 100644 --- a/scripts/azure-cc/Dockerfile +++ b/scripts/azure-cc/Dockerfile @@ -26,7 +26,6 @@ ENV JAR_NAME=${JAR_NAME} ENV JAR_VERSION=${JAR_VERSION} ENV IMAGE_VERSION=${IMAGE_VERSION} ENV REGION=default -ENV LOKI_HOSTNAME=loki # Copy application files COPY ./target/${JAR_NAME}-${JAR_VERSION}-jar-with-dependencies.jar /app/${JAR_NAME}-${JAR_VERSION}.jar From 1892af5a1fcaf37f5fc34546c95b567979c7df78 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Fri, 7 Feb 2025 18:07:06 +0000 Subject: [PATCH 253/431] [CI Pipeline] Released Patch version: 5.47.13 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ee2e6993d..51cbe627e 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.47.10 + 5.47.13 UTF-8 From 1b3522c5d80cbd6d3ee56bbc14887847c24ee178 Mon Sep 17 00:00:00 2001 From: Xu Yang <58192524+clarkxuyang@users.noreply.github.com> Date: Mon, 10 Feb 2025 09:09:03 -0800 Subject: [PATCH 254/431] fix logback conf env (#1369) * fix logback conf env From f857c9dac3642af2363af13b8827ea20b1dbc5c9 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Mon, 10 Feb 2025 17:33:16 +0000 Subject: [PATCH 255/431] [CI Pipeline] Released Patch version: 5.47.16 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 51cbe627e..46510a059 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.47.13 + 5.47.16 UTF-8 From 255d98dfff68ec6c1a46d2e1c9c82fd2406dc9ea Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Tue, 11 Feb 2025 09:30:17 +1100 Subject: [PATCH 256/431] Change remote_config_store with http type to use single "url" field - Update ConfigRetrieverFactory to parse url into URI for http store --- conf/integ-config.json | 4 +--- conf/local-e2e-docker-private-config.json | 4 +--- conf/local-e2e-docker-public-config.json | 4 +--- conf/local-e2e-private-config.json | 4 +--- conf/local-e2e-public-config.json | 4 +--- conf/validator-latest-e2e-docker-public-config.json | 4 +--- scripts/aws/conf/euid-integ-config.json | 5 +---- scripts/aws/conf/euid-prod-config.json | 5 +---- scripts/aws/conf/uid2-integ-config.json | 5 +---- scripts/aws/conf/uid2-prod-config.json | 5 +---- scripts/azure-cc/conf/integ-uid2-config.json | 5 +---- scripts/azure-cc/conf/prod-uid2-config.json | 5 +---- scripts/gcp-oidc/conf/integ-config.json | 5 +---- scripts/gcp-oidc/conf/prod-config.json | 5 +---- .../com/uid2/operator/service/ConfigRetrieverFactory.java | 8 ++++++++ 15 files changed, 22 insertions(+), 50 deletions(-) diff --git a/conf/integ-config.json b/conf/integ-config.json index 8f7fa2385..87940a527 100644 --- a/conf/integ-config.json +++ b/conf/integ-config.json @@ -18,9 +18,7 @@ "runtime_config_store": { "type": "http", "config" : { - "host": "localhost", - "port": 8088, - "path": "/operator/config" + "url": "http://localhost:8088/operator/config" }, "config_scan_period_ms": 300000 } diff --git a/conf/local-e2e-docker-private-config.json b/conf/local-e2e-docker-private-config.json index 218f64e2c..9751e1f39 100644 --- a/conf/local-e2e-docker-private-config.json +++ b/conf/local-e2e-docker-private-config.json @@ -31,9 +31,7 @@ "runtime_config_store": { "type": "http", "config" : { - "host": "core", - "port": 8088, - "path": "/operator/config" + "url": "http://core:8088/operator/config" }, "config_scan_period_ms": 300000 } diff --git a/conf/local-e2e-docker-public-config.json b/conf/local-e2e-docker-public-config.json index 3357a2b85..1c3ea4f84 100644 --- a/conf/local-e2e-docker-public-config.json +++ b/conf/local-e2e-docker-public-config.json @@ -37,9 +37,7 @@ "runtime_config_store": { "type": "http", "config" : { - "host": "core", - "port": 8088, - "path": "/operator/config" + "url": "http://core:8088/operator/config" }, "config_scan_period_ms": 300000 } diff --git a/conf/local-e2e-private-config.json b/conf/local-e2e-private-config.json index 00d94d552..8c59036b7 100644 --- a/conf/local-e2e-private-config.json +++ b/conf/local-e2e-private-config.json @@ -42,9 +42,7 @@ "runtime_config_store": { "type": "http", "config" : { - "host": "localhost", - "port": 8088, - "path": "/operator/config" + "url": "http://localhost:8088/operator/config" }, "config_scan_period_ms": 300000 } diff --git a/conf/local-e2e-public-config.json b/conf/local-e2e-public-config.json index edd7af5cb..1d4b2a12e 100644 --- a/conf/local-e2e-public-config.json +++ b/conf/local-e2e-public-config.json @@ -43,9 +43,7 @@ "runtime_config_store": { "type": "http", "config" : { - "host": "localhost", - "port": 8088, - "path": "/operator/config" + "url": "http://localhost:8088/operator/config" }, "config_scan_period_ms": 300000 } diff --git a/conf/validator-latest-e2e-docker-public-config.json b/conf/validator-latest-e2e-docker-public-config.json index f9b201ee8..331862b7e 100644 --- a/conf/validator-latest-e2e-docker-public-config.json +++ b/conf/validator-latest-e2e-docker-public-config.json @@ -36,9 +36,7 @@ "runtime_config_store": { "type": "http", "config" : { - "host": "core", - "port": 8088, - "path": "/operator/config" + "url": "http://core:8080/operator/config" }, "config_scan_period_ms": 300000 } diff --git a/scripts/aws/conf/euid-integ-config.json b/scripts/aws/conf/euid-integ-config.json index 66a82183e..ca7c331e3 100644 --- a/scripts/aws/conf/euid-integ-config.json +++ b/scripts/aws/conf/euid-integ-config.json @@ -15,10 +15,7 @@ "runtime_config_store": { "type": "http", "config" : { - "host": "core.integ.euid.eu", - "port": 443, - "ssl": true, - "path": "/operator/config" + "url": "https://core.integ.euid.eu/operator/config" }, "config_scan_period_ms": 300000 } diff --git a/scripts/aws/conf/euid-prod-config.json b/scripts/aws/conf/euid-prod-config.json index ac00370bc..9d19bc91b 100644 --- a/scripts/aws/conf/euid-prod-config.json +++ b/scripts/aws/conf/euid-prod-config.json @@ -31,10 +31,7 @@ "runtime_config_store": { "type": "http", "config" : { - "host": "core.prod.euid.eu", - "port": 443, - "ssl": true, - "path": "/operator/config" + "url": "https://core.prod.euid.eu/operator/config" }, "config_scan_period_ms": 300000 } diff --git a/scripts/aws/conf/uid2-integ-config.json b/scripts/aws/conf/uid2-integ-config.json index 4daf69cf7..8f0252082 100644 --- a/scripts/aws/conf/uid2-integ-config.json +++ b/scripts/aws/conf/uid2-integ-config.json @@ -15,10 +15,7 @@ "runtime_config_store": { "type": "http", "config" : { - "host": "core-integ.uidapi.com", - "port": 443, - "ssl": true, - "path": "/operator/config" + "url": "https://core-integ.uidapi.com/operator/config" }, "config_scan_period_ms": 300000 } diff --git a/scripts/aws/conf/uid2-prod-config.json b/scripts/aws/conf/uid2-prod-config.json index a46670746..ccbce2e32 100644 --- a/scripts/aws/conf/uid2-prod-config.json +++ b/scripts/aws/conf/uid2-prod-config.json @@ -26,10 +26,7 @@ "runtime_config_store": { "type": "http", "config" : { - "host": "core-prod.uidapi.com", - "port": 443, - "ssl": true, - "path": "/operator/config" + "url": "https://core-prod.uidapi.com/operator/config" }, "config_scan_period_ms": 300000 } diff --git a/scripts/azure-cc/conf/integ-uid2-config.json b/scripts/azure-cc/conf/integ-uid2-config.json index d1b697ad7..e3d34d846 100644 --- a/scripts/azure-cc/conf/integ-uid2-config.json +++ b/scripts/azure-cc/conf/integ-uid2-config.json @@ -14,10 +14,7 @@ "runtime_config_store": { "type": "http", "config" : { - "host": "core-integ.uidapi.com", - "port": 443, - "ssl": true, - "path": "/operator/config" + "url": "https://core-integ.uidapi.com/operator/config" }, "config_scan_period_ms": 300000 } diff --git a/scripts/azure-cc/conf/prod-uid2-config.json b/scripts/azure-cc/conf/prod-uid2-config.json index 1305fffcf..bccbc2444 100644 --- a/scripts/azure-cc/conf/prod-uid2-config.json +++ b/scripts/azure-cc/conf/prod-uid2-config.json @@ -15,10 +15,7 @@ "runtime_config_store": { "type": "http", "config" : { - "host": "core-prod.uidapi.com", - "port": 443, - "ssl": true, - "path": "/operator/config" + "url": "https://core-prod.uidapi.com/operator/config" }, "config_scan_period_ms": 300000 } diff --git a/scripts/gcp-oidc/conf/integ-config.json b/scripts/gcp-oidc/conf/integ-config.json index 6876be36d..e07aeea24 100644 --- a/scripts/gcp-oidc/conf/integ-config.json +++ b/scripts/gcp-oidc/conf/integ-config.json @@ -14,10 +14,7 @@ "runtime_config_store": { "type": "http", "config" : { - "host": "core.uidapi.com", - "port": 443, - "ssl": true, - "path": "/operator/config" + "url": "https://core.uidapi.com/operator/config" }, "config_scan_period_ms": 300000 } diff --git a/scripts/gcp-oidc/conf/prod-config.json b/scripts/gcp-oidc/conf/prod-config.json index 82c3a0d81..9c4eba18a 100644 --- a/scripts/gcp-oidc/conf/prod-config.json +++ b/scripts/gcp-oidc/conf/prod-config.json @@ -15,10 +15,7 @@ "runtime_config_store": { "type": "http", "config" : { - "host": "core.uidapi.com", - "port": 443, - "ssl": true, - "path": "/operator/config" + "url": "https://core.uidapi.com/operator/config" }, "config_scan_period_ms": 300000 } diff --git a/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java b/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java index 0df88955b..215485fd2 100644 --- a/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java +++ b/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java @@ -6,6 +6,8 @@ import io.vertx.core.Vertx; import io.vertx.core.json.JsonObject; +import java.net.URI; + import static com.uid2.operator.Const.Config.ConfigScanPeriodMsProp; public class ConfigRetrieverFactory { @@ -13,6 +15,12 @@ public static ConfigRetriever create(Vertx vertx, JsonObject bootstrapConfig, St String type = bootstrapConfig.getString("type"); JsonObject storeConfig = bootstrapConfig.getJsonObject("config"); if (type.equals("http")) { + URI uri = URI.create(storeConfig.getString("url")); + storeConfig.remove("url"); + storeConfig.put("host", uri.getHost()); + storeConfig.put("port", uri.getPort() == -1 ? (uri.getScheme().equals("https") ? 443 : 80) : uri.getPort()); + storeConfig.put("path", uri.getPath()); + storeConfig.put("ssl", uri.getScheme().equals("https")); storeConfig.put("headers", new JsonObject() .put("Authorization", "Bearer " + operatorKey)); } From 953c6b7e054bf30e5ffc0c79473996c3a231100e Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Tue, 11 Feb 2025 09:40:18 +1100 Subject: [PATCH 257/431] Update port logic to be clearer --- .../com/uid2/operator/service/ConfigRetrieverFactory.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java b/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java index 215485fd2..d04d341b2 100644 --- a/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java +++ b/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java @@ -18,7 +18,11 @@ public static ConfigRetriever create(Vertx vertx, JsonObject bootstrapConfig, St URI uri = URI.create(storeConfig.getString("url")); storeConfig.remove("url"); storeConfig.put("host", uri.getHost()); - storeConfig.put("port", uri.getPort() == -1 ? (uri.getScheme().equals("https") ? 443 : 80) : uri.getPort()); + int port = uri.getPort(); + if (port == -1) { + port = uri.getScheme().equals("https") ? 443 : 80; + } + storeConfig.put("port", port); storeConfig.put("path", uri.getPath()); storeConfig.put("ssl", uri.getScheme().equals("https")); storeConfig.put("headers", new JsonObject() From 3812efce1962c9a13f9be917abd6cc1cbf3c4ea5 Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Tue, 11 Feb 2025 11:01:10 +1100 Subject: [PATCH 258/431] Update ConfigServiceTest to use new runtime_config_store format --- src/test/java/com/uid2/operator/ConfigServiceTest.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/test/java/com/uid2/operator/ConfigServiceTest.java b/src/test/java/com/uid2/operator/ConfigServiceTest.java index b1b448b02..ccabe7eb3 100644 --- a/src/test/java/com/uid2/operator/ConfigServiceTest.java +++ b/src/test/java/com/uid2/operator/ConfigServiceTest.java @@ -34,9 +34,7 @@ void setUp() { bootstrapConfig = new JsonObject() .put("type", "http") .put("config", new JsonObject() - .put("host", "localhost") - .put("port", 8088) - .put("path", "/operator/config")) + .put("url", "http://localhost:8088/operator/config")) .put(ConfigScanPeriodMsProp, 300000); runtimeConfig = new JsonObject() @@ -71,7 +69,7 @@ private Future startMockServer(JsonObject config) { server = vertx.createHttpServer() .requestHandler(router) - .listen(Const.Port.ServicePortForCore,"127.0.0.1", http -> { + .listen(Const.Port.ServicePortForCore,"localhost", http -> { if (http.succeeded()) { promise.complete(); } else { From 731d657c612029498c28cfad7247395d9e1c91fe Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Tue, 11 Feb 2025 16:48:31 +1100 Subject: [PATCH 259/431] Suppress vulnerability CVE-2025-24970 --- .trivyignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.trivyignore b/.trivyignore index be4a48a9d..1d9d2b1dc 100644 --- a/.trivyignore +++ b/.trivyignore @@ -5,3 +5,6 @@ # https://thetradedesk.atlassian.net/browse/UID2-4460 CVE-2024-47535 + +# https://thetradedesk.atlassian.net/browse/UID2-4874 +CVE-2025-24970 \ No newline at end of file From b25f266c6cdd71dac223e392aa0185189eaa9cdb Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Tue, 11 Feb 2025 06:21:32 +0000 Subject: [PATCH 260/431] [CI Pipeline] Released Snapshot version: 5.47.11-alpha-158-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ee2e6993d..6f334a3f6 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.47.10 + 5.47.11-alpha-158-SNAPSHOT UTF-8 From 9946d98d21cb1a7b2a862bb331c91e16c5aaba4a Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Tue, 11 Feb 2025 06:32:58 +0000 Subject: [PATCH 261/431] [CI Pipeline] Released Snapshot version: 5.47.12-alpha-120-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 6f334a3f6..52493acbb 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.47.11-alpha-158-SNAPSHOT + 5.47.12-alpha-120-SNAPSHOT UTF-8 From cc6b10c9a1b9ce89d33acad7670db4981fb4456d Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Tue, 11 Feb 2025 22:56:04 +0000 Subject: [PATCH 262/431] [CI Pipeline] Released Patch version: 5.47.20 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 4bc8ec3b9..c190deb9b 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.47.16 + 5.47.20 UTF-8 From d5eb498cf153a627a6dfa79a5fe7fb2c4f4ed902 Mon Sep 17 00:00:00 2001 From: Katherine Chen Date: Wed, 12 Feb 2025 17:43:12 +1100 Subject: [PATCH 263/431] Fix CVE-2024-12797 (#1385) --- Dockerfile | 4 ++-- scripts/azure-cc/Dockerfile | 4 ++-- scripts/gcp-oidc/Dockerfile | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 38acf8ebd..1ad81a4c7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ -# sha from https://hub.docker.com/layers/amd64/eclipse-temurin/21.0.4_7-jre-alpine/images/sha256-8179ddc8a6c5ac9af935020628763b9a5a671e0914976715d2b61b21881cefca -FROM eclipse-temurin@sha256:8179ddc8a6c5ac9af935020628763b9a5a671e0914976715d2b61b21881cefca +# sha from https://hub.docker.com/layers/amd64/eclipse-temurin/21.0.6_7-jre-ubi9-minimal/images/sha256-8ec9c77b1d5979fec8da633da7a66fa5cd26b8b02a96458cecf2809f54c7391e +FROM eclipse-temurin@sha256:5f204213149c5f988b7d8b5532973e13437e23240ed996612d92e25d0e5b782f WORKDIR /app EXPOSE 8080 diff --git a/scripts/azure-cc/Dockerfile b/scripts/azure-cc/Dockerfile index a3845ac94..4d728c912 100644 --- a/scripts/azure-cc/Dockerfile +++ b/scripts/azure-cc/Dockerfile @@ -1,5 +1,5 @@ -# Use Alpine-based JRE image -FROM eclipse-temurin@sha256:8179ddc8a6c5ac9af935020628763b9a5a671e0914976715d2b61b21881cefca +# sha from https://hub.docker.com/layers/amd64/eclipse-temurin/21.0.6_7-jre-ubi9-minimal/images/sha256-8ec9c77b1d5979fec8da633da7a66fa5cd26b8b02a96458cecf2809f54c7391e +FROM eclipse-temurin@sha256:5f204213149c5f988b7d8b5532973e13437e23240ed996612d92e25d0e5b782f # Install necessary packages and set up virtual environment RUN apk update && apk add --no-cache jq python3 py3-pip && \ diff --git a/scripts/gcp-oidc/Dockerfile b/scripts/gcp-oidc/Dockerfile index b8abd015f..b70813769 100644 --- a/scripts/gcp-oidc/Dockerfile +++ b/scripts/gcp-oidc/Dockerfile @@ -1,5 +1,5 @@ -# sha from https://hub.docker.com/layers/amd64/eclipse-temurin/21.0.4_7-jre-alpine/images/sha256-8179ddc8a6c5ac9af935020628763b9a5a671e0914976715d2b61b21881cefca -FROM eclipse-temurin@sha256:8179ddc8a6c5ac9af935020628763b9a5a671e0914976715d2b61b21881cefca +# sha from https://hub.docker.com/layers/amd64/eclipse-temurin/21.0.6_7-jre-ubi9-minimal/images/sha256-8ec9c77b1d5979fec8da633da7a66fa5cd26b8b02a96458cecf2809f54c7391e +FROM eclipse-temurin@sha256:5f204213149c5f988b7d8b5532973e13437e23240ed996612d92e25d0e5b782f LABEL "tee.launch_policy.allow_env_override"="API_TOKEN_SECRET_NAME,DEPLOYMENT_ENVIRONMENT,CORE_BASE_URL,OPTOUT_BASE_URL,DEBUG_MODE,SKIP_VALIDATIONS" LABEL "tee.launch_policy.log_redirect"="always" From f03b8f83a7a91b848a01d191b738248c9152ddf4 Mon Sep 17 00:00:00 2001 From: Katherine Chen Date: Wed, 12 Feb 2025 19:02:57 +1100 Subject: [PATCH 264/431] Revert "Fix CVE-2024-12797 (#1385)" (#1387) This reverts commit d5eb498cf153a627a6dfa79a5fe7fb2c4f4ed902. --- Dockerfile | 4 ++-- scripts/azure-cc/Dockerfile | 4 ++-- scripts/gcp-oidc/Dockerfile | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1ad81a4c7..38acf8ebd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ -# sha from https://hub.docker.com/layers/amd64/eclipse-temurin/21.0.6_7-jre-ubi9-minimal/images/sha256-8ec9c77b1d5979fec8da633da7a66fa5cd26b8b02a96458cecf2809f54c7391e -FROM eclipse-temurin@sha256:5f204213149c5f988b7d8b5532973e13437e23240ed996612d92e25d0e5b782f +# sha from https://hub.docker.com/layers/amd64/eclipse-temurin/21.0.4_7-jre-alpine/images/sha256-8179ddc8a6c5ac9af935020628763b9a5a671e0914976715d2b61b21881cefca +FROM eclipse-temurin@sha256:8179ddc8a6c5ac9af935020628763b9a5a671e0914976715d2b61b21881cefca WORKDIR /app EXPOSE 8080 diff --git a/scripts/azure-cc/Dockerfile b/scripts/azure-cc/Dockerfile index 4d728c912..a3845ac94 100644 --- a/scripts/azure-cc/Dockerfile +++ b/scripts/azure-cc/Dockerfile @@ -1,5 +1,5 @@ -# sha from https://hub.docker.com/layers/amd64/eclipse-temurin/21.0.6_7-jre-ubi9-minimal/images/sha256-8ec9c77b1d5979fec8da633da7a66fa5cd26b8b02a96458cecf2809f54c7391e -FROM eclipse-temurin@sha256:5f204213149c5f988b7d8b5532973e13437e23240ed996612d92e25d0e5b782f +# Use Alpine-based JRE image +FROM eclipse-temurin@sha256:8179ddc8a6c5ac9af935020628763b9a5a671e0914976715d2b61b21881cefca # Install necessary packages and set up virtual environment RUN apk update && apk add --no-cache jq python3 py3-pip && \ diff --git a/scripts/gcp-oidc/Dockerfile b/scripts/gcp-oidc/Dockerfile index b70813769..b8abd015f 100644 --- a/scripts/gcp-oidc/Dockerfile +++ b/scripts/gcp-oidc/Dockerfile @@ -1,5 +1,5 @@ -# sha from https://hub.docker.com/layers/amd64/eclipse-temurin/21.0.6_7-jre-ubi9-minimal/images/sha256-8ec9c77b1d5979fec8da633da7a66fa5cd26b8b02a96458cecf2809f54c7391e -FROM eclipse-temurin@sha256:5f204213149c5f988b7d8b5532973e13437e23240ed996612d92e25d0e5b782f +# sha from https://hub.docker.com/layers/amd64/eclipse-temurin/21.0.4_7-jre-alpine/images/sha256-8179ddc8a6c5ac9af935020628763b9a5a671e0914976715d2b61b21881cefca +FROM eclipse-temurin@sha256:8179ddc8a6c5ac9af935020628763b9a5a671e0914976715d2b61b21881cefca LABEL "tee.launch_policy.allow_env_override"="API_TOKEN_SECRET_NAME,DEPLOYMENT_ENVIRONMENT,CORE_BASE_URL,OPTOUT_BASE_URL,DEBUG_MODE,SKIP_VALIDATIONS" LABEL "tee.launch_policy.log_redirect"="always" From 5f0e564db35601ac1b6df3eff27e4c75808b485b Mon Sep 17 00:00:00 2001 From: Katherine Chen Date: Wed, 12 Feb 2025 19:24:52 +1100 Subject: [PATCH 265/431] Update to 21.0.6_7-jre-alpine-3.21 (#1388) * Update to 21.0.6_7-jre-alpine-3.21 * Add CVE-2024-12797 to `.trivyignore` * Revert Docker base image change --- .trivyignore | 5 ++++- scripts/azure-cc/Dockerfile | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.trivyignore b/.trivyignore index 1d9d2b1dc..49d44bf36 100644 --- a/.trivyignore +++ b/.trivyignore @@ -7,4 +7,7 @@ CVE-2024-47535 # https://thetradedesk.atlassian.net/browse/UID2-4874 -CVE-2025-24970 \ No newline at end of file +CVE-2025-24970 + +# https://thetradedesk.atlassian.net/browse/UID2-4891 +CVE-2024-12797 \ No newline at end of file diff --git a/scripts/azure-cc/Dockerfile b/scripts/azure-cc/Dockerfile index a3845ac94..5dfe55a51 100644 --- a/scripts/azure-cc/Dockerfile +++ b/scripts/azure-cc/Dockerfile @@ -1,4 +1,4 @@ -# Use Alpine-based JRE image +# sha from https://hub.docker.com/layers/amd64/eclipse-temurin/21.0.4_7-jre-alpine/images/sha256-8179ddc8a6c5ac9af935020628763b9a5a671e0914976715d2b61b21881cefca FROM eclipse-temurin@sha256:8179ddc8a6c5ac9af935020628763b9a5a671e0914976715d2b61b21881cefca # Install necessary packages and set up virtual environment From 82168fabcb4d59f2db5023eb13a4d2639b85956a Mon Sep 17 00:00:00 2001 From: Ian Nara Date: Wed, 12 Feb 2025 13:54:06 -0700 Subject: [PATCH 266/431] update workflow to use shared action --- .../publish-gcp-oidc-enclave-docker.yaml | 25 ++----------------- 1 file changed, 2 insertions(+), 23 deletions(-) diff --git a/.github/workflows/publish-gcp-oidc-enclave-docker.yaml b/.github/workflows/publish-gcp-oidc-enclave-docker.yaml index 742d73b8e..b4af0bba2 100644 --- a/.github/workflows/publish-gcp-oidc-enclave-docker.yaml +++ b/.github/workflows/publish-gcp-oidc-enclave-docker.yaml @@ -156,31 +156,10 @@ jobs: IMAGE_VERSION=${{ steps.update_version.outputs.new_version }} BUILD_TARGET=${{ env.ENCLAVE_PROTOCOL }} - - name: Generate Trivy vulnerability scan report - uses: aquasecurity/trivy-action@0.14.0 + - name: Vulnerability Scan + uses: IABTechLab/uid2-shared-actions/actions/vulnerability_scan@v3 with: image-ref: ${{ steps.meta.outputs.tags }} - format: 'sarif' - exit-code: '0' - ignore-unfixed: true - severity: 'CRITICAL,HIGH' - output: 'trivy-results.sarif' - hide-progress: true - - - name: Upload Trivy scan report to GitHub Security tab - uses: github/codeql-action/upload-sarif@v3 - with: - sarif_file: 'trivy-results.sarif' - - - name: Test with Trivy vulnerability scanner - uses: aquasecurity/trivy-action@0.14.0 - with: - image-ref: ${{ steps.meta.outputs.tags }} - format: 'table' - exit-code: '1' - ignore-unfixed: true - severity: ${{ inputs.vulnerability_severity }} - hide-progress: true - name: Push to Docker id: push-to-docker From 01ee14b3fe1f856f2a801f355c9e3d0cec7310a4 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Wed, 12 Feb 2025 20:55:32 +0000 Subject: [PATCH 267/431] [CI Pipeline] Released Snapshot version: 5.47.21-alpha-193-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c190deb9b..61443a04b 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.47.20 + 5.47.21-alpha-193-SNAPSHOT UTF-8 From 1453e5a085bed58041c647167cbe115ff5d19287 Mon Sep 17 00:00:00 2001 From: Ian Nara Date: Wed, 12 Feb 2025 13:56:25 -0700 Subject: [PATCH 268/431] update input --- .github/workflows/publish-gcp-oidc-enclave-docker.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-gcp-oidc-enclave-docker.yaml b/.github/workflows/publish-gcp-oidc-enclave-docker.yaml index b4af0bba2..08a9a6067 100644 --- a/.github/workflows/publish-gcp-oidc-enclave-docker.yaml +++ b/.github/workflows/publish-gcp-oidc-enclave-docker.yaml @@ -159,7 +159,7 @@ jobs: - name: Vulnerability Scan uses: IABTechLab/uid2-shared-actions/actions/vulnerability_scan@v3 with: - image-ref: ${{ steps.meta.outputs.tags }} + image_ref: ${{ steps.meta.outputs.tags }} - name: Push to Docker id: push-to-docker From 39e47be1423ffc12898dfc2c74cd80edf747c5b5 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Wed, 12 Feb 2025 20:57:40 +0000 Subject: [PATCH 269/431] [CI Pipeline] Released Snapshot version: 5.47.22-alpha-194-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 61443a04b..1ec18e8d7 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.47.21-alpha-193-SNAPSHOT + 5.47.22-alpha-194-SNAPSHOT UTF-8 From 738c662ab208766111b6bf0c214c114a5abd09c3 Mon Sep 17 00:00:00 2001 From: Ian Nara Date: Wed, 12 Feb 2025 14:01:26 -0700 Subject: [PATCH 270/431] set scan type --- .github/workflows/publish-gcp-oidc-enclave-docker.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/publish-gcp-oidc-enclave-docker.yaml b/.github/workflows/publish-gcp-oidc-enclave-docker.yaml index 08a9a6067..0c9fa31bd 100644 --- a/.github/workflows/publish-gcp-oidc-enclave-docker.yaml +++ b/.github/workflows/publish-gcp-oidc-enclave-docker.yaml @@ -160,6 +160,7 @@ jobs: uses: IABTechLab/uid2-shared-actions/actions/vulnerability_scan@v3 with: image_ref: ${{ steps.meta.outputs.tags }} + scan_type: 'image' - name: Push to Docker id: push-to-docker From 66743a0eeba32cb0dd4d44e675423e5360f85b18 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Wed, 12 Feb 2025 21:02:22 +0000 Subject: [PATCH 271/431] [CI Pipeline] Released Snapshot version: 5.47.23-alpha-195-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1ec18e8d7..53436e33d 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.47.22-alpha-194-SNAPSHOT + 5.47.23-alpha-195-SNAPSHOT UTF-8 From ab331d56b55774b44d90734dc63f2add8ce77b81 Mon Sep 17 00:00:00 2001 From: Ian Nara Date: Wed, 12 Feb 2025 14:04:39 -0700 Subject: [PATCH 272/431] set failure severity --- .github/workflows/publish-gcp-oidc-enclave-docker.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/publish-gcp-oidc-enclave-docker.yaml b/.github/workflows/publish-gcp-oidc-enclave-docker.yaml index 0c9fa31bd..bf3d58c11 100644 --- a/.github/workflows/publish-gcp-oidc-enclave-docker.yaml +++ b/.github/workflows/publish-gcp-oidc-enclave-docker.yaml @@ -161,6 +161,7 @@ jobs: with: image_ref: ${{ steps.meta.outputs.tags }} scan_type: 'image' + failure_severity: ${{ inputs.vulnerability_severity }} - name: Push to Docker id: push-to-docker From 0d28ae27cc99cd704ce733002475d1ed3b866d2b Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Wed, 12 Feb 2025 21:07:06 +0000 Subject: [PATCH 273/431] [CI Pipeline] Released Snapshot version: 5.47.24-alpha-196-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 53436e33d..0c1c5fc31 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.47.23-alpha-195-SNAPSHOT + 5.47.24-alpha-196-SNAPSHOT UTF-8 From e84a5f28ff9d186ed4ff59c3c766f4e469ef54cd Mon Sep 17 00:00:00 2001 From: Ian Nara Date: Wed, 12 Feb 2025 14:14:18 -0700 Subject: [PATCH 274/431] update input --- .github/workflows/publish-gcp-oidc-enclave-docker.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-gcp-oidc-enclave-docker.yaml b/.github/workflows/publish-gcp-oidc-enclave-docker.yaml index bf3d58c11..d49ad397d 100644 --- a/.github/workflows/publish-gcp-oidc-enclave-docker.yaml +++ b/.github/workflows/publish-gcp-oidc-enclave-docker.yaml @@ -161,7 +161,7 @@ jobs: with: image_ref: ${{ steps.meta.outputs.tags }} scan_type: 'image' - failure_severity: ${{ inputs.vulnerability_severity }} + failure_severity: ${{ inputs.vulnerability_severity == 'CRITICAL (DO NOT use if JIRA ticket not raised)' ? 'CRITICAL' : inputs.vulnerability_severity }} - name: Push to Docker id: push-to-docker From 4b7355e75a196e7e33756274310aefd8d26f72e9 Mon Sep 17 00:00:00 2001 From: Ian Nara Date: Wed, 12 Feb 2025 14:18:17 -0700 Subject: [PATCH 275/431] reset pom --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0c1c5fc31..c190deb9b 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.47.24-alpha-196-SNAPSHOT + 5.47.20 UTF-8 From f145d60a1febce9f1382b70ee5548edc25facc8d Mon Sep 17 00:00:00 2001 From: Ian Nara Date: Wed, 12 Feb 2025 14:20:54 -0700 Subject: [PATCH 276/431] fix conditional --- .github/workflows/publish-gcp-oidc-enclave-docker.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-gcp-oidc-enclave-docker.yaml b/.github/workflows/publish-gcp-oidc-enclave-docker.yaml index d49ad397d..356a5e1eb 100644 --- a/.github/workflows/publish-gcp-oidc-enclave-docker.yaml +++ b/.github/workflows/publish-gcp-oidc-enclave-docker.yaml @@ -161,7 +161,7 @@ jobs: with: image_ref: ${{ steps.meta.outputs.tags }} scan_type: 'image' - failure_severity: ${{ inputs.vulnerability_severity == 'CRITICAL (DO NOT use if JIRA ticket not raised)' ? 'CRITICAL' : inputs.vulnerability_severity }} + failure_severity: ${{ (inputs.vulnerability_severity == 'CRITICAL (DO NOT use if JIRA ticket not raised)' && 'CRITICAL') || inputs.vulnerability_severity }} - name: Push to Docker id: push-to-docker From 54751661634c01fff3fe44cbcafc53c4a380fffb Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Wed, 12 Feb 2025 21:21:52 +0000 Subject: [PATCH 277/431] [CI Pipeline] Released Snapshot version: 5.47.21-alpha-199-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c190deb9b..08ecda2d7 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.47.20 + 5.47.21-alpha-199-SNAPSHOT UTF-8 From d471790585ff151a66b21d8bacfa9fd2dbe690cb Mon Sep 17 00:00:00 2001 From: Ian Nara <135270994+Ian-Nara@users.noreply.github.com> Date: Wed, 12 Feb 2025 14:26:18 -0700 Subject: [PATCH 278/431] Update pom.xml --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 08ecda2d7..c190deb9b 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.47.21-alpha-199-SNAPSHOT + 5.47.20 UTF-8 From f24ecb33320d12c09b7a3c7f85f6160996c4235b Mon Sep 17 00:00:00 2001 From: Katherine Chen Date: Mon, 17 Feb 2025 19:56:50 +1100 Subject: [PATCH 279/431] Fix vulnerability (#1417) * Fix vulnerability * Update to eclipse-temurin@sha256:5c183bdf9d266f79272de8b40f99d579a1da3ceb720cc3c2e5c93f60a15f2636 * Use eclipse-temurin@sha256:f184bb601f9e6068dd0a92738764d1ff447ab68c15ddbf8c303c5c29de9a1df8 * [CI Pipeline] Released Snapshot version: 5.47.21-alpha-164-SNAPSHOT --------- Co-authored-by: Release Workflow --- .trivyignore | 3 --- Dockerfile | 4 ++-- pom.xml | 2 +- scripts/azure-cc/Dockerfile | 4 ++-- scripts/gcp-oidc/Dockerfile | 4 ++-- 5 files changed, 7 insertions(+), 10 deletions(-) diff --git a/.trivyignore b/.trivyignore index 49d44bf36..9d173d399 100644 --- a/.trivyignore +++ b/.trivyignore @@ -8,6 +8,3 @@ CVE-2024-47535 # https://thetradedesk.atlassian.net/browse/UID2-4874 CVE-2025-24970 - -# https://thetradedesk.atlassian.net/browse/UID2-4891 -CVE-2024-12797 \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 38acf8ebd..40b6424e9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ -# sha from https://hub.docker.com/layers/amd64/eclipse-temurin/21.0.4_7-jre-alpine/images/sha256-8179ddc8a6c5ac9af935020628763b9a5a671e0914976715d2b61b21881cefca -FROM eclipse-temurin@sha256:8179ddc8a6c5ac9af935020628763b9a5a671e0914976715d2b61b21881cefca +# sha from https://hub.docker.com/layers/amd64/eclipse-temurin/21.0.6_7-jre-alpine/images/sha256-f184bb601f9e6068dd0a92738764d1ff447ab68c15ddbf8c303c5c29de9a1df8 +FROM eclipse-temurin@sha256:f184bb601f9e6068dd0a92738764d1ff447ab68c15ddbf8c303c5c29de9a1df8 WORKDIR /app EXPOSE 8080 diff --git a/pom.xml b/pom.xml index c190deb9b..67b85da55 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.47.20 + 5.47.21-alpha-164-SNAPSHOT UTF-8 diff --git a/scripts/azure-cc/Dockerfile b/scripts/azure-cc/Dockerfile index 5dfe55a51..c34661fa1 100644 --- a/scripts/azure-cc/Dockerfile +++ b/scripts/azure-cc/Dockerfile @@ -1,5 +1,5 @@ -# sha from https://hub.docker.com/layers/amd64/eclipse-temurin/21.0.4_7-jre-alpine/images/sha256-8179ddc8a6c5ac9af935020628763b9a5a671e0914976715d2b61b21881cefca -FROM eclipse-temurin@sha256:8179ddc8a6c5ac9af935020628763b9a5a671e0914976715d2b61b21881cefca +# sha from https://hub.docker.com/layers/amd64/eclipse-temurin/21.0.6_7-jre-alpine/images/sha256-f184bb601f9e6068dd0a92738764d1ff447ab68c15ddbf8c303c5c29de9a1df8 +FROM eclipse-temurin@sha256:f184bb601f9e6068dd0a92738764d1ff447ab68c15ddbf8c303c5c29de9a1df8 # Install necessary packages and set up virtual environment RUN apk update && apk add --no-cache jq python3 py3-pip && \ diff --git a/scripts/gcp-oidc/Dockerfile b/scripts/gcp-oidc/Dockerfile index b8abd015f..aa3ead08f 100644 --- a/scripts/gcp-oidc/Dockerfile +++ b/scripts/gcp-oidc/Dockerfile @@ -1,5 +1,5 @@ -# sha from https://hub.docker.com/layers/amd64/eclipse-temurin/21.0.4_7-jre-alpine/images/sha256-8179ddc8a6c5ac9af935020628763b9a5a671e0914976715d2b61b21881cefca -FROM eclipse-temurin@sha256:8179ddc8a6c5ac9af935020628763b9a5a671e0914976715d2b61b21881cefca +# sha from https://hub.docker.com/layers/amd64/eclipse-temurin/21.0.6_7-jre-alpine/images/sha256-f184bb601f9e6068dd0a92738764d1ff447ab68c15ddbf8c303c5c29de9a1df8 +FROM eclipse-temurin@sha256:f184bb601f9e6068dd0a92738764d1ff447ab68c15ddbf8c303c5c29de9a1df8 LABEL "tee.launch_policy.allow_env_override"="API_TOKEN_SECRET_NAME,DEPLOYMENT_ENVIRONMENT,CORE_BASE_URL,OPTOUT_BASE_URL,DEBUG_MODE,SKIP_VALIDATIONS" LABEL "tee.launch_policy.log_redirect"="always" From db99edeedfce6c714f2a1131cff67d1ff3b25707 Mon Sep 17 00:00:00 2001 From: Katherine Chen Date: Tue, 18 Feb 2025 09:55:29 +1100 Subject: [PATCH 280/431] Include CVE-2024-57699 to trivyignore (#1422) --- .trivyignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.trivyignore b/.trivyignore index 9d173d399..6219198f4 100644 --- a/.trivyignore +++ b/.trivyignore @@ -8,3 +8,6 @@ CVE-2024-47535 # https://thetradedesk.atlassian.net/browse/UID2-4874 CVE-2025-24970 + +# https://thetradedesk.atlassian.net/browse/UID2-4925 +CVE-2024-57699 From 342964a80e68ee30e8b9975ee89fa947bd3df73b Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Tue, 18 Feb 2025 23:09:29 +0000 Subject: [PATCH 281/431] [CI Pipeline] Released Patch version: 5.47.42 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 67b85da55..831234a30 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.47.21-alpha-164-SNAPSHOT + 5.47.42 UTF-8 From 2f2e881b30754dedd1a4295cc1e6a1d3d5ae0043 Mon Sep 17 00:00:00 2001 From: Matt Collins Date: Wed, 19 Feb 2025 13:13:22 +1100 Subject: [PATCH 282/431] Fix core port --- conf/validator-latest-e2e-docker-public-config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/validator-latest-e2e-docker-public-config.json b/conf/validator-latest-e2e-docker-public-config.json index 331862b7e..2c9b11442 100644 --- a/conf/validator-latest-e2e-docker-public-config.json +++ b/conf/validator-latest-e2e-docker-public-config.json @@ -36,7 +36,7 @@ "runtime_config_store": { "type": "http", "config" : { - "url": "http://core:8080/operator/config" + "url": "http://core:8088/operator/config" }, "config_scan_period_ms": 300000 } From a9c010c272e5bd6462b294cce66f85761ecbc8ba Mon Sep 17 00:00:00 2001 From: Matt Collins Date: Thu, 20 Feb 2025 11:35:49 +1100 Subject: [PATCH 283/431] Disable remote config --- src/main/java/com/uid2/operator/Main.java | 79 +++++------------------ 1 file changed, 15 insertions(+), 64 deletions(-) diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index bc285d4fa..f25d2d1fb 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -39,9 +39,9 @@ import io.micrometer.prometheus.PrometheusRenameFilter; import io.vertx.config.ConfigRetriever; import io.vertx.config.ConfigRetrieverOptions; +import io.vertx.config.ConfigStoreOptions; import io.vertx.core.*; import io.vertx.core.http.HttpServerOptions; -import io.vertx.core.json.JsonArray; import io.vertx.core.json.JsonObject; import io.vertx.micrometer.*; import io.vertx.micrometer.backends.BackendRegistries; @@ -59,7 +59,6 @@ import java.util.*; import java.util.function.Supplier; -import static com.uid2.operator.Const.Config.ConfigScanPeriodMsProp; import static io.micrometer.core.instrument.Metrics.globalRegistry; public class Main { @@ -269,68 +268,20 @@ private ICloudStorage wrapCloudStorageForOptOut(ICloudStorage cloudStorage) { } } - private Future initialiseConfigService() throws Exception { - Promise promise = Promise.promise(); - - ConfigRetriever dynamicConfigRetriever = ConfigRetrieverFactory.create( - vertx, - config.getJsonObject("runtime_config_store"), - this.createOperatorKeyRetriever().retrieve() - ); - Future dynamicConfigFuture = ConfigService.create(dynamicConfigRetriever); - - ConfigRetriever staticConfigRetriever = ConfigRetrieverFactory.create( - vertx, - new JsonObject() - .put("type", "json") - .put("config", config) - .put(ConfigScanPeriodMsProp, -1), - "" - ); - - Future staticConfigFuture = ConfigService.create(staticConfigRetriever); - - // Create ConfigRetriever with options specified in JsonObject. - // Includes an optional file store and scan period of 10000 ms (10 sec). - // See https://vertx.io/docs/vertx-config/java/#_file for Vertx file store docs. - ConfigRetriever featureFlagConfigRetriever = ConfigRetriever.create( - vertx, - new ConfigRetrieverOptions( - new JsonObject() - .put("stores", new JsonArray() - .add(new JsonObject() - .put("type", "file") - .put("optional", true) - .put("config", new JsonObject() - .put("path", "conf/feat-flag/feat-flag.json")))) - .put("scanPeriod", 10000)) - ); - - Future.all(dynamicConfigFuture, staticConfigFuture, featureFlagConfigRetriever.getConfig()) - .onComplete(ar -> { - if (ar.succeeded()) { - CompositeFuture configServiceManagerCompositeFuture = ar.result(); - IConfigService dynamicConfigService = configServiceManagerCompositeFuture.resultAt(0); - IConfigService staticConfigService = configServiceManagerCompositeFuture.resultAt(1); - JsonObject featureFlagConfig = configServiceManagerCompositeFuture.resultAt(2); - - boolean remoteConfigFeatureFlag = featureFlagConfig - .getJsonObject("remote_config") - .getBoolean("enabled", false); - - ConfigServiceManager configServiceManager = new ConfigServiceManager( - vertx, dynamicConfigService, staticConfigService, remoteConfigFeatureFlag); - - setupFeatureFlagListener(configServiceManager, featureFlagConfigRetriever); - - IConfigService configService = configServiceManager.getDelegatingConfigService(); - promise.complete(configService); - } else { - LOGGER.error("Failed to initialise ConfigService: ", ar.cause()); - promise.fail(ar.cause()); - } - }); - return promise.future(); + private Future initialiseConfigService() { + // Read runtime config values from this.config. + final ConfigStoreOptions configStoreOptions = new ConfigStoreOptions() + .setType("json") + .setConfig(config); + + final ConfigRetrieverOptions configRetrieverOptions = new ConfigRetrieverOptions() + .addStore(configStoreOptions) + // Don't scan as config values won't change. + .setScanPeriod(-1); + + final ConfigRetriever configRetriever = ConfigRetriever.create(vertx, configRetrieverOptions); + + return ConfigService.create(configRetriever).map(x -> (IConfigService) x); } private void run() throws Exception { From 9f44b6f867e7773e11bf1f0e88ccb5c59e443e8b Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Thu, 20 Feb 2025 02:29:06 +0000 Subject: [PATCH 284/431] [CI Pipeline] Released Snapshot version: 5.47.43-alpha-166-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 831234a30..af2663e16 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.47.42 + 5.47.43-alpha-166-SNAPSHOT UTF-8 From a65ee0b2ff324cb44e0077a7e518855e3b9058e6 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Thu, 20 Feb 2025 05:02:41 +0000 Subject: [PATCH 285/431] [CI Pipeline] Released Patch version: 5.47.47 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index af2663e16..ad02103be 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.47.43-alpha-166-SNAPSHOT + 5.47.47 UTF-8 From f1c5ab098de2897590a60f3edc40830db89ea11f Mon Sep 17 00:00:00 2001 From: Scott Sundahl Date: Fri, 21 Feb 2025 16:00:05 -0700 Subject: [PATCH 286/431] version header in old sdk files --- static/js/uid2-sdk-0.0.1a-source.ts | 3 +-- static/js/uid2-sdk-0.0.1a.js | 2 ++ static/js/uid2-sdk-0.0.1b.js | 2 ++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/static/js/uid2-sdk-0.0.1a-source.ts b/static/js/uid2-sdk-0.0.1a-source.ts index c94526e9b..c77e5900d 100644 --- a/static/js/uid2-sdk-0.0.1a-source.ts +++ b/static/js/uid2-sdk-0.0.1a-source.ts @@ -1,7 +1,5 @@ class UID2 { - - public init = (opts : object) => { const identity = opts["identity"]; if (identity) { @@ -21,6 +19,7 @@ class UID2 { req.overrideMimeType("application/json"); var cb = this.handleRefreshResponse; req.open("GET", url, false); + req.setRequestHeader('X-UID2-Client-Version', 'uid2-sdk-0.0.1a-source.ts'); req.onload = function() { cb(req.responseText); } diff --git a/static/js/uid2-sdk-0.0.1a.js b/static/js/uid2-sdk-0.0.1a.js index c6920ad11..161fa8149 100644 --- a/static/js/uid2-sdk-0.0.1a.js +++ b/static/js/uid2-sdk-0.0.1a.js @@ -1,5 +1,6 @@ class UID2 { constructor() { + this.version = "0.0.1a.js"; this.init = (opts) => { const identity = opts["identity"]; if (identity) { @@ -17,6 +18,7 @@ class UID2 { req.overrideMimeType("application/json"); var cb = this.handleRefreshResponse; req.open("GET", url, false); + req.setRequestHeader('X-UID2-Client-Version', 'uid2-sdk-' + this.version); req.onload = function () { cb(req.responseText); }; diff --git a/static/js/uid2-sdk-0.0.1b.js b/static/js/uid2-sdk-0.0.1b.js index 712704d91..f7357a056 100644 --- a/static/js/uid2-sdk-0.0.1b.js +++ b/static/js/uid2-sdk-0.0.1b.js @@ -21,6 +21,7 @@ if (typeof (googletag) !== "undefined" && googletag && googletag.encryptedSignal class UID2 { constructor() { + this.version = "0.0.1b.js"; this.init = (opts) => { const identity = opts["identity"]; if (identity) { @@ -38,6 +39,7 @@ class UID2 { req.overrideMimeType("application/json"); var cb = this.handleRefreshResponse; req.open("GET", url, false); + req.setRequestHeader('X-UID2-Client-Version', 'uid2-sdk-' + this.version); req.onload = function () { cb(req.responseText); }; From 8a39bceff2150609f604ff1b08dae4a735816803 Mon Sep 17 00:00:00 2001 From: abuabraham-ttd <130511456+abuabraham-ttd@users.noreply.github.com> Date: Mon, 24 Feb 2025 10:46:10 -0800 Subject: [PATCH 287/431] Abu UID2 4880 fix e2e (#1412) * Change URL update to entire URL --- Makefile.eif | 5 ++- pom.xml | 4 +-- scripts/aws/Dockerfile | 2 +- scripts/aws/conf/default-config.json | 7 ++-- scripts/aws/conf/euid-integ-config.json | 9 +---- scripts/aws/conf/uid2-integ-config.json | 18 +++------- scripts/aws/ec2.py | 2 +- scripts/aws/entrypoint.sh | 36 +++++++++---------- scripts/azure-cc/azureEntryPoint.py | 38 +++++++------------- scripts/azure-cc/conf/integ-uid2-config.json | 24 ++++++------- scripts/azure-cc/conf/prod-uid2-config.json | 24 ++++++------- scripts/confidential_compute.py | 4 +-- scripts/gcp-oidc/gcp.py | 6 ++-- 13 files changed, 76 insertions(+), 103 deletions(-) diff --git a/Makefile.eif b/Makefile.eif index 38e47c13c..2b65069f5 100644 --- a/Makefile.eif +++ b/Makefile.eif @@ -28,11 +28,14 @@ euidoperator.eif: build_artifacts build_configs build/proxies.nitro.yaml build/s .PHONY: build_configs -build_configs: build/conf/default-config.json build/conf/euid-integ-config.json build/conf/euid-prod-config.json build/conf/uid2-integ-config.json build/conf/uid2-prod-config.json build/conf/logback.xml build/conf/logback-debug.xml +build_configs: build/conf/default-config.json build/conf/feat-flag.json build/conf/euid-integ-config.json build/conf/euid-prod-config.json build/conf/uid2-integ-config.json build/conf/uid2-prod-config.json build/conf/logback.xml build/conf/logback-debug.xml build/conf/default-config.json: build_artifacts ./scripts/aws/conf/default-config.json cp ./scripts/aws/conf/default-config.json ./build/conf/ +build/conf/feat-flag.json: build_artifacts ./scripts/aws/conf/feat-flag/feat-flag.json + cp ./scripts/aws/conf/feat-flag/feat-flag.json ./build/conf/ + build/conf/euid-integ-config.json: build_artifacts ./scripts/aws/conf/euid-integ-config.json cp ./scripts/aws/conf/euid-integ-config.json ./build/conf/ diff --git a/pom.xml b/pom.xml index ad02103be..00fac30af 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.47.47 + 5.47.73-alpha-180-SNAPSHOT UTF-8 @@ -22,7 +22,7 @@ 2.1.0 2.1.0 2.1.0 - 8.0.32 + 8.1.10 ${project.version} 21 21 diff --git a/scripts/aws/Dockerfile b/scripts/aws/Dockerfile index b0d6a10ff..88e1a2242 100644 --- a/scripts/aws/Dockerfile +++ b/scripts/aws/Dockerfile @@ -37,7 +37,7 @@ COPY ./conf/default-config.json /app/conf/ COPY ./conf/*.json /app/conf/ COPY ./conf/*.xml /app/conf/ COPY ./syslog-ng-client.conf /etc/syslog-ng/syslog-ng.conf -COPY ./conf/feat-flag/feat-flag.json /app/conf/feat-flag/ +COPY ./conf/feat-flag.json /app/conf/feat-flag/ RUN chmod +x /app/vsockpx && chmod +x /app/entrypoint.sh diff --git a/scripts/aws/conf/default-config.json b/scripts/aws/conf/default-config.json index 35c3be58c..8f4477336 100644 --- a/scripts/aws/conf/default-config.json +++ b/scripts/aws/conf/default-config.json @@ -30,9 +30,12 @@ "service_links_metadata_path": "service_links/metadata.json", "optout_metadata_path": null, "optout_inmem_cache": false, - "enclave_platform": null, + "enclave_platform": "aws-nitro", "failure_shutdown_wait_hours": 120, "sharing_token_expiry_seconds": 2592000, "validate_service_links": false, + "identity_token_expires_after_seconds": 86400, + "refresh_token_expires_after_seconds": 2592000, + "refresh_identity_token_after_seconds": 3600, "operator_type": "private" -} +} \ No newline at end of file diff --git a/scripts/aws/conf/euid-integ-config.json b/scripts/aws/conf/euid-integ-config.json index ca7c331e3..9e82cee9d 100644 --- a/scripts/aws/conf/euid-integ-config.json +++ b/scripts/aws/conf/euid-integ-config.json @@ -11,12 +11,5 @@ "core_attest_url": "https://core.integ.euid.eu/attest", "optout_api_uri": "https://optout.integ.euid.eu/optout/replicate", "optout_s3_folder": "optout/", - "allow_legacy_api": false, - "runtime_config_store": { - "type": "http", - "config" : { - "url": "https://core.integ.euid.eu/operator/config" - }, - "config_scan_period_ms": 300000 - } + "allow_legacy_api": false } \ No newline at end of file diff --git a/scripts/aws/conf/uid2-integ-config.json b/scripts/aws/conf/uid2-integ-config.json index 8f0252082..8ab5b3e7b 100644 --- a/scripts/aws/conf/uid2-integ-config.json +++ b/scripts/aws/conf/uid2-integ-config.json @@ -1,22 +1,14 @@ { + "core_attest_url": "https://core-integ.uidapi.com/attest", + "optout_api_uri": "https://optout-integ.uidapi.com/optout/replicate", "sites_metadata_path": "https://core-integ.uidapi.com/sites/refresh", "clients_metadata_path": "https://core-integ.uidapi.com/clients/refresh", + "client_side_keypairs_metadata_path": "https://core-integ.uidapi.com/client_side_keypairs/refresh", "keysets_metadata_path": "https://core-integ.uidapi.com/key/keyset/refresh", "keyset_keys_metadata_path": "https://core-integ.uidapi.com/key/keyset-keys/refresh", - "client_side_keypairs_metadata_path": "https://core-integ.uidapi.com/client_side_keypairs/refresh", "salts_metadata_path": "https://core-integ.uidapi.com/salt/refresh", "services_metadata_path": "https://core-integ.uidapi.com/services/refresh", "service_links_metadata_path": "https://core-integ.uidapi.com/service_links/refresh", "optout_metadata_path": "https://optout-integ.uidapi.com/optout/refresh", - "core_attest_url": "https://core-integ.uidapi.com/attest", - "optout_api_uri": "https://optout-integ.uidapi.com/optout/replicate", - "optout_s3_folder": "uid-optout-integ/", - "allow_legacy_api": false, - "runtime_config_store": { - "type": "http", - "config" : { - "url": "https://core-integ.uidapi.com/operator/config" - }, - "config_scan_period_ms": 300000 - } -} \ No newline at end of file + "optout_s3_folder": "uid-optout-integ/" +} diff --git a/scripts/aws/ec2.py b/scripts/aws/ec2.py index fc9243fc2..627a6c88c 100644 --- a/scripts/aws/ec2.py +++ b/scripts/aws/ec2.py @@ -15,7 +15,7 @@ import sys import time import yaml - +logging.basicConfig(level=logging.INFO) sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from confidential_compute import ConfidentialCompute, ConfidentialComputeConfig, InstanceProfileMissingError, OperatorKeyNotFoundError, ConfigurationValueError, ConfidentialComputeStartupError diff --git a/scripts/aws/entrypoint.sh b/scripts/aws/entrypoint.sh index 5f324ce20..4d67a14fa 100755 --- a/scripts/aws/entrypoint.sh +++ b/scripts/aws/entrypoint.sh @@ -10,21 +10,18 @@ exec &> >(tee -a "$LOG_FILE") PARAMETERIZED_CONFIG="/app/conf/config-overrides.json" OPERATOR_CONFIG="/tmp/final-config.json" -setup_auxiliaries() { - set -o pipefail - ulimit -n 65536 +set -o pipefail +ulimit -n 65536 - # -- setup loopback device - echo "Setting up loopback device..." - ifconfig lo 127.0.0.1 +# -- setup loopback device +echo "Setting up loopback device..." +ifconfig lo 127.0.0.1 - # -- start vsock proxy - echo "Starting vsock proxy..." - /app/vsockpx --config /app/proxies.nitro.yaml --daemon --workers $(( ( $(nproc) + 3 ) / 4 )) --log-level 3 - - /usr/sbin/syslog-ng --verbose -} +# -- start vsock proxy +echo "Starting vsock proxy..." +/app/vsockpx --config /app/proxies.nitro.yaml --daemon --workers $(( ( $(nproc) + 3 ) / 4 )) --log-level 3 +/usr/sbin/syslog-ng --verbose build_parameterized_config() { curl -s -f -o "${PARAMETERIZED_CONFIG}" -x socks5h://127.0.0.1:3305 http://127.0.0.1:27015/getConfig @@ -46,9 +43,7 @@ build_parameterized_config() { build_operator_config() { CORE_BASE_URL=$(jq -r ".core_base_url" < "${PARAMETERIZED_CONFIG}") - CORE_BASE_URL=$(echo "$CORE_BASE_URL" | sed -E 's#^(https?://)?([^/]+).*#\2#') OPTOUT_BASE_URL=$(jq -r ".optout_base_url" < "${PARAMETERIZED_CONFIG}") - OPTOUT_BASE_URL=$(echo "$OPTOUT_BASE_URL" | sed -E 's#^(https?://)?([^/]+).*#\2#') DEPLOYMENT_ENVIRONMENT=$(jq -r ".environment" < "${PARAMETERIZED_CONFIG}") DEBUG_MODE=$(jq -r ".debug_mode" < "${PARAMETERIZED_CONFIG}") @@ -69,11 +64,11 @@ build_operator_config() { if [[ "$DEPLOYMENT_ENVIRONMENT" != "prod" ]]; then #Allow override of base URL in non-prod environments - CORE_PATTERN="core.*uidapi.com" - OPTOUT_PATTERN="optout.*uidapi.com" - if [[ "$DEPLOYMENT_ENVIRONMENT" == "euid" ]]; then - CORE_PATTERN="core.*euid.eu" - OPTOUT_PATTERN="optout.*euid.eu" + CORE_PATTERN="https://core.*uidapi.com" + OPTOUT_PATTERN="https://optout.*uidapi.com" + if [[ "$IDENTITY_SCOPE_LOWER" == "euid" ]]; then + CORE_PATTERN="https://core.*euid.eu" + OPTOUT_PATTERN="https://optout.*euid.eu" fi sed -i "s#${CORE_PATTERN}#${CORE_BASE_URL}#g" "${OPERATOR_CONFIG}" sed -i "s#${OPTOUT_PATTERN}#${OPTOUT_BASE_URL}#g" "${OPERATOR_CONFIG}" @@ -81,10 +76,10 @@ build_operator_config() { } -setup_auxiliaries build_parameterized_config build_operator_config + DEBUG_MODE=$(jq -r ".debug_mode" < "${OPERATOR_CONFIG}") LOGBACK_CONF="./conf/logback.xml" @@ -107,3 +102,4 @@ java \ -Dlogback.configurationFile=${LOGBACK_CONF} \ -Dhttp_proxy=socks5://127.0.0.1:3305 \ -jar /app/"${JAR_NAME}"-"${JAR_VERSION}".jar + diff --git a/scripts/azure-cc/azureEntryPoint.py b/scripts/azure-cc/azureEntryPoint.py index 25348e9fd..ffa49b3c7 100644 --- a/scripts/azure-cc/azureEntryPoint.py +++ b/scripts/azure-cc/azureEntryPoint.py @@ -8,7 +8,6 @@ import shutil import requests import logging -from urllib.parse import urlparse from confidential_compute import ConfidentialCompute, ConfigurationMissingError, OperatorKeyPermissionError, OperatorKeyNotFoundError, ConfidentialComputeStartupError from azure.keyvault.secrets import SecretClient from azure.identity import DefaultAzureCredential, CredentialUnavailableError @@ -22,6 +21,8 @@ class AzureEntryPoint(ConfidentialCompute): env_name = os.getenv("DEPLOYMENT_ENVIRONMENT") jar_name = os.getenv("JAR_NAME", "default-jar-name") jar_version = os.getenv("JAR_VERSION", "default-jar-version") + default_core_endpoint = f"https://core-{env_name}.uidapi.com".lower() + default_optout_endpoint = f"https://optout-{env_name}.uidapi.com".lower() FINAL_CONFIG = "/tmp/final-config.json" @@ -50,37 +51,25 @@ def __create_final_config(self): except IOError as e: logging.error(f"Failed to create {AzureEntryPoint.FINAL_CONFIG} with error: {e}") sys.exit(1) - - CORE_BASE_URL = os.getenv("CORE_BASE_URL") - OPTOUT_BASE_URL = os.getenv("OPTOUT_BASE_URL") - if CORE_BASE_URL and OPTOUT_BASE_URL and AzureEntryPoint.env_name != 'prod': - logging.info(f"-- replacing URLs by {CORE_BASE_URL} and {OPTOUT_BASE_URL}") - with open(AzureEntryPoint.FINAL_CONFIG, "r") as file: - config = file.read() - - config = config.replace("core-integ.uidapi.com", urlparse(CORE_BASE_URL).netloc) - config = config.replace("optout-integ.uidapi.com", urlparse(OPTOUT_BASE_URL).netloc) + logging.info(f"-- replacing URLs by {self.configs["core_base_url"]} and {self.configs["optout_base_url"]}") + with open(AzureEntryPoint.FINAL_CONFIG, "r") as file: + config = file.read() - with open(AzureEntryPoint.FINAL_CONFIG, "w") as file: - file.write(config) + config = config.replace("https://core.uidapi.com", self.configs["core_base_url"]) + config = config.replace("https://optout.uidapi.com", self.configs["optout_base_url"]) + with open(AzureEntryPoint.FINAL_CONFIG, "w") as file: + file.write(config) with open(AzureEntryPoint.FINAL_CONFIG, "r") as file: logging.info(file.read()) - def __set_base_urls(self): - with open(AzureEntryPoint.FINAL_CONFIG, "r") as file: - jdata = json.load(file) - self.configs["core_base_url"] = jdata["core_attest_url"] - self.configs["optout_base_url"] = jdata["optout_api_uri"] - def __set_operator_key(self): try: credential = DefaultAzureCredential() kv_URL = f"https://{AzureEntryPoint.kv_name}.vault.azure.net" secret_client = SecretClient(vault_url=kv_URL, credential=credential) secret = secret_client.get_secret(AzureEntryPoint.secret_name) - # print(f"Secret Value: {secret.value}") self.configs["operator_key"] = secret.value except (CredentialUnavailableError, ClientAuthenticationError) as auth_error: @@ -92,14 +81,13 @@ def __set_operator_key(self): def _set_confidential_config(self, secret_identifier: str = None): + """Builds and sets ConfidentialComputeConfig""" self.configs["skip_validations"] = os.getenv("SKIP_VALIDATIONS", "false").lower() == "true" self.configs["debug_mode"] = os.getenv("DEBUG_MODE", "false").lower() == "true" self.configs["environment"] = AzureEntryPoint.env_name - - # set self.configs["operator_key"] + self.configs["core_base_url"] = os.getenv("CORE_BASE_URL") if os.getenv("CORE_BASE_URL") and AzureEntryPoint.env_name == "integ" else AzureEntryPoint.default_core_endpoint + self.configs["optout_base_url"] = os.getenv("OPTOUT_BASE_URL") if os.getenv("OPTOUT_BASE_URL") and AzureEntryPoint.env_name == "integ" else AzureEntryPoint.default_optout_endpoint self.__set_operator_key() - # set base urls from final config file - self.__set_base_urls() def __run_operator(self): @@ -150,10 +138,10 @@ def _validate_auxiliaries(self): def run_compute(self) -> None: """Main execution flow for confidential compute.""" self.__check_env_variables() - self.__create_final_config() self._set_confidential_config() if not self.configs.get("skip_validations"): self.validate_configuration() + self.__create_final_config() self._setup_auxiliaries() self.__run_operator() diff --git a/scripts/azure-cc/conf/integ-uid2-config.json b/scripts/azure-cc/conf/integ-uid2-config.json index e3d34d846..569304e35 100644 --- a/scripts/azure-cc/conf/integ-uid2-config.json +++ b/scripts/azure-cc/conf/integ-uid2-config.json @@ -1,20 +1,20 @@ { - "sites_metadata_path": "https://core-integ.uidapi.com/sites/refresh", - "clients_metadata_path": "https://core-integ.uidapi.com/clients/refresh", - "keysets_metadata_path": "https://core-integ.uidapi.com/key/keyset/refresh", - "keyset_keys_metadata_path": "https://core-integ.uidapi.com/key/keyset-keys/refresh", - "client_side_keypairs_metadata_path": "https://core-integ.uidapi.com/client_side_keypairs/refresh", - "salts_metadata_path": "https://core-integ.uidapi.com/salt/refresh", - "services_metadata_path": "https://core-integ.uidapi.com/services/refresh", - "service_links_metadata_path": "https://core-integ.uidapi.com/service_links/refresh", - "optout_metadata_path": "https://optout-integ.uidapi.com/optout/refresh", - "core_attest_url": "https://core-integ.uidapi.com/attest", - "optout_api_uri": "https://optout-integ.uidapi.com/optout/replicate", + "sites_metadata_path": "https://core.uidapi.com/sites/refresh", + "clients_metadata_path": "https://core.uidapi.com/clients/refresh", + "keysets_metadata_path": "https://core.uidapi.com/key/keyset/refresh", + "keyset_keys_metadata_path": "https://core.uidapi.com/key/keyset-keys/refresh", + "client_side_keypairs_metadata_path": "https://core.uidapi.com/client_side_keypairs/refresh", + "salts_metadata_path": "https://core.uidapi.com/salt/refresh", + "services_metadata_path": "https://core.uidapi.com/services/refresh", + "service_links_metadata_path": "https://core.uidapi.com/service_links/refresh", + "optout_metadata_path": "https://optout.uidapi.com/optout/refresh", + "core_attest_url": "https://core.uidapi.com/attest", + "optout_api_uri": "https://optout.uidapi.com/optout/replicate", "optout_s3_folder": "uid-optout-integ/", "runtime_config_store": { "type": "http", "config" : { - "url": "https://core-integ.uidapi.com/operator/config" + "url": "https://core.uidapi.com/operator/config" }, "config_scan_period_ms": 300000 } diff --git a/scripts/azure-cc/conf/prod-uid2-config.json b/scripts/azure-cc/conf/prod-uid2-config.json index bccbc2444..9c4eba18a 100644 --- a/scripts/azure-cc/conf/prod-uid2-config.json +++ b/scripts/azure-cc/conf/prod-uid2-config.json @@ -1,21 +1,21 @@ { - "sites_metadata_path": "https://core-prod.uidapi.com/sites/refresh", - "clients_metadata_path": "https://core-prod.uidapi.com/clients/refresh", - "keysets_metadata_path": "https://core-prod.uidapi.com/key/keyset/refresh", - "keyset_keys_metadata_path": "https://core-prod.uidapi.com/key/keyset-keys/refresh", - "client_side_keypairs_metadata_path": "https://core-prod.uidapi.com/client_side_keypairs/refresh", - "salts_metadata_path": "https://core-prod.uidapi.com/salt/refresh", - "services_metadata_path": "https://core-prod.uidapi.com/services/refresh", - "service_links_metadata_path": "https://core-prod.uidapi.com/service_links/refresh", - "optout_metadata_path": "https://optout-prod.uidapi.com/optout/refresh", - "core_attest_url": "https://core-prod.uidapi.com/attest", - "optout_api_uri": "https://optout-prod.uidapi.com/optout/replicate", + "sites_metadata_path": "https://core.uidapi.com/sites/refresh", + "clients_metadata_path": "https://core.uidapi.com/clients/refresh", + "keysets_metadata_path": "https://core.uidapi.com/key/keyset/refresh", + "keyset_keys_metadata_path": "https://core.uidapi.com/key/keyset-keys/refresh", + "client_side_keypairs_metadata_path": "https://core.uidapi.com/client_side_keypairs/refresh", + "salts_metadata_path": "https://core.uidapi.com/salt/refresh", + "services_metadata_path": "https://core.uidapi.com/services/refresh", + "service_links_metadata_path": "https://core.uidapi.com/service_links/refresh", + "optout_metadata_path": "https://optout.uidapi.com/optout/refresh", + "core_attest_url": "https://core.uidapi.com/attest", + "optout_api_uri": "https://optout.uidapi.com/optout/replicate", "optout_s3_folder": "optout-v2/", "identity_token_expires_after_seconds": 259200, "runtime_config_store": { "type": "http", "config" : { - "url": "https://core-prod.uidapi.com/operator/config" + "url": "https://core.uidapi.com/operator/config" }, "config_scan_period_ms": 300000 } diff --git a/scripts/confidential_compute.py b/scripts/confidential_compute.py index dd8f2d5bc..bb77fb39a 100644 --- a/scripts/confidential_compute.py +++ b/scripts/confidential_compute.py @@ -98,7 +98,7 @@ def validate_connectivity() -> None: type_hints = get_type_hints(ConfidentialComputeConfig, include_extras=True) required_keys = [field for field, hint in type_hints.items() if "NotRequired" not in str(hint)] - missing_keys = [key for key in required_keys if key not in self.configs] + missing_keys = [key for key in required_keys if key not in self.configs or self.configs[key] == None] if missing_keys: raise ConfigurationMissingError(self.__class__.__name__, missing_keys) @@ -144,7 +144,7 @@ def run_command(command, separate_process=False): if separate_process: subprocess.Popen(command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) else: - subprocess.run(command,check=True) + subprocess.run(command,check=True,text=True) except Exception as e: logging.error(f"Failed to run command: {e}", exc_info=True) raise RuntimeError (f"Failed to start {' '.join(command)} ") \ No newline at end of file diff --git a/scripts/gcp-oidc/gcp.py b/scripts/gcp-oidc/gcp.py index c2704c7ed..ce91c9f32 100644 --- a/scripts/gcp-oidc/gcp.py +++ b/scripts/gcp-oidc/gcp.py @@ -6,10 +6,8 @@ import sys import logging from google.cloud import secretmanager -from google.auth import default from google.auth.exceptions import DefaultCredentialsError from google.api_core.exceptions import PermissionDenied, NotFound -from urllib.parse import urlparse sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from confidential_compute import ConfidentialCompute, ConfidentialComputeConfig, ConfigurationMissingError, OperatorKeyNotFoundError, OperatorKeyPermissionError, ConfidentialComputeStartupError @@ -50,8 +48,8 @@ def __populate_operator_config(self, destination): shutil.copy(target_config, destination) with open(destination, 'r') as file: config = file.read() - config = config.replace("core.uidapi.com", urlparse(self.configs.get("core_base_url")).netloc) - config = config.replace("optout.uidapi.com", urlparse(self.configs.get("optout_base_url")).netloc) + config = config.replace("https://core.uidapi.com", self.configs.get("core_base_url")) + config = config.replace("https://optout.uidapi.com", self.configs.get("optout_base_url")) with open(destination, 'w') as file: file.write(config) From b0f76d0f6bc443054d9356196416a928d15e51da Mon Sep 17 00:00:00 2001 From: Cody Constine Date: Mon, 24 Feb 2025 13:14:46 -0700 Subject: [PATCH 288/431] Trying to add indentity scope to euid aws integ config --- scripts/aws/conf/euid-integ-config.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/aws/conf/euid-integ-config.json b/scripts/aws/conf/euid-integ-config.json index 9e82cee9d..651046136 100644 --- a/scripts/aws/conf/euid-integ-config.json +++ b/scripts/aws/conf/euid-integ-config.json @@ -11,5 +11,6 @@ "core_attest_url": "https://core.integ.euid.eu/attest", "optout_api_uri": "https://optout.integ.euid.eu/optout/replicate", "optout_s3_folder": "optout/", - "allow_legacy_api": false + "allow_legacy_api": false, + "identity_scope": "euid" } \ No newline at end of file From 74e80afeb091f79139f3c95d69945df446cbdfae Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Mon, 24 Feb 2025 20:15:49 +0000 Subject: [PATCH 289/431] [CI Pipeline] Released Snapshot version: 5.47.74-alpha-181-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 00fac30af..cae474edd 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.47.73-alpha-180-SNAPSHOT + 5.47.74-alpha-181-SNAPSHOT UTF-8 From 0a9d44b48446c60b3dff08f162dbd9bd2a675eee Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Mon, 24 Feb 2025 20:18:47 +0000 Subject: [PATCH 290/431] [CI Pipeline] Released Snapshot version: 5.47.75-alpha-168-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index cae474edd..91fd9b248 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.47.74-alpha-181-SNAPSHOT + 5.47.75-alpha-168-SNAPSHOT UTF-8 From 398287a256b18bb31c2dc6d15454f0bdb2f4e7c5 Mon Sep 17 00:00:00 2001 From: Cody Constine Date: Mon, 24 Feb 2025 13:59:53 -0700 Subject: [PATCH 291/431] Adding test e2e branch to pipeline --- .github/workflows/run-e2e-tests-on-operator.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-e2e-tests-on-operator.yaml b/.github/workflows/run-e2e-tests-on-operator.yaml index e57756c1b..4da06bf4e 100644 --- a/.github/workflows/run-e2e-tests-on-operator.yaml +++ b/.github/workflows/run-e2e-tests-on-operator.yaml @@ -106,7 +106,7 @@ on: jobs: e2e-test: name: E2E Test - uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-run-e2e-tests.yaml@v3 + uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-run-e2e-tests.yaml@cbc-UID2-4962-fix-euid-e2e with: operator_type: ${{ inputs.operator_type }} operator_image_version: ${{ inputs.operator_image_version }} From 35ffcf28908f61edc8b5a5ece1f51398cb743243 Mon Sep 17 00:00:00 2001 From: Scott Sundahl Date: Mon, 24 Feb 2025 15:02:24 -0700 Subject: [PATCH 292/431] track old sdk file usage --- .../vertx/ClientVersionCapturingHandler.java | 2 +- .../com/uid2/operator/vertx/Endpoints.java | 2 +- .../operator/vertx/UIDOperatorVerticle.java | 7 ++ static/js/openid-sdk-1.0.js | 7 ++ static/js/uid2-esp-0.0.1a.js | 5 + static/js/uid2-sdk-0.0.1a-source.ts | 96 ------------------- 6 files changed, 21 insertions(+), 98 deletions(-) delete mode 100644 static/js/uid2-sdk-0.0.1a-source.ts diff --git a/src/main/java/com/uid2/operator/vertx/ClientVersionCapturingHandler.java b/src/main/java/com/uid2/operator/vertx/ClientVersionCapturingHandler.java index 1626bb397..5cac8d12f 100644 --- a/src/main/java/com/uid2/operator/vertx/ClientVersionCapturingHandler.java +++ b/src/main/java/com/uid2/operator/vertx/ClientVersionCapturingHandler.java @@ -52,7 +52,7 @@ public void handle(RoutingContext context) { apiContact = profile.getContact(); apiContact = apiContact == null ? "unknown" : apiContact; } catch (Exception ex) { - apiContact = "unknown"; + apiContact = !context.queryParam("apiContact").isEmpty() ? context.queryParam("apiContact").get(0) : "unknown"; } if (clientVersion != null && versions.contains(clientVersion)) { _clientVersionCounters.computeIfAbsent(new Tuple.Tuple2<>(apiContact, clientVersion), tuple -> Counter diff --git a/src/main/java/com/uid2/operator/vertx/Endpoints.java b/src/main/java/com/uid2/operator/vertx/Endpoints.java index 2643f943b..f778f7c72 100644 --- a/src/main/java/com/uid2/operator/vertx/Endpoints.java +++ b/src/main/java/com/uid2/operator/vertx/Endpoints.java @@ -6,7 +6,7 @@ public enum Endpoints { OPS_HEALTHCHECK("/ops/healthcheck"), - + OLD_SDK_LOG("/ops/logSdk"), V0_KEY_LATEST("/key/latest"), V0_TOKEN_GENERATE("/token/generate"), V0_TOKEN_REFRESH("/token/refresh"), diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index 068196015..496186279 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -247,6 +247,8 @@ private Router createRoutesSetup() throws IOException { // Static and health check router.get(OPS_HEALTHCHECK.toString()).handler(this::handleHealthCheck); + // Temporary Old SDK logging + router.get(OLD_SDK_LOG.toString()).handler(this::handleOldSDKVersionLogging); if (this.allowLegacyAPI) { // V1 APIs @@ -813,6 +815,11 @@ private void handleHealthCheck(RoutingContext rc) { } } + private void handleOldSDKVersionLogging(RoutingContext rc) { + // throwaway endpoint to log version + rc.response().end("OK"); + } + private void handleTokenRefreshV1(RoutingContext rc) { final List tokenList = rc.queryParam("refresh_token"); TokenResponseStatsCollector.PlatformType platformType = getPlatformType(rc); diff --git a/static/js/openid-sdk-1.0.js b/static/js/openid-sdk-1.0.js index cc5811c09..4351750a5 100644 --- a/static/js/openid-sdk-1.0.js +++ b/static/js/openid-sdk-1.0.js @@ -1,6 +1,13 @@ var __openId = { init : function(opts) { + // verify if this is still being used + const url = "http://localhost:8080/ops/logSdk?apiContact=" + document.location.hostname; + const req = new XMLHttpRequest(); + req.open("GET", url, false); + req.setRequestHeader("X-UID2-Client-Version", "openid-sdk-1.0"); + req.send(); + this.opts = opts; if (!this.opts["events"]) { this.opts["events"] = {} diff --git a/static/js/uid2-esp-0.0.1a.js b/static/js/uid2-esp-0.0.1a.js index 0cac4edef..1e5b48d4c 100644 --- a/static/js/uid2-esp-0.0.1a.js +++ b/static/js/uid2-esp-0.0.1a.js @@ -1,4 +1,9 @@ function __esp_getUID2Async(cb) { + const url = "http://localhost:8080/ops/logSdk?apiContact=" + document.location.hostname; + const req = new XMLHttpRequest(); + req.open("GET", url, false); + req.setRequestHeader("X-UID2-Client-Version", "uid2-esp-0.0.1a"); + req.send(); return new Promise(function(cb) { if (window.__uid2 && window.__uid2.getAdvertisingToken) { cb(__uid2.getAdvertisingToken()); diff --git a/static/js/uid2-sdk-0.0.1a-source.ts b/static/js/uid2-sdk-0.0.1a-source.ts deleted file mode 100644 index c77e5900d..000000000 --- a/static/js/uid2-sdk-0.0.1a-source.ts +++ /dev/null @@ -1,96 +0,0 @@ -class UID2 { - - public init = (opts : object) => { - const identity = opts["identity"]; - if (identity) { - this.setIdentity(identity); - } else { - this.refreshIfNeeded(); - } - - } - - public refreshIfNeeded = () => { - - const identity = this.getIdentity(); - if (identity) { - const url = "https://prod.uidapi.com/token/refresh?refresh_token="+encodeURIComponent(identity["refresh_token"]); - const req = new XMLHttpRequest(); - req.overrideMimeType("application/json"); - var cb = this.handleRefreshResponse; - req.open("GET", url, false); - req.setRequestHeader('X-UID2-Client-Version', 'uid2-sdk-0.0.1a-source.ts'); - req.onload = function() { - cb(req.responseText); - } - req.send(); - - - - } - } - - private handleRefreshResponse = (body: string) => { - this.setIdentity(body); - } - - public getIdentity = () => { - const payload = this.getCookie("__uid_2"); - if (payload) { - return JSON.parse(payload); - } - } - - public getAdvertisingToken = () => { - const identity = this.getIdentity(); - if (identity) { - return identity["advertisement_token"]; - } - } - - public setIdentity = (value: object) => { - var payload; - if (typeof(value) === "object") { - payload = JSON.stringify(value); - } else { - payload = value; - } - this.setCookie("__uid_2", payload); - - } - - public setCookie = (name: string, value: string) => { - var days = 7; - var date = new Date(); - date.setTime(date.getTime()+(days*24*60*60*1000)); - - document.cookie=name + "=" + encodeURIComponent(value) +" ;path=/;expires="+date.toUTCString(); - - - } - public getCookie = (name: string) => { - const docCookie = document.cookie; - if (docCookie) { - var payload = docCookie.split('; ').find(row => row.startsWith(name)); - if (payload) { - return decodeURIComponent(payload.split('=')[1]) - } - } else { - return undefined; - } - } - - public removeCookie = (name: string) => { - document.cookie=name+"=;path=/;expires=Tue, 1 Jan 1980 23:59:59 GMT"; - } - - public disconnect = () { - this.removeCookie("__uid_2"); -} - - -} - -window.__uid2 = new UID2(); - - From 62d3031bae3b5f3d71e46a2fa6147cdd9c1a4acb Mon Sep 17 00:00:00 2001 From: Scott Sundahl Date: Mon, 24 Feb 2025 15:04:45 -0700 Subject: [PATCH 293/431] correct version string --- static/js/uid2-sdk-0.0.1a.js | 3 +-- static/js/uid2-sdk-0.0.1b.js | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/static/js/uid2-sdk-0.0.1a.js b/static/js/uid2-sdk-0.0.1a.js index 161fa8149..66e068bc1 100644 --- a/static/js/uid2-sdk-0.0.1a.js +++ b/static/js/uid2-sdk-0.0.1a.js @@ -1,6 +1,5 @@ class UID2 { constructor() { - this.version = "0.0.1a.js"; this.init = (opts) => { const identity = opts["identity"]; if (identity) { @@ -18,7 +17,7 @@ class UID2 { req.overrideMimeType("application/json"); var cb = this.handleRefreshResponse; req.open("GET", url, false); - req.setRequestHeader('X-UID2-Client-Version', 'uid2-sdk-' + this.version); + req.setRequestHeader("X-UID2-Client-Version", "uid2-sdk-0.0.1a"); req.onload = function () { cb(req.responseText); }; diff --git a/static/js/uid2-sdk-0.0.1b.js b/static/js/uid2-sdk-0.0.1b.js index f7357a056..c62cd85f8 100644 --- a/static/js/uid2-sdk-0.0.1b.js +++ b/static/js/uid2-sdk-0.0.1b.js @@ -21,7 +21,6 @@ if (typeof (googletag) !== "undefined" && googletag && googletag.encryptedSignal class UID2 { constructor() { - this.version = "0.0.1b.js"; this.init = (opts) => { const identity = opts["identity"]; if (identity) { @@ -39,7 +38,7 @@ class UID2 { req.overrideMimeType("application/json"); var cb = this.handleRefreshResponse; req.open("GET", url, false); - req.setRequestHeader('X-UID2-Client-Version', 'uid2-sdk-' + this.version); + req.setRequestHeader("X-UID2-Client-Version", "uid2-sdk-0.0.1b"); req.onload = function () { cb(req.responseText); }; From ca0ef670b228d3fd2159426fc0b1b27d56a7e206 Mon Sep 17 00:00:00 2001 From: Scott Sundahl Date: Mon, 24 Feb 2025 15:31:48 -0700 Subject: [PATCH 294/431] correct url --- static/js/openid-sdk-1.0.js | 2 +- static/js/uid2-esp-0.0.1a.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/static/js/openid-sdk-1.0.js b/static/js/openid-sdk-1.0.js index 4351750a5..3b8a5f9bb 100644 --- a/static/js/openid-sdk-1.0.js +++ b/static/js/openid-sdk-1.0.js @@ -2,7 +2,7 @@ var __openId = { init : function(opts) { // verify if this is still being used - const url = "http://localhost:8080/ops/logSdk?apiContact=" + document.location.hostname; + const url = "https://prod.uidapi.com/ops/logSdk?apiContact=" + document.location.hostname; const req = new XMLHttpRequest(); req.open("GET", url, false); req.setRequestHeader("X-UID2-Client-Version", "openid-sdk-1.0"); diff --git a/static/js/uid2-esp-0.0.1a.js b/static/js/uid2-esp-0.0.1a.js index 1e5b48d4c..708d50df5 100644 --- a/static/js/uid2-esp-0.0.1a.js +++ b/static/js/uid2-esp-0.0.1a.js @@ -1,5 +1,5 @@ function __esp_getUID2Async(cb) { - const url = "http://localhost:8080/ops/logSdk?apiContact=" + document.location.hostname; + const url = "https://prod.uidapi.com/ops/logSdk?apiContact=" + document.location.hostname; const req = new XMLHttpRequest(); req.open("GET", url, false); req.setRequestHeader("X-UID2-Client-Version", "uid2-esp-0.0.1a"); From 42b0aa4bada732fff12953d5da79d8a090aa42f5 Mon Sep 17 00:00:00 2001 From: Cody Constine Date: Mon, 24 Feb 2025 15:42:26 -0700 Subject: [PATCH 295/431] Adding idenity scope to all the places --- .github/workflows/run-e2e-tests-on-operator.yaml | 2 +- scripts/aws/conf/uid2-integ-config.json | 3 ++- scripts/aws/conf/uid2-prod-config.json | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-e2e-tests-on-operator.yaml b/.github/workflows/run-e2e-tests-on-operator.yaml index 4da06bf4e..e57756c1b 100644 --- a/.github/workflows/run-e2e-tests-on-operator.yaml +++ b/.github/workflows/run-e2e-tests-on-operator.yaml @@ -106,7 +106,7 @@ on: jobs: e2e-test: name: E2E Test - uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-run-e2e-tests.yaml@cbc-UID2-4962-fix-euid-e2e + uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-run-e2e-tests.yaml@v3 with: operator_type: ${{ inputs.operator_type }} operator_image_version: ${{ inputs.operator_image_version }} diff --git a/scripts/aws/conf/uid2-integ-config.json b/scripts/aws/conf/uid2-integ-config.json index 8ab5b3e7b..a99b56813 100644 --- a/scripts/aws/conf/uid2-integ-config.json +++ b/scripts/aws/conf/uid2-integ-config.json @@ -10,5 +10,6 @@ "services_metadata_path": "https://core-integ.uidapi.com/services/refresh", "service_links_metadata_path": "https://core-integ.uidapi.com/service_links/refresh", "optout_metadata_path": "https://optout-integ.uidapi.com/optout/refresh", - "optout_s3_folder": "uid-optout-integ/" + "optout_s3_folder": "uid-optout-integ/", + "identity_scope": "uid2" } diff --git a/scripts/aws/conf/uid2-prod-config.json b/scripts/aws/conf/uid2-prod-config.json index ccbce2e32..d7a03d679 100644 --- a/scripts/aws/conf/uid2-prod-config.json +++ b/scripts/aws/conf/uid2-prod-config.json @@ -19,6 +19,7 @@ "optout_synthetic_logs_count": 0, "optout_inmem_cache": true, "optout_s3_folder": "optout-v2/", + "identity_scope": "uid2", "identity_token_expires_after_seconds": 259200, "refresh_token_expires_after_seconds": 2592000, "refresh_identity_token_after_seconds": 3600, From 098b22074820cb9cf50b6aa6c3207332020d406b Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Tue, 25 Feb 2025 02:56:37 +0000 Subject: [PATCH 296/431] [CI Pipeline] Released Snapshot version: 5.47.74-alpha-209-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 00fac30af..86be55c61 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.47.73-alpha-180-SNAPSHOT + 5.47.74-alpha-209-SNAPSHOT UTF-8 From fa836ff8d8fd1da67aa45bfb207c297f5b4b97a6 Mon Sep 17 00:00:00 2001 From: Vishal Egbert Date: Tue, 25 Feb 2025 16:31:19 +1100 Subject: [PATCH 297/431] Test change --- .github/workflows/publish-gcp-oidc-enclave-docker.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-gcp-oidc-enclave-docker.yaml b/.github/workflows/publish-gcp-oidc-enclave-docker.yaml index 356a5e1eb..991e8c25b 100644 --- a/.github/workflows/publish-gcp-oidc-enclave-docker.yaml +++ b/.github/workflows/publish-gcp-oidc-enclave-docker.yaml @@ -157,7 +157,7 @@ jobs: BUILD_TARGET=${{ env.ENCLAVE_PROTOCOL }} - name: Vulnerability Scan - uses: IABTechLab/uid2-shared-actions/actions/vulnerability_scan@v3 + uses: IABTechLab/uid2-shared-actions/actions/vulnerability_scan@vse-UID2-4968-ignore-google-auth-private-key-string-secret with: image_ref: ${{ steps.meta.outputs.tags }} scan_type: 'image' From 0d61edcfc7a61018b76a46d6a3d4ff5295ebaa68 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Tue, 25 Feb 2025 05:36:44 +0000 Subject: [PATCH 298/431] [CI Pipeline] Released Snapshot version: 5.47.76-alpha-210-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 91fd9b248..5acc3bc0a 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.47.75-alpha-168-SNAPSHOT + 5.47.76-alpha-210-SNAPSHOT UTF-8 From 7169e692eb286d1f30c04ef67e2dc872aa693544 Mon Sep 17 00:00:00 2001 From: Vishal Egbert Date: Tue, 25 Feb 2025 18:39:16 +1100 Subject: [PATCH 299/431] Test changes --- .github/workflows/publish-gcp-oidc-enclave-docker.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/publish-gcp-oidc-enclave-docker.yaml b/.github/workflows/publish-gcp-oidc-enclave-docker.yaml index 991e8c25b..e0cacb324 100644 --- a/.github/workflows/publish-gcp-oidc-enclave-docker.yaml +++ b/.github/workflows/publish-gcp-oidc-enclave-docker.yaml @@ -161,6 +161,7 @@ jobs: with: image_ref: ${{ steps.meta.outputs.tags }} scan_type: 'image' + skip_files: '/*/google/auth/crypt/__pycache__/_python_rsa.cpython-312.pyc' failure_severity: ${{ (inputs.vulnerability_severity == 'CRITICAL (DO NOT use if JIRA ticket not raised)' && 'CRITICAL') || inputs.vulnerability_severity }} - name: Push to Docker From 86018ce6ff3a2845c0fee3facb320fb18f429ce7 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Tue, 25 Feb 2025 07:40:25 +0000 Subject: [PATCH 300/431] [CI Pipeline] Released Snapshot version: 5.47.77-alpha-211-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5acc3bc0a..fdf4bd1e3 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.47.76-alpha-210-SNAPSHOT + 5.47.77-alpha-211-SNAPSHOT UTF-8 From 43d285a3efd0056b2f4be5951fc7a581071ee01c Mon Sep 17 00:00:00 2001 From: Vishal Egbert Date: Tue, 25 Feb 2025 18:51:06 +1100 Subject: [PATCH 301/431] Update skip file --- .github/workflows/publish-gcp-oidc-enclave-docker.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-gcp-oidc-enclave-docker.yaml b/.github/workflows/publish-gcp-oidc-enclave-docker.yaml index e0cacb324..9bab21a3d 100644 --- a/.github/workflows/publish-gcp-oidc-enclave-docker.yaml +++ b/.github/workflows/publish-gcp-oidc-enclave-docker.yaml @@ -161,7 +161,7 @@ jobs: with: image_ref: ${{ steps.meta.outputs.tags }} scan_type: 'image' - skip_files: '/*/google/auth/crypt/__pycache__/_python_rsa.cpython-312.pyc' + skip_files: '/venv/lib/python3.12/site-packages/google/auth/crypt/__pycache__/_python_rsa.cpython-312.pyc' failure_severity: ${{ (inputs.vulnerability_severity == 'CRITICAL (DO NOT use if JIRA ticket not raised)' && 'CRITICAL') || inputs.vulnerability_severity }} - name: Push to Docker From c048df59dbfcea76cb299f4268d7a8969e1648ff Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Tue, 25 Feb 2025 08:09:13 +0000 Subject: [PATCH 302/431] [CI Pipeline] Released Snapshot version: 5.47.78-alpha-212-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index fdf4bd1e3..ebc120a99 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.47.77-alpha-211-SNAPSHOT + 5.47.78-alpha-212-SNAPSHOT UTF-8 From a97368303a0d76e3278256e2f83e747fa04123f5 Mon Sep 17 00:00:00 2001 From: Scott Sundahl Date: Tue, 25 Feb 2025 10:55:39 -0700 Subject: [PATCH 303/431] version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 86be55c61..91fd9b248 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.47.74-alpha-209-SNAPSHOT + 5.47.75-alpha-168-SNAPSHOT UTF-8 From 9f808924b490716fea04ab85cf59c9a5398b94b6 Mon Sep 17 00:00:00 2001 From: Xu Yang <58192524+clarkxuyang@users.noreply.github.com> Date: Tue, 25 Feb 2025 10:07:31 -0800 Subject: [PATCH 304/431] xyang-UID2-4369-enable-nitro-logging (#1403) * xyang-UID2-4369-enable-nitro-logging Co-authored-by: Release Workflow --- pom.xml | 4 +- scripts/aws/UID_CloudFormation.template.yml | 6 +- scripts/aws/ec2.py | 78 ++++++++++++++++----- scripts/aws/sockd.conf | 3 +- scripts/confidential_compute.py | 7 +- 5 files changed, 72 insertions(+), 26 deletions(-) diff --git a/pom.xml b/pom.xml index 91fd9b248..97356ba9f 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.47.75-alpha-168-SNAPSHOT + 5.47.76-alpha-183-SNAPSHOT UTF-8 @@ -22,7 +22,7 @@ 2.1.0 2.1.0 2.1.0 - 8.1.10 + 8.1.15 ${project.version} 21 21 diff --git a/scripts/aws/UID_CloudFormation.template.yml b/scripts/aws/UID_CloudFormation.template.yml index 0c0e9b359..e2422c1fe 100644 --- a/scripts/aws/UID_CloudFormation.template.yml +++ b/scripts/aws/UID_CloudFormation.template.yml @@ -189,11 +189,11 @@ Resources: SecretString: !Join - '' - - '{' - - '"core_base_url": ' + - '"core_base_url": "' - !If [IsIntegEnvironment, 'https://core-integ.uidapi.com', 'https://core.uidapi.com'] - - ', "optout_base_url": ' + - '", "optout_base_url": "' - !If [IsIntegEnvironment, 'https://optout-integ.uidapi.com', 'https://optout.uidapi.com'] - - ', "operator_key": "' + - '", "operator_key": "' - Ref: APIToken - '"' - ', "service_instances": 6' diff --git a/scripts/aws/ec2.py b/scripts/aws/ec2.py index 627a6c88c..a972a290b 100644 --- a/scripts/aws/ec2.py +++ b/scripts/aws/ec2.py @@ -11,7 +11,7 @@ import argparse import logging from botocore.exceptions import ClientError, NoCredentialsError -from typing import Dict +from typing import Dict, List import sys import time import yaml @@ -105,6 +105,8 @@ def add_defaults(configs: Dict[str, any]) -> AWSConfidentialComputeConfig: try: self.configs = add_defaults(json.loads(client.get_secret_value(SecretId=secret_identifier)["SecretString"])) self.__validate_aws_specific_config() + except json.JSONDecodeError as e: + raise OperatorKeyNotFoundError(self.__class__.__name__, f"Can not parse secret {secret_identifier} in {region}") except NoCredentialsError as _: raise InstanceProfileMissingError(self.__class__.__name__) except ClientError as _: @@ -119,38 +121,80 @@ def __get_max_capacity(): except Exception as e: raise RuntimeError("/etc/nitro_enclaves/allocator.yaml does not have CPU, memory allocated") - def __setup_vsockproxy(self, log_level: int) -> None: - """ - Sets up the vsock proxy service. - """ + def __setup_vsockproxy(self) -> None: + logging.info("Sets up the vSock proxy service") thread_count = (multiprocessing.cpu_count() + 1) // 2 command = [ "/usr/bin/vsockpx", "-c", "/etc/uid2operator/proxy.yaml", - "--workers", str(thread_count), "--log-level", str(log_level), "--daemon" + "--workers", str(thread_count), "--daemon" + ] + + debug_command = [ + "/usr/bin/vsockpx", "-c", "/etc/uid2operator/proxy.yaml", + "--workers", str(thread_count), "--log-level", "0" ] - self.run_command(command) + + self.run_service([command, debug_command], "vsock_proxy") def __run_config_server(self) -> None: - """ - Starts the Flask configuration server. - """ + logging.info("Starts the Flask configuration server") os.makedirs("/etc/secret/secret-value", exist_ok=True) config_path = "/etc/secret/secret-value/config" + + # Save configs to a file with open(config_path, 'w') as config_file: json.dump(self.configs, config_file) + os.chdir("/opt/uid2operator/config-server") command = ["./bin/flask", "run", "--host", AuxiliaryConfig.LOCALHOST, "--port", AuxiliaryConfig.FLASK_PORT] - self.run_command(command, separate_process=True) + + self.run_service([command, command], "flask_config_server", separate_process=True) def __run_socks_proxy(self) -> None: + logging.info("Starts the SOCKS proxy service") + command = ["sockd", "-D"] + + # -d specifies debug level + debug_command = ["sockd", "-d", "0"] + + self.run_service([command, debug_command], "socks_proxy") + + def run_service(self, command: List[List[str]], log_filename: str, separate_process: bool = False) -> None: """ - Starts the SOCKS proxy service. + Runs a service command with logging if debug_mode is enabled. + + :param command: command[0] regular command, command[1] debug mode command + :param log_filename: Base name of the log file (e.g., "flask_config_server", "socks_proxy", "vsock_proxy") + :param separate_process: Whether to run in a separate process """ - command = ["sockd", "-D"] - self.run_command(command) + log_file = f"/var/log/{log_filename}.log" + + if self.configs.get("debug_mode") is True: + + # Remove old log file to start fresh + if os.path.exists(log_file): + os.remove(log_file) + + # Set up logging + logging.basicConfig( + filename=log_file, + filemode="w", + level=logging.DEBUG, + format="%(asctime)s %(levelname)s: %(message)s" + ) + + logging.info(f"Debug mode is on, logging into {log_file}") + + # Run debug mode command + with open(log_file, "a") as log: + self.run_command(command[1], separate_process=True, stdout=log, stderr=log) + else: + # Run regular command, possibly daemon + self.run_command(command[0], separate_process=separate_process) def __get_secret_name_from_userdata(self) -> str: """Extracts the secret name from EC2 user data.""" + logging.info("Extracts the secret name from EC2 user data") token = self.__get_aws_token() response = requests.get(AuxiliaryConfig.get_user_data_url(), headers={"X-aws-ec2-metadata-token": token}) user_data = response.text @@ -165,8 +209,7 @@ def __get_secret_name_from_userdata(self) -> str: def _setup_auxiliaries(self) -> None: """Sets up the vsock tunnel, socks proxy and flask server""" - log_level = 1 if self.configs["debug_mode"] else 3 - self.__setup_vsockproxy(log_level) + self.__setup_vsockproxy() self.__run_config_server() self.__run_socks_proxy() logging.info("Finished setting up all auxiliaries") @@ -206,7 +249,7 @@ def __run_nitro_enclave(self): "--enclave-name", "uid2operator" ] if self.configs.get('debug_mode', False): - logging.info("Running in debug_mode") + logging.info("Running nitro in debug_mode") command += ["--debug-mode", "--attach-console"] self.run_command(command, separate_process=False) @@ -248,6 +291,7 @@ def __kill_auxiliaries(self) -> None: parser = argparse.ArgumentParser(description="Manage EC2-based confidential compute workflows.") parser.add_argument("-o", "--operation", choices=["stop", "start"], default="start", help="Operation to perform.") args = parser.parse_args() + try: ec2 = EC2EntryPoint() if args.operation == "stop": diff --git a/scripts/aws/sockd.conf b/scripts/aws/sockd.conf index 6e8814445..1f903407c 100644 --- a/scripts/aws/sockd.conf +++ b/scripts/aws/sockd.conf @@ -3,10 +3,11 @@ external: ens5 user.notprivileged: ec2-user clientmethod: none socksmethod: none +logoutput: stderr client pass { from: 127.0.0.1/32 to: 127.0.0.1/32 - log: error # connect disconnect iooperation + log: error connect # disconnect iooperation } socks pass { diff --git a/scripts/confidential_compute.py b/scripts/confidential_compute.py index bb77fb39a..73b572a0b 100644 --- a/scripts/confidential_compute.py +++ b/scripts/confidential_compute.py @@ -138,13 +138,14 @@ def run_compute(self) -> None: pass @staticmethod - def run_command(command, separate_process=False): + def run_command(command, separate_process=False, stdout=None, stderr=None): logging.info(f"Running command: {' '.join(command)}") try: if separate_process: - subprocess.Popen(command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + subprocess.Popen(command, stdout=stdout, stderr=stderr) else: - subprocess.run(command,check=True,text=True) + subprocess.run(command, check=True, stdout=stdout, stderr=stderr) + except Exception as e: logging.error(f"Failed to run command: {e}", exc_info=True) raise RuntimeError (f"Failed to start {' '.join(command)} ") \ No newline at end of file From d5ebfeed6310ed00b323e8027332cd11c555a05b Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Tue, 25 Feb 2025 18:54:13 +0000 Subject: [PATCH 305/431] [CI Pipeline] Released Patch version: 5.47.57 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 97356ba9f..1af21c75d 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.47.76-alpha-183-SNAPSHOT + 5.47.57 UTF-8 From d067a05670f7194a020487b30527f0ed1fadf4c7 Mon Sep 17 00:00:00 2001 From: Scott Sundahl Date: Tue, 25 Feb 2025 13:44:57 -0700 Subject: [PATCH 306/431] move apiContact logic --- .../vertx/ClientVersionCapturingHandler.java | 20 +++++++++++-------- .../operator/vertx/UIDOperatorVerticle.java | 1 - 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/uid2/operator/vertx/ClientVersionCapturingHandler.java b/src/main/java/com/uid2/operator/vertx/ClientVersionCapturingHandler.java index 5cac8d12f..62351caac 100644 --- a/src/main/java/com/uid2/operator/vertx/ClientVersionCapturingHandler.java +++ b/src/main/java/com/uid2/operator/vertx/ClientVersionCapturingHandler.java @@ -45,14 +45,18 @@ public void handle(RoutingContext context) { clientVersion = !context.queryParam("client").isEmpty() ? context.queryParam("client").get(0) : null; } String apiContact; - try { - final String authHeaderValue = context.request().getHeader("Authorization"); - final String authKey = extractBearerToken(authHeaderValue); - final IAuthorizable profile = this.authKeyStore.get(authKey); - apiContact = profile.getContact(); - apiContact = apiContact == null ? "unknown" : apiContact; - } catch (Exception ex) { - apiContact = !context.queryParam("apiContact").isEmpty() ? context.queryParam("apiContact").get(0) : "unknown"; + // remove in UID2-4990 + apiContact = !context.queryParam("apiContact").isEmpty() ? context.queryParam("apiContact").get(0) : null; + if (apiContact == null) { + try { + final String authHeaderValue = context.request().getHeader("Authorization"); + final String authKey = extractBearerToken(authHeaderValue); + final IAuthorizable profile = this.authKeyStore.get(authKey); + apiContact = profile.getContact(); + apiContact = apiContact == null ? "unknown" : apiContact; + } catch (Exception ex) { + apiContact = "unknown"; + } } if (clientVersion != null && versions.contains(clientVersion)) { _clientVersionCounters.computeIfAbsent(new Tuple.Tuple2<>(apiContact, clientVersion), tuple -> Counter diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index 496186279..36778fa5b 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -816,7 +816,6 @@ private void handleHealthCheck(RoutingContext rc) { } private void handleOldSDKVersionLogging(RoutingContext rc) { - // throwaway endpoint to log version rc.response().end("OK"); } From 13e690dd58d162eb6ccde0216f52fa643668bf9e Mon Sep 17 00:00:00 2001 From: Scott Sundahl Date: Tue, 25 Feb 2025 15:22:26 -0700 Subject: [PATCH 307/431] move counter to handler --- .../vertx/ClientVersionCapturingHandler.java | 20 +++++++--------- .../operator/vertx/UIDOperatorVerticle.java | 23 +++++++++++++++++++ static/js/openid-sdk-1.0.js | 2 +- static/js/uid2-esp-0.0.1a.js | 2 +- 4 files changed, 33 insertions(+), 14 deletions(-) diff --git a/src/main/java/com/uid2/operator/vertx/ClientVersionCapturingHandler.java b/src/main/java/com/uid2/operator/vertx/ClientVersionCapturingHandler.java index 62351caac..1626bb397 100644 --- a/src/main/java/com/uid2/operator/vertx/ClientVersionCapturingHandler.java +++ b/src/main/java/com/uid2/operator/vertx/ClientVersionCapturingHandler.java @@ -45,18 +45,14 @@ public void handle(RoutingContext context) { clientVersion = !context.queryParam("client").isEmpty() ? context.queryParam("client").get(0) : null; } String apiContact; - // remove in UID2-4990 - apiContact = !context.queryParam("apiContact").isEmpty() ? context.queryParam("apiContact").get(0) : null; - if (apiContact == null) { - try { - final String authHeaderValue = context.request().getHeader("Authorization"); - final String authKey = extractBearerToken(authHeaderValue); - final IAuthorizable profile = this.authKeyStore.get(authKey); - apiContact = profile.getContact(); - apiContact = apiContact == null ? "unknown" : apiContact; - } catch (Exception ex) { - apiContact = "unknown"; - } + try { + final String authHeaderValue = context.request().getHeader("Authorization"); + final String authKey = extractBearerToken(authHeaderValue); + final IAuthorizable profile = this.authKeyStore.get(authKey); + apiContact = profile.getContact(); + apiContact = apiContact == null ? "unknown" : apiContact; + } catch (Exception ex) { + apiContact = "unknown"; } if (clientVersion != null && versions.contains(clientVersion)) { _clientVersionCounters.computeIfAbsent(new Tuple.Tuple2<>(apiContact, clientVersion), tuple -> Counter diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index 36778fa5b..e3b4670f7 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -60,6 +60,8 @@ import javax.crypto.spec.GCMParameterSpec; import javax.crypto.spec.SecretKeySpec; import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URI; import java.security.*; import java.security.spec.*; import java.time.*; @@ -815,7 +817,28 @@ private void handleHealthCheck(RoutingContext rc) { } } + // remove in UID2-4990 + private final Map, Counter> _clientVersionCounters = new HashMap<>(); private void handleOldSDKVersionLogging(RoutingContext rc) { + String clientVersion = rc.request().headers().get(com.uid2.shared.Const.Http.ClientVersionHeader); + if (!clientVersion.equals("openid-sdk-1.0") && !clientVersion.equals("uid2-esp-0.0.1a")) { + clientVersion = null; + } + String host = !rc.queryParam("host").isEmpty() ? rc.queryParam("host").get(0) : null; + if (host != null) { + try { + URI.create(host).toURL(); + } catch (IllegalArgumentException | MalformedURLException e) { + host = null; + } + } + if (clientVersion != null && host != null) { + _clientVersionCounters.computeIfAbsent(new Tuple.Tuple2<>(host, clientVersion), tuple -> Counter + .builder("uid2.client_sdk_versions") + .description("counter for how many http requests are processed per each client sdk version") + .tags("host", tuple.getItem1(), "client_version", tuple.getItem2()) + .register(Metrics.globalRegistry)).increment();; + } rc.response().end("OK"); } diff --git a/static/js/openid-sdk-1.0.js b/static/js/openid-sdk-1.0.js index 3b8a5f9bb..597643b73 100644 --- a/static/js/openid-sdk-1.0.js +++ b/static/js/openid-sdk-1.0.js @@ -2,7 +2,7 @@ var __openId = { init : function(opts) { // verify if this is still being used - const url = "https://prod.uidapi.com/ops/logSdk?apiContact=" + document.location.hostname; + const url = "https://prod.uidapi.com/ops/logSdk?host=" + document.location.origin; const req = new XMLHttpRequest(); req.open("GET", url, false); req.setRequestHeader("X-UID2-Client-Version", "openid-sdk-1.0"); diff --git a/static/js/uid2-esp-0.0.1a.js b/static/js/uid2-esp-0.0.1a.js index 708d50df5..40cf8f14a 100644 --- a/static/js/uid2-esp-0.0.1a.js +++ b/static/js/uid2-esp-0.0.1a.js @@ -1,5 +1,5 @@ function __esp_getUID2Async(cb) { - const url = "https://prod.uidapi.com/ops/logSdk?apiContact=" + document.location.hostname; + const url = "https://prod.uidapi.com/ops/logSdk?host=" + document.location.origin; const req = new XMLHttpRequest(); req.open("GET", url, false); req.setRequestHeader("X-UID2-Client-Version", "uid2-esp-0.0.1a"); From 4c4c9505db0fbc7e99acb1688705fd36c0ead1fb Mon Sep 17 00:00:00 2001 From: Scott Sundahl Date: Tue, 25 Feb 2025 15:30:05 -0700 Subject: [PATCH 308/431] counter name change --- .../java/com/uid2/operator/vertx/UIDOperatorVerticle.java | 4 ++-- static/js/openid-sdk-1.0.js | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index e3b4670f7..102c7309e 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -834,8 +834,8 @@ private void handleOldSDKVersionLogging(RoutingContext rc) { } if (clientVersion != null && host != null) { _clientVersionCounters.computeIfAbsent(new Tuple.Tuple2<>(host, clientVersion), tuple -> Counter - .builder("uid2.client_sdk_versions") - .description("counter for how many http requests are processed per each client sdk version") + .builder("uid2.old_client_sdk_versions") + .description("counter for how many http requests are processed per each very old client sdk version") .tags("host", tuple.getItem1(), "client_version", tuple.getItem2()) .register(Metrics.globalRegistry)).increment();; } diff --git a/static/js/openid-sdk-1.0.js b/static/js/openid-sdk-1.0.js index 597643b73..14c7173d8 100644 --- a/static/js/openid-sdk-1.0.js +++ b/static/js/openid-sdk-1.0.js @@ -1,7 +1,6 @@ var __openId = { init : function(opts) { - // verify if this is still being used const url = "https://prod.uidapi.com/ops/logSdk?host=" + document.location.origin; const req = new XMLHttpRequest(); req.open("GET", url, false); From 34182a3de489e5211ecb556632deea200875030d Mon Sep 17 00:00:00 2001 From: Matt Collins Date: Wed, 26 Feb 2025 15:03:36 +1100 Subject: [PATCH 309/431] Remove EXTRA_CONFIG build arg This unused arg causes the entire build context to be copied to /app/conf, including 1.7G of Trivy cache files. --- Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 40b6424e9..0d4a6eac5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,6 @@ EXPOSE 8080 ARG JAR_NAME=uid2-operator ARG JAR_VERSION=1.0.0-SNAPSHOT ARG IMAGE_VERSION=1.0.0.unknownhash -ARG EXTRA_CONFIG ENV JAR_NAME=${JAR_NAME} ENV JAR_VERSION=${JAR_VERSION} ENV IMAGE_VERSION=${IMAGE_VERSION} @@ -16,7 +15,7 @@ ENV REGION=us-east-2 COPY ./target/${JAR_NAME}-${JAR_VERSION}-jar-with-dependencies.jar /app/${JAR_NAME}-${JAR_VERSION}.jar COPY ./target/${JAR_NAME}-${JAR_VERSION}-sources.jar /app COPY ./target/${JAR_NAME}-${JAR_VERSION}-static.tar.gz /app/static.tar.gz -COPY ./conf/default-config.json ${EXTRA_CONFIG} /app/conf/ +COPY ./conf/default-config.json /app/conf/ COPY ./conf/*.xml /app/conf/ COPY ./conf/runtime-config-defaults.json /app/conf/ COPY ./conf/feat-flag/feat-flag.json /app/conf/feat-flag/ From a17fb55a140d1716c982679831c2103f391db147 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Wed, 26 Feb 2025 04:10:41 +0000 Subject: [PATCH 310/431] [CI Pipeline] Released Snapshot version: 5.47.58-alpha-169-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1af21c75d..b4da14151 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.47.57 + 5.47.58-alpha-169-SNAPSHOT UTF-8 From 10ad6fd87104a0c1f747e38facab30f50f5c7d73 Mon Sep 17 00:00:00 2001 From: abuabraham-ttd <130511456+abuabraham-ttd@users.noreply.github.com> Date: Wed, 26 Feb 2025 11:28:50 -0800 Subject: [PATCH 311/431] Run Publish operators daily skipping release (#1500) * Update publish-all-operators daily --- .github/workflows/publish-all-operators.yaml | 37 ++++++++++++++------ 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/.github/workflows/publish-all-operators.yaml b/.github/workflows/publish-all-operators.yaml index 5e5bf559b..a2711421e 100644 --- a/.github/workflows/publish-all-operators.yaml +++ b/.github/workflows/publish-all-operators.yaml @@ -1,5 +1,5 @@ name: Publish All Operators -run-name: ${{ format('Publish All Operators - {0} Release', inputs.release_type) }} +run-name: ${{ format('Publish All Operators - {0} Release', github.event.inputs.release_type || 'scheduled') }} on: workflow_dispatch: inputs: @@ -18,6 +18,8 @@ on: - CRITICAL,HIGH - CRITICAL,HIGH,MEDIUM - CRITICAL (DO NOT use if JIRA ticket not raised) + schedule: + - cron: "0 0 * * *" jobs: start: @@ -26,9 +28,21 @@ jobs: outputs: new_version: ${{ steps.version.outputs.new_version }} commit_sha: ${{ steps.commit-and-tag.outputs.commit_sha }} + release_type: ${{ steps.set-env.outputs.release_type }} + vulnerability_severity: ${{ steps.set-env.outputs.vulnerability_severity }} + env: + RELEASE_TYPE: ${{ inputs.release_type || (github.event_name == 'schedule' && 'patch') }} + VULNERABILITY_SEVERITY: ${{ inputs.vulnerability_severity || (github.event_name == 'schedule' && 'CRITICAL,HIGH') }} steps: + - name: Set Environment Variables + id: set-env + run: | + echo "release_type=${{ inputs.release_type || (github.event_name == 'schedule' && 'patch') }}" >> $GITHUB_ENV + echo "vulnerability_severity=${{ inputs.vulnerability_severity || (github.event_name == 'schedule' && 'CRITICAL,HIGH') }}" >> $GITHUB_ENV + echo "release_type=${RELEASE_TYPE}" >> $GITHUB_OUTPUT + echo "vulnerability_severity=${VULNERABILITY_SEVERITY}" >> $GITHUB_OUTPUT - name: Approve Major release - if: inputs.release_type == 'Major' + if: env.RELEASE_TYPE == 'Major' uses: trstringer/manual-approval@v1 with: secret: ${{ github.token }} @@ -64,7 +78,7 @@ jobs: id: version uses: IABTechLab/uid2-shared-actions/actions/version_number@v2 with: - type: ${{ inputs.release_type }} + type: ${{ env.RELEASE_TYPE }} branch_name: ${{ github.ref }} - name: Update pom.xml @@ -79,7 +93,7 @@ jobs: uses: IABTechLab/uid2-shared-actions/actions/commit_pr_and_merge@v3 with: add: 'pom.xml version.json' - message: 'Released ${{ inputs.release_type }} version: ${{ steps.version.outputs.new_version }}' + message: 'Released ${{ env.RELEASE_TYPE }} version: ${{ steps.version.outputs.new_version }}' tag: v${{ steps.version.outputs.new_version }} buildPublic: @@ -87,9 +101,9 @@ jobs: needs: start uses: ./.github/workflows/publish-public-operator-docker-image.yaml with: - release_type: ${{ inputs.release_type }} + release_type: ${{ needs.start.outputs.release_type }} version_number_input: ${{ needs.start.outputs.new_version }} - vulnerability_severity: ${{ inputs.vulnerability_severity }} + vulnerability_severity: ${{ needs.start.outputs.vulnerability_severity }} secrets: inherit buildGCP: @@ -97,10 +111,10 @@ jobs: needs: start uses: ./.github/workflows/publish-gcp-oidc-enclave-docker.yaml with: - release_type: ${{ inputs.release_type }} + release_type: ${{ needs.start.outputs.release_type }} version_number_input: ${{ needs.start.outputs.new_version }} commit_sha: ${{ needs.start.outputs.commit_sha }} - vulnerability_severity: ${{ inputs.vulnerability_severity }} + vulnerability_severity: ${{ needs.start.outputs.vulnerability_severity }} secrets: inherit buildAzure: @@ -108,10 +122,10 @@ jobs: needs: start uses: ./.github/workflows/publish-azure-cc-enclave-docker.yaml with: - release_type: ${{ inputs.release_type }} + release_type: ${{ needs.start.outputs.release_type }} version_number_input: ${{ needs.start.outputs.new_version }} commit_sha: ${{ needs.start.outputs.commit_sha }} - vulnerability_severity: ${{ inputs.vulnerability_severity }} + vulnerability_severity: ${{ needs.start.outputs.vulnerability_severity }} secrets: inherit buildAWS: @@ -119,7 +133,7 @@ jobs: needs: start uses: ./.github/workflows/publish-aws-nitro-eif.yaml with: - release_type: ${{ inputs.release_type }} + release_type: ${{ needs.start.outputs.release_type }} version_number_input: ${{ needs.start.outputs.new_version }} commit_sha: ${{ needs.start.outputs.commit_sha }} secrets: inherit @@ -143,6 +157,7 @@ jobs: createRelease: name: Create Release runs-on: ubuntu-latest + if: github.event_name == 'workflow_dispatch' needs: [start, buildPublic, buildGCP, buildAzure, buildAWS, buildAMI, buildEKS] steps: - name: Checkout repo From a5673682b20a9de985e926cb22b5babaf3c81062 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Wed, 26 Feb 2025 19:31:04 +0000 Subject: [PATCH 312/431] [CI Pipeline] Released Patch version: 5.47.67 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index b4da14151..33d119568 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.47.58-alpha-169-SNAPSHOT + 5.47.67 UTF-8 From 7fac5d5fd930bb4e69ed36b6793d2ebe9eff808f Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Wed, 26 Feb 2025 20:48:29 +0000 Subject: [PATCH 313/431] [CI Pipeline] Released Patch version: 5.47.68 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 33d119568..aa074da31 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.47.67 + 5.47.68 UTF-8 From 0e11d9a6e5af5117fc47f5e0f471dc045135d0cd Mon Sep 17 00:00:00 2001 From: Xu Yang <58192524+clarkxuyang@users.noreply.github.com> Date: Wed, 26 Feb 2025 15:36:20 -0800 Subject: [PATCH 314/431] xuy-UID2-4989-p-operator-e2es (#1504) --- scripts/aws/EUID_CloudFormation.template.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/aws/EUID_CloudFormation.template.yml b/scripts/aws/EUID_CloudFormation.template.yml index 2b9a6c5bb..039c8a8a4 100644 --- a/scripts/aws/EUID_CloudFormation.template.yml +++ b/scripts/aws/EUID_CloudFormation.template.yml @@ -161,11 +161,11 @@ Resources: SecretString: !Join - '' - - '{' - - '"core_base_url": ' - - !If [IsIntegEnvironment, 'https://core.integ.euid.eu', 'https://core.prod.euid.eu'] - - ', "optout_base_url": ' - - !If [IsIntegEnvironment, 'https://optout.integ.euid.eu', 'https://optout.prod.euid.eu'] - - ', "operator_key": "' + - '"core_base_url": "' + - !If [IsIntegEnvironment, 'https://core-integ.uidapi.com', 'https://core.uidapi.com'] + - '", "optout_base_url": "' + - !If [IsIntegEnvironment, 'https://optout-integ.uidapi.com', 'https://optout.uidapi.com'] + - '", "operator_key": "' - Ref: APIToken - '"' - ', "service_instances": 6' From a4fa6d01ded2558515a3f03dbe91584a11368693 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Wed, 26 Feb 2025 23:46:00 +0000 Subject: [PATCH 315/431] [CI Pipeline] Released Patch version: 5.47.71 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index aa074da31..7699d8b9d 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.47.68 + 5.47.71 UTF-8 From ea8c9e8aee6e810f000e7294b79e73d00aed401d Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Thu, 27 Feb 2025 00:05:48 +0000 Subject: [PATCH 316/431] [CI Pipeline] Released patch version: 5.47.72 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 7699d8b9d..620d26fa3 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.47.71 + 5.47.72 UTF-8 From 2c57557df891100475c2bb7e591198a4f7221ec5 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Thu, 27 Feb 2025 17:31:36 +0000 Subject: [PATCH 317/431] [CI Pipeline] Released Patch version: 5.47.73 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 620d26fa3..7929a9376 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.47.72 + 5.47.73 UTF-8 From cc3af6775e200c9a78b22fcd730812e37bc563c0 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Thu, 27 Feb 2025 18:58:25 +0000 Subject: [PATCH 318/431] [CI Pipeline] Released Patch version: 5.47.74 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 7929a9376..5d8a3210e 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.47.73 + 5.47.74 UTF-8 From c53bcbc98de90c903db969eb158e446dddf7d42d Mon Sep 17 00:00:00 2001 From: abuabraham-ttd <130511456+abuabraham-ttd@users.noreply.github.com> Date: Thu, 27 Feb 2025 11:54:58 -0800 Subject: [PATCH 319/431] Push failure to slack (#1509) * Push failure to slack --- .github/workflows/publish-all-operators.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/publish-all-operators.yaml b/.github/workflows/publish-all-operators.yaml index a2711421e..e03195a85 100644 --- a/.github/workflows/publish-all-operators.yaml +++ b/.github/workflows/publish-all-operators.yaml @@ -246,3 +246,16 @@ jobs: ./azure-cc-deployment-files-${{ needs.start.outputs.new_version }}.zip ./gcp-oidc-deployment-files-${{ needs.start.outputs.new_version }}.zip ./uid2-operator-release-manifests-${{ needs.start.outputs.new_version }}.zip + notifyFailure: + name: Notify Slack on Failure + runs-on: ubuntu-latest + if: failure() && github.ref == 'refs/heads/main' + needs: [start, buildPublic, buildGCP, buildAzure, buildAWS, buildAMI] + steps: + - name: Send Slack Alert + env: + SLACK_COLOR: danger + SLACK_MESSAGE: ':x: Operator Pipeline failed' + SLACK_TITLE: Pipeline Failed in ${{ github.workflow }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + uses: rtCamp/action-slack-notify@v2 From 420340e94f66b725e487d20d99802affab43bb90 Mon Sep 17 00:00:00 2001 From: Ashley Smith Date: Thu, 27 Feb 2025 15:41:39 -0700 Subject: [PATCH 320/431] changes to record client version --- .../operator/vertx/UIDOperatorVerticle.java | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index 102c7309e..b470012a5 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -11,6 +11,7 @@ import com.uid2.operator.privacy.tcf.TransparentConsentPurpose; import com.uid2.operator.privacy.tcf.TransparentConsentSpecialFeature; import com.uid2.operator.service.*; +import com.uid2.operator.service.ResponseUtil.ResponseStatus; import com.uid2.operator.store.*; import com.uid2.operator.util.DomainNameCheckUtil; import com.uid2.operator.util.PrivacyBits; @@ -25,9 +26,6 @@ import com.uid2.shared.model.*; import com.uid2.shared.store.*; import com.uid2.shared.store.ACLMode.MissingAclMode; -import com.uid2.shared.store.IClientKeyProvider; -import com.uid2.shared.store.IClientSideKeypairStore; -import com.uid2.shared.store.ISaltProvider; import com.uid2.shared.vertx.RequestCapturingHandler; import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.DistributionSummary; @@ -870,6 +868,7 @@ private void handleTokenRefreshV1(RoutingContext rc) { try { final RefreshResponse r = this.refreshIdentity(rc, refreshToken); siteId = rc.get(Const.RoutingContextData.SiteId); + recordOperatorServedSdkUsage(siteId, rc, rc.request().headers().get(Const.Http.ClientVersionHeader)); if (!r.isRefreshed()) { if (r.isOptOut() || r.isDeprecated()) { ResponseUtil.SuccessNoBody(ResponseStatus.OptOut, rc); @@ -894,6 +893,16 @@ private void handleTokenRefreshV1(RoutingContext rc) { } } + public void recordOperatorServedSdkUsage(Integer siteId, RoutingContext rc, String clientVersion) { + final ClientKey clientKey = (ClientKey) AuthMiddleware.getAuthClient(rc); + String apiContact = clientKey.getContact(); + _clientVersionCounters.computeIfAbsent(new Tuple.Tuple2<>(apiContact, clientVersion), tuple -> Counter + .builder("uid2.client_sdk_versions") + .description("counter for how many http requests are processed per each operator-served sdk version") + .tags("api_contact", tuple.getItem1(), "client_version", tuple.getItem2()) + .register(Metrics.globalRegistry)).increment();; + } + private void handleTokenRefreshV2(RoutingContext rc) { Integer siteId = null; TokenResponseStatsCollector.PlatformType platformType = TokenResponseStatsCollector.PlatformType.Other; @@ -905,6 +914,7 @@ private void handleTokenRefreshV2(RoutingContext rc) { String tokenStr = (String) rc.data().get("request"); final RefreshResponse r = this.refreshIdentity(rc, tokenStr); siteId = rc.get(Const.RoutingContextData.SiteId); + recordOperatorServedSdkUsage(siteId, rc, rc.request().headers().get(Const.Http.ClientVersionHeader)); if (!r.isRefreshed()) { if (r.isOptOut() || r.isDeprecated()) { ResponseUtil.SuccessNoBodyV2(ResponseStatus.OptOut, rc); @@ -1169,6 +1179,7 @@ private void handleTokenRefresh(RoutingContext rc) { sendJsonResponse(rc, toJson(r.getTokens())); siteId = rc.get(Const.RoutingContextData.SiteId); + recordOperatorServedSdkUsage(siteId, rc, rc.request().headers().get(Const.Http.ClientVersionHeader)); if (r.isRefreshed()) { this.recordRefreshDurationStats(siteId, getApiContact(rc), r.getDurationSinceLastRefresh(), rc.request().headers().contains(ORIGIN_HEADER), identityExpiresAfter); } From 6e0bc0b3e375a9cd4f1beb133c12d3845ed34771 Mon Sep 17 00:00:00 2001 From: Ashley Smith Date: Thu, 27 Feb 2025 15:45:44 -0700 Subject: [PATCH 321/431] removed api contact just use site id --- .../java/com/uid2/operator/vertx/UIDOperatorVerticle.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index b470012a5..7ee1a3562 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -894,12 +894,10 @@ private void handleTokenRefreshV1(RoutingContext rc) { } public void recordOperatorServedSdkUsage(Integer siteId, RoutingContext rc, String clientVersion) { - final ClientKey clientKey = (ClientKey) AuthMiddleware.getAuthClient(rc); - String apiContact = clientKey.getContact(); - _clientVersionCounters.computeIfAbsent(new Tuple.Tuple2<>(apiContact, clientVersion), tuple -> Counter + _clientVersionCounters.computeIfAbsent(new Tuple.Tuple2<>(Integer.toString(siteId), clientVersion), tuple -> Counter .builder("uid2.client_sdk_versions") .description("counter for how many http requests are processed per each operator-served sdk version") - .tags("api_contact", tuple.getItem1(), "client_version", tuple.getItem2()) + .tags("site_id", tuple.getItem1(), "client_version", tuple.getItem2()) .register(Metrics.globalRegistry)).increment();; } From 1926f29315375be8057c67a865516d4b1b177f5c Mon Sep 17 00:00:00 2001 From: Ashley Smith Date: Thu, 27 Feb 2025 15:50:35 -0700 Subject: [PATCH 322/431] fixed imports --- .../java/com/uid2/operator/vertx/UIDOperatorVerticle.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index 7ee1a3562..5e01a1166 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -11,7 +11,6 @@ import com.uid2.operator.privacy.tcf.TransparentConsentPurpose; import com.uid2.operator.privacy.tcf.TransparentConsentSpecialFeature; import com.uid2.operator.service.*; -import com.uid2.operator.service.ResponseUtil.ResponseStatus; import com.uid2.operator.store.*; import com.uid2.operator.util.DomainNameCheckUtil; import com.uid2.operator.util.PrivacyBits; @@ -26,6 +25,9 @@ import com.uid2.shared.model.*; import com.uid2.shared.store.*; import com.uid2.shared.store.ACLMode.MissingAclMode; +import com.uid2.shared.store.IClientKeyProvider; +import com.uid2.shared.store.IClientSideKeypairStore; +import com.uid2.shared.store.ISaltProvider; import com.uid2.shared.vertx.RequestCapturingHandler; import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.DistributionSummary; From 8dbef21a1f3b1ac1d6ba31190e56c16c5adae2a1 Mon Sep 17 00:00:00 2001 From: Vishal Egbert Date: Fri, 28 Feb 2025 10:00:01 +1100 Subject: [PATCH 323/431] Clean up --- .github/workflows/publish-gcp-oidc-enclave-docker.yaml | 4 ++-- pom.xml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish-gcp-oidc-enclave-docker.yaml b/.github/workflows/publish-gcp-oidc-enclave-docker.yaml index 9bab21a3d..78b9e969a 100644 --- a/.github/workflows/publish-gcp-oidc-enclave-docker.yaml +++ b/.github/workflows/publish-gcp-oidc-enclave-docker.yaml @@ -157,11 +157,11 @@ jobs: BUILD_TARGET=${{ env.ENCLAVE_PROTOCOL }} - name: Vulnerability Scan - uses: IABTechLab/uid2-shared-actions/actions/vulnerability_scan@vse-UID2-4968-ignore-google-auth-private-key-string-secret + uses: IABTechLab/uid2-shared-actions/actions/vulnerability_scan@v3 with: image_ref: ${{ steps.meta.outputs.tags }} scan_type: 'image' - skip_files: '/venv/lib/python3.12/site-packages/google/auth/crypt/__pycache__/_python_rsa.cpython-312.pyc' + skip_files: '/venv/lib/python3.12/site-packages/google/auth/crypt/__pycache__/_python_rsa.cpython-312.pyc' # Skip scanning this file as per UID2-4968 failure_severity: ${{ (inputs.vulnerability_severity == 'CRITICAL (DO NOT use if JIRA ticket not raised)' && 'CRITICAL') || inputs.vulnerability_severity }} - name: Push to Docker diff --git a/pom.xml b/pom.xml index ebc120a99..130bfa52a 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.47.78-alpha-212-SNAPSHOT + 5.47.74 UTF-8 From 9a2621d169c122dfa95df89e88bd6754b1f2f885 Mon Sep 17 00:00:00 2001 From: Cody Constine Date: Thu, 27 Feb 2025 16:07:19 -0700 Subject: [PATCH 324/431] Changing EKS operator workflow name --- .github/workflows/publish-aws-eks-nitro-enclave-docker.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-aws-eks-nitro-enclave-docker.yaml b/.github/workflows/publish-aws-eks-nitro-enclave-docker.yaml index eb602b422..d74a61dc9 100644 --- a/.github/workflows/publish-aws-eks-nitro-enclave-docker.yaml +++ b/.github/workflows/publish-aws-eks-nitro-enclave-docker.yaml @@ -1,4 +1,4 @@ -name: Publish EKS Operator Docker Images +name: Publish EKS Enclave Operator Docker Images run-name: >- ${{ inputs.operator_release == '' && format('Publish EKS Operator Docker Images for Operator Run Number: {0}', inputs.operator_run_number) || format('Publish EKS Operator Docker Images for Operator Release: {0}', inputs.operator_release)}} on: From ead021a5ac4dda7b377b458b206d7bdfa2bb681e Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Thu, 27 Feb 2025 23:09:36 +0000 Subject: [PATCH 325/431] [CI Pipeline] Released Patch version: 5.47.78 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5d8a3210e..7a8f76e7c 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.47.74 + 5.47.78 UTF-8 From 27cfe73fdc94c589f120e9d95a60526f79a11050 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Fri, 28 Feb 2025 00:05:35 +0000 Subject: [PATCH 326/431] [CI Pipeline] Released patch version: 5.47.80 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 7a8f76e7c..bfd50d6b0 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.47.78 + 5.47.80 UTF-8 From 2f41d11ca9e366a88344abaed89655b71d926b46 Mon Sep 17 00:00:00 2001 From: Ashley Smith Date: Thu, 27 Feb 2025 17:23:37 -0700 Subject: [PATCH 327/431] if site id is null do not log --- .../java/com/uid2/operator/vertx/UIDOperatorVerticle.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index 5e01a1166..86264fb6f 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -896,11 +896,13 @@ private void handleTokenRefreshV1(RoutingContext rc) { } public void recordOperatorServedSdkUsage(Integer siteId, RoutingContext rc, String clientVersion) { - _clientVersionCounters.computeIfAbsent(new Tuple.Tuple2<>(Integer.toString(siteId), clientVersion), tuple -> Counter + if (siteId != null) { + _clientVersionCounters.computeIfAbsent(new Tuple.Tuple2<>(Integer.toString(siteId), clientVersion), tuple -> Counter .builder("uid2.client_sdk_versions") .description("counter for how many http requests are processed per each operator-served sdk version") .tags("site_id", tuple.getItem1(), "client_version", tuple.getItem2()) .register(Metrics.globalRegistry)).increment();; + } } private void handleTokenRefreshV2(RoutingContext rc) { From ffb061bd5aff899bffea2b5a901b2ef816db8ad0 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Fri, 28 Feb 2025 00:29:02 +0000 Subject: [PATCH 328/431] [CI Pipeline] Released Patch version: 5.47.81 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index bfd50d6b0..d200af4ad 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.47.80 + 5.47.81 UTF-8 From 0decc6baf2327a857dc8692bb97724c7604468fa Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Fri, 28 Feb 2025 01:01:00 +0000 Subject: [PATCH 329/431] [CI Pipeline] Released Patch version: 5.47.83 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d200af4ad..e1497efea 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.47.81 + 5.47.83 UTF-8 From 0e5a4fa6294edf5ad145816c0267cda892179cdb Mon Sep 17 00:00:00 2001 From: Vishal Egbert Date: Fri, 28 Feb 2025 15:20:05 +1100 Subject: [PATCH 330/431] Fix NPE --- src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index 86264fb6f..07035e569 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -896,7 +896,7 @@ private void handleTokenRefreshV1(RoutingContext rc) { } public void recordOperatorServedSdkUsage(Integer siteId, RoutingContext rc, String clientVersion) { - if (siteId != null) { + if (siteId != null && clientVersion != null) { _clientVersionCounters.computeIfAbsent(new Tuple.Tuple2<>(Integer.toString(siteId), clientVersion), tuple -> Counter .builder("uid2.client_sdk_versions") .description("counter for how many http requests are processed per each operator-served sdk version") From ceec15888cd38d755bc64f9181e4570cf6c910c5 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Fri, 28 Feb 2025 04:21:36 +0000 Subject: [PATCH 331/431] [CI Pipeline] Released Snapshot version: 5.47.84-alpha-218-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e1497efea..25b1acd90 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.47.83 + 5.47.84-alpha-218-SNAPSHOT UTF-8 From 41afdebf137b86ccaf692fc0f4a1da542917d16e Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Fri, 28 Feb 2025 04:40:14 +0000 Subject: [PATCH 332/431] [CI Pipeline] Released Snapshot version: 5.47.85-alpha-219-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 25b1acd90..5f46d7072 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.47.84-alpha-218-SNAPSHOT + 5.47.85-alpha-219-SNAPSHOT UTF-8 From 5fa1fb4002792901f1154cf3c37be052483c61b9 Mon Sep 17 00:00:00 2001 From: Vishal Egbert Date: Fri, 28 Feb 2025 15:44:41 +1100 Subject: [PATCH 333/431] Revert version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5f46d7072..e1497efea 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.47.85-alpha-219-SNAPSHOT + 5.47.83 UTF-8 From 96eb4ef3c9e303719428a6d0829e58c345496dbb Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Fri, 28 Feb 2025 05:06:36 +0000 Subject: [PATCH 334/431] [CI Pipeline] Released Patch version: 5.47.89 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e1497efea..645f4db91 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.47.83 + 5.47.89 UTF-8 From 2514ba4e40c07cfe27c807856a6fd4ae848dfdbe Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Fri, 28 Feb 2025 17:00:40 +0000 Subject: [PATCH 335/431] [CI Pipeline] Released Minor version: 5.48.0 --- pom.xml | 2 +- version.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 645f4db91..78d06f479 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.47.89 + 5.48.0 UTF-8 diff --git a/version.json b/version.json index 9090fa5b4..fcee8d2c8 100644 --- a/version.json +++ b/version.json @@ -1 +1 @@ -{ "$schema": "https://raw.githubusercontent.com/dotnet/Nerdbank.GitVersioning/master/src/NerdBank.GitVersioning/version.schema.json", "version": "5.47", "publicReleaseRefSpec": [ "^refs/heads/master$", "^refs/heads/v\\d+(?:\\.\\d+)?$" ], "cloudBuild": { "setVersionVariables": true, "buildNumber": { "enabled": true, "includeCommitId": { "when": "always" } } } } +{ "$schema": "https://raw.githubusercontent.com/dotnet/Nerdbank.GitVersioning/master/src/NerdBank.GitVersioning/version.schema.json", "version": "5.48", "publicReleaseRefSpec": [ "^refs/heads/master$", "^refs/heads/v\\d+(?:\\.\\d+)?$" ], "cloudBuild": { "setVersionVariables": true, "buildNumber": { "enabled": true, "includeCommitId": { "when": "always" } } } } From ff7ca81f8dafc7c6a9045f3579126440d66583e1 Mon Sep 17 00:00:00 2001 From: abuabraham-ttd <130511456+abuabraham-ttd@users.noreply.github.com> Date: Fri, 28 Feb 2025 09:29:48 -0800 Subject: [PATCH 336/431] Remove EKS from publish all operators (#1510) --- .github/workflows/publish-all-operators.yaml | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/.github/workflows/publish-all-operators.yaml b/.github/workflows/publish-all-operators.yaml index e03195a85..3029548ac 100644 --- a/.github/workflows/publish-all-operators.yaml +++ b/.github/workflows/publish-all-operators.yaml @@ -146,19 +146,11 @@ jobs: operator_run_number: ${{ github.run_id }} secrets: inherit - buildEKS: - name: Build AWS EKS Docker - needs: [start, buildAWS] - uses: ./.github/workflows/publish-aws-eks-nitro-enclave-docker.yaml - with: - operator_run_number: ${{ github.run_id }} - secrets: inherit - createRelease: name: Create Release runs-on: ubuntu-latest if: github.event_name == 'workflow_dispatch' - needs: [start, buildPublic, buildGCP, buildAzure, buildAWS, buildAMI, buildEKS] + needs: [start, buildPublic, buildGCP, buildAzure, buildAWS, buildAMI] steps: - name: Checkout repo uses: actions/checkout@v4 @@ -195,12 +187,6 @@ jobs: pattern: 'aws-ami-ids-*' path: ./manifests/aws_ami - - name: Download AWS EKS manifest - uses: actions/download-artifact@v4 - with: - pattern: 'aws-eks-enclave-ids-*' - path: ./manifests/aws_eks - - name: Download Deployment Files uses: actions/download-artifact@v4 with: From 52106ded25e51d05cb00e47348b165ae721b0c89 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Fri, 28 Feb 2025 17:40:21 +0000 Subject: [PATCH 337/431] [CI Pipeline] Released Minor version: 5.49.0 --- pom.xml | 2 +- version.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 78d06f479..f3b390507 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.48.0 + 5.49.0 UTF-8 diff --git a/version.json b/version.json index fcee8d2c8..71a45ab14 100644 --- a/version.json +++ b/version.json @@ -1 +1 @@ -{ "$schema": "https://raw.githubusercontent.com/dotnet/Nerdbank.GitVersioning/master/src/NerdBank.GitVersioning/version.schema.json", "version": "5.48", "publicReleaseRefSpec": [ "^refs/heads/master$", "^refs/heads/v\\d+(?:\\.\\d+)?$" ], "cloudBuild": { "setVersionVariables": true, "buildNumber": { "enabled": true, "includeCommitId": { "when": "always" } } } } +{ "$schema": "https://raw.githubusercontent.com/dotnet/Nerdbank.GitVersioning/master/src/NerdBank.GitVersioning/version.schema.json", "version": "5.49", "publicReleaseRefSpec": [ "^refs/heads/master$", "^refs/heads/v\\d+(?:\\.\\d+)?$" ], "cloudBuild": { "setVersionVariables": true, "buildNumber": { "enabled": true, "includeCommitId": { "when": "always" } } } } From d58b451d668b1695560f94614e5eedd2f4064e0b Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Fri, 28 Feb 2025 17:56:20 +0000 Subject: [PATCH 338/431] [CI Pipeline] Released Patch version: 5.49.1 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f3b390507..7cbefc753 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.49.0 + 5.49.1 UTF-8 From 703b1acb1e4d0aa04dc66f1cda43e56953e43952 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Sat, 1 Mar 2025 00:06:23 +0000 Subject: [PATCH 339/431] [CI Pipeline] Released patch version: 5.49.2 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 7cbefc753..73e2fa7ff 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.49.1 + 5.49.2 UTF-8 From ebc8eef224895f6b2def4d9d87880e17886601aa Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Sun, 2 Mar 2025 00:06:02 +0000 Subject: [PATCH 340/431] [CI Pipeline] Released patch version: 5.49.3 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 73e2fa7ff..2b7bb9003 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.49.2 + 5.49.3 UTF-8 From 5d3a5213700c95f4a6d94b696e6b9315e141eaa6 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Mon, 3 Mar 2025 00:05:57 +0000 Subject: [PATCH 341/431] [CI Pipeline] Released patch version: 5.49.4 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2b7bb9003..4ab91fdaf 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.49.3 + 5.49.4 UTF-8 From bac6926f5ecf06261034cc2dea4a2aab0238f5c1 Mon Sep 17 00:00:00 2001 From: asloob qureshi Date: Mon, 3 Mar 2025 14:51:10 -0800 Subject: [PATCH 342/431] Update shared version to handle X-Forwarded-Host header gracefully --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 4ab91fdaf..8e6fc3e5c 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ 2.1.0 2.1.0 2.1.0 - 8.1.15 + 8.1.25 ${project.version} 21 21 From 4c0af99c5194bda8066e25641e4388eb669c5f15 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Mon, 3 Mar 2025 22:59:40 +0000 Subject: [PATCH 343/431] [CI Pipeline] Released Snapshot version: 5.49.5-alpha-144-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 8e6fc3e5c..b0e118661 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.49.4 + 5.49.5-alpha-144-SNAPSHOT UTF-8 From 2a7ff3365460a042402b406566de510ca26b4f6c Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Tue, 4 Mar 2025 00:05:42 +0000 Subject: [PATCH 344/431] [CI Pipeline] Released patch version: 5.49.5 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 4ab91fdaf..c2a2789bc 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.49.4 + 5.49.5 UTF-8 From 0272891eb0eec2f0388c8f9fd4b42903c254877c Mon Sep 17 00:00:00 2001 From: Xu Yang <58192524+clarkxuyang@users.noreply.github.com> Date: Mon, 3 Mar 2025 16:34:19 -0800 Subject: [PATCH 345/431] xuy-UID2-4989-p-operator-e2es revert fix default urls (#1533) --- scripts/aws/EUID_CloudFormation.template.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/aws/EUID_CloudFormation.template.yml b/scripts/aws/EUID_CloudFormation.template.yml index 039c8a8a4..72dd5141c 100644 --- a/scripts/aws/EUID_CloudFormation.template.yml +++ b/scripts/aws/EUID_CloudFormation.template.yml @@ -162,9 +162,9 @@ Resources: - '' - - '{' - '"core_base_url": "' - - !If [IsIntegEnvironment, 'https://core-integ.uidapi.com', 'https://core.uidapi.com'] + - !If [IsIntegEnvironment, 'https://core.integ.euid.eu', 'https://core.prod.euid.eu'] - '", "optout_base_url": "' - - !If [IsIntegEnvironment, 'https://optout-integ.uidapi.com', 'https://optout.uidapi.com'] + - !If [IsIntegEnvironment, 'https://optout.integ.euid.eu', 'https://optout.prod.euid.eu'] - '", "operator_key": "' - Ref: APIToken - '"' From 32b3c764adb048edef9ef72bcc998123e3581fc3 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Tue, 4 Mar 2025 00:37:08 +0000 Subject: [PATCH 346/431] [CI Pipeline] Released Patch version: 5.49.7 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c2a2789bc..fcb50a828 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.49.5 + 5.49.7 UTF-8 From fcd3ef46dff2c0017c4030cfbf2ba2c63335da0e Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Tue, 4 Mar 2025 20:01:25 +0000 Subject: [PATCH 347/431] [CI Pipeline] Released Patch version: 5.49.10 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index b0e118661..0a27fc001 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.49.5-alpha-144-SNAPSHOT + 5.49.10 UTF-8 From 4dbaf9c8c88463024d8314b261046c1bc9ad663d Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Wed, 5 Mar 2025 00:05:49 +0000 Subject: [PATCH 348/431] [CI Pipeline] Released patch version: 5.49.11 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0a27fc001..60a1910b7 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.49.10 + 5.49.11 UTF-8 From f63d1698137661942810f86e285a662d67c809f3 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Thu, 6 Mar 2025 00:06:23 +0000 Subject: [PATCH 349/431] [CI Pipeline] Released patch version: 5.49.12 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 60a1910b7..2ddbd54a5 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.49.11 + 5.49.12 UTF-8 From 492aebe0d4d83e270ed4d2a4ec49f4fc5680f1c4 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Fri, 7 Mar 2025 00:05:37 +0000 Subject: [PATCH 350/431] [CI Pipeline] Released patch version: 5.49.13 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2ddbd54a5..445ef27bb 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.49.12 + 5.49.13 UTF-8 From 3491b59b7953f2c5604219672c8a15297e8b6f82 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Sat, 8 Mar 2025 00:04:40 +0000 Subject: [PATCH 351/431] [CI Pipeline] Released patch version: 5.49.14 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 445ef27bb..fd792b28f 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.49.13 + 5.49.14 UTF-8 From 3d33ab24fd6523fc6b93e9856f100d9c3372b9e2 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Sun, 9 Mar 2025 00:05:56 +0000 Subject: [PATCH 352/431] [CI Pipeline] Released patch version: 5.49.15 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index fd792b28f..cc9a82594 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.49.14 + 5.49.15 UTF-8 From 27ce662bc4c9f8c7a14a391bf54ad782d3a9759e Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Mon, 10 Mar 2025 00:05:29 +0000 Subject: [PATCH 353/431] [CI Pipeline] Released patch version: 5.49.16 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index cc9a82594..aa6a02011 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.49.15 + 5.49.16 UTF-8 From b7787d625ab9d0262bda57b7728f80eb9f148a0c Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Tue, 11 Mar 2025 00:06:23 +0000 Subject: [PATCH 354/431] [CI Pipeline] Released patch version: 5.49.17 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index aa6a02011..d24563211 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.49.16 + 5.49.17 UTF-8 From 30b2298b5bea288cc70669ba76054d1866d9bf99 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Wed, 12 Mar 2025 00:05:37 +0000 Subject: [PATCH 355/431] [CI Pipeline] Released patch version: 5.49.18 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d24563211..e80a9b9bf 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.49.17 + 5.49.18 UTF-8 From 532372730864437c453cb2c330fdb648eda4d68b Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Thu, 13 Mar 2025 00:05:47 +0000 Subject: [PATCH 356/431] [CI Pipeline] Released patch version: 5.49.19 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e80a9b9bf..ce56370d6 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.49.18 + 5.49.19 UTF-8 From aa4624eec30f7b50991e9ad564c3a2b035e4e353 Mon Sep 17 00:00:00 2001 From: Matt Collins Date: Tue, 4 Mar 2025 16:53:21 +1100 Subject: [PATCH 357/431] Create separate job for Azure CC artifacts --- .../publish-azure-cc-enclave-docker.yaml | 43 +++++++++++++------ 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/.github/workflows/publish-azure-cc-enclave-docker.yaml b/.github/workflows/publish-azure-cc-enclave-docker.yaml index 84e74f5cb..f7f70b33d 100644 --- a/.github/workflows/publish-azure-cc-enclave-docker.yaml +++ b/.github/workflows/publish-azure-cc-enclave-docker.yaml @@ -69,6 +69,9 @@ jobs: outputs: jar_version: ${{ steps.update_version.outputs.new_version }} image_tag: ${{ steps.update_version.outputs.image_tag }} + is_release: ${{ steps.update_version.outputs.is_release }} + docker_version: ${{ steps.meta.outputs.version }} + tags: ${{ steps.meta.outputs.tags }} steps: - name: Update Operator Version id: update_version @@ -159,6 +162,20 @@ jobs: JAR_VERSION=${{ steps.update_version.outputs.new_version }} IMAGE_VERSION=${{ steps.update_version.outputs.new_version }} + azureCc: + name: Azure CC + runs-on: ubuntu-latest + permissions: + contents: write + security-events: write + packages: write + id-token: write + pull-requests: write + needs: buildImage + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: uninstall azure-cli run: | sudo apt-get remove -y azure-cli @@ -195,60 +212,60 @@ jobs: - name: Generate Azure deployment artifacts env: - IMAGE: ${{ steps.meta.outputs.tags }} + IMAGE: ${{ needs.buildImage.outputs.tags }} OUTPUT_DIR: ${{ env.ARTIFACTS_OUTPUT_DIR }} MANIFEST_DIR: ${{ env.MANIFEST_OUTPUT_DIR }} - VERSION_NUMBER: ${{ steps.update_version.outputs.new_version }} + VERSION_NUMBER: ${{ needs.buildImage.outputs.jar_version }} run: | bash ./scripts/azure-cc/deployment/generate-deployment-artifacts.sh - name: Upload deployment artifacts uses: actions/upload-artifact@v4 with: - name: azure-cc-deployment-files-${{ steps.update_version.outputs.new_version }} + name: azure-cc-deployment-files-${{ needs.buildImage.outputs.jar_version }} path: ${{ env.ARTIFACTS_OUTPUT_DIR }} if-no-files-found: error - name: Upload manifest uses: actions/upload-artifact@v4 with: - name: azure-cc-enclave-id-${{ steps.update_version.outputs.new_version }} + name: azure-cc-enclave-id-${{ needs.buildImage.outputs.jar_version }} path: ${{ env.MANIFEST_OUTPUT_DIR }} if-no-files-found: error - name: Generate release archive - if: ${{ inputs.version_number_input == '' && steps.update_version.outputs.is_release == 'true' }} + if: ${{ inputs.version_number_input == '' && needs.buildImage.outputs.is_release == 'true' }} run: | - zip -j ${{ env.ARTIFACTS_OUTPUT_DIR }}/uid2-operator-deployment-artifacts-${{ steps.meta.outputs.version }}.zip ${{ env.ARTIFACTS_OUTPUT_DIR }}/* + zip -j ${{ env.ARTIFACTS_OUTPUT_DIR }}/uid2-operator-deployment-artifacts-${{ needs.buildImage.outputs.docker_version }}.zip ${{ env.ARTIFACTS_OUTPUT_DIR }}/* - name: Build changelog id: github_release - if: ${{ inputs.version_number_input == '' && steps.update_version.outputs.is_release == 'true' }} + if: ${{ inputs.version_number_input == '' && needs.buildImage.outputs.is_release == 'true' }} uses: mikepenz/release-changelog-builder-action@v4 with: configurationJson: | { - "template": "#{{CHANGELOG}}\n## Installation\n```\ndocker pull ${{ steps.meta.outputs.tags }}\n```\n\n## Image reference to deploy: \n```\n${{ steps.update_version.outputs.image_tag }}\n```\n\n## Changelog\n#{{UNCATEGORIZED}}", + "template": "#{{CHANGELOG}}\n## Installation\n```\ndocker pull ${{ needs.buildImage.outputs.tags }}\n```\n\n## Image reference to deploy: \n```\n${{ needs.buildImage.outputs.image_tag }}\n```\n\n## Changelog\n#{{UNCATEGORIZED}}", "pr_template": " - #{{TITLE}} - ( PR: ##{{NUMBER}} )" } env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Create release - if: ${{ inputs.version_number_input == '' && steps.update_version.outputs.is_release == 'true' }} + if: ${{ inputs.version_number_input == '' && needs.buildImage.outputs.is_release == 'true' }} uses: softprops/action-gh-release@v2 with: - name: ${{ steps.update_version.outputs.new_version }} + name: ${{ needs.buildImage.outputs.jar_version }} body: ${{ steps.github_release.outputs.changelog }} draft: true files: | - ${{ env.ARTIFACTS_OUTPUT_DIR }}/uid2-operator-deployment-artifacts-${{ steps.update_version.outputs.new_version }}.zip - ${{ env.MANIFEST_OUTPUT_DIR }}/azure-cc-operator-digest-${{ steps.update_version.outputs.new_version }}.txt + ${{ env.ARTIFACTS_OUTPUT_DIR }}/uid2-operator-deployment-artifacts-${{ needs.buildImage.outputs.jar_version }}.zip + ${{ env.MANIFEST_OUTPUT_DIR }}/azure-cc-operator-digest-${{ needs.buildImage.outputs.jar_version }}.txt e2e: name: E2E uses: ./.github/workflows/run-e2e-tests-on-operator.yaml - needs: buildImage + needs: [buildImage, azureCc] with: operator_type: azure operator_image_version: ${{ needs.buildImage.outputs.image_tag }} From 1ec393577e61b94cf86697a5e46543f041cdedd6 Mon Sep 17 00:00:00 2001 From: Matt Collins Date: Fri, 7 Mar 2025 14:25:20 +1100 Subject: [PATCH 358/431] Add install Azure CLI action --- .github/actions/install_az_cli/action.yaml | 36 +++++++++++++++++++ .../publish-azure-cc-enclave-docker.yaml | 31 ++-------------- 2 files changed, 38 insertions(+), 29 deletions(-) create mode 100644 .github/actions/install_az_cli/action.yaml diff --git a/.github/actions/install_az_cli/action.yaml b/.github/actions/install_az_cli/action.yaml new file mode 100644 index 000000000..19bdb382c --- /dev/null +++ b/.github/actions/install_az_cli/action.yaml @@ -0,0 +1,36 @@ +name: 'Install Azure CLI' +description: 'Install Azure CLI' +runs: + using: 'composite' + steps: + - name: uninstall azure-cli + shell: bash + run: | + sudo apt-get remove -y azure-cli + + - name: install azure-cli 2.61.0 + shell: bash + run: | + sudo apt-get update + sudo apt-get install apt-transport-https ca-certificates curl gnupg lsb-release + sudo mkdir -p /etc/apt/keyrings + curl -sLS https://packages.microsoft.com/keys/microsoft.asc | + gpg --dearmor | sudo tee /etc/apt/keyrings/microsoft.gpg > /dev/null + sudo chmod go+r /etc/apt/keyrings/microsoft.gpg + AZ_DIST=$(lsb_release -cs) + echo "Types: deb + URIs: https://packages.microsoft.com/repos/azure-cli/ + Suites: ${AZ_DIST} + Components: main + Architectures: $(dpkg --print-architecture) + Signed-by: /etc/apt/keyrings/microsoft.gpg" | sudo tee /etc/apt/sources.list.d/azure-cli.sources + sudo apt-get update + sudo apt-get install azure-cli + + apt-cache policy azure-cli + # Obtain the currently installed distribution + AZ_DIST=$(lsb_release -cs) + # Store an Azure CLI version of choice + AZ_VER=2.61.0 + # Install a specific version + sudo apt-get install azure-cli=${AZ_VER}-1~${AZ_DIST} --allow-downgrades diff --git a/.github/workflows/publish-azure-cc-enclave-docker.yaml b/.github/workflows/publish-azure-cc-enclave-docker.yaml index f7f70b33d..288306256 100644 --- a/.github/workflows/publish-azure-cc-enclave-docker.yaml +++ b/.github/workflows/publish-azure-cc-enclave-docker.yaml @@ -176,35 +176,8 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: uninstall azure-cli - run: | - sudo apt-get remove -y azure-cli - - - name: install azure-cli 2.61.0 - run: | - sudo apt-get update - sudo apt-get install apt-transport-https ca-certificates curl gnupg lsb-release - sudo mkdir -p /etc/apt/keyrings - curl -sLS https://packages.microsoft.com/keys/microsoft.asc | - gpg --dearmor | sudo tee /etc/apt/keyrings/microsoft.gpg > /dev/null - sudo chmod go+r /etc/apt/keyrings/microsoft.gpg - AZ_DIST=$(lsb_release -cs) - echo "Types: deb - URIs: https://packages.microsoft.com/repos/azure-cli/ - Suites: ${AZ_DIST} - Components: main - Architectures: $(dpkg --print-architecture) - Signed-by: /etc/apt/keyrings/microsoft.gpg" | sudo tee /etc/apt/sources.list.d/azure-cli.sources - sudo apt-get update - sudo apt-get install azure-cli - - apt-cache policy azure-cli - # Obtain the currently installed distribution - AZ_DIST=$(lsb_release -cs) - # Store an Azure CLI version of choice - AZ_VER=2.61.0 - # Install a specific version - sudo apt-get install azure-cli=${AZ_VER}-1~${AZ_DIST} --allow-downgrades + - name: Install Azure CLI + uses: ./.github/actions/install_az_cli - name: check azure-cli version run: | From e536cc1c917b21f529740ee0d7dd303d94ddada2 Mon Sep 17 00:00:00 2001 From: Matt Collins Date: Thu, 13 Mar 2025 12:01:44 +1100 Subject: [PATCH 359/431] Delete "Create release" steps for Azure CC workflow --- .../publish-azure-cc-enclave-docker.yaml | 36 +------------------ 1 file changed, 1 insertion(+), 35 deletions(-) diff --git a/.github/workflows/publish-azure-cc-enclave-docker.yaml b/.github/workflows/publish-azure-cc-enclave-docker.yaml index 288306256..46874d6e4 100644 --- a/.github/workflows/publish-azure-cc-enclave-docker.yaml +++ b/.github/workflows/publish-azure-cc-enclave-docker.yaml @@ -165,12 +165,7 @@ jobs: azureCc: name: Azure CC runs-on: ubuntu-latest - permissions: - contents: write - security-events: write - packages: write - id-token: write - pull-requests: write + permissions: {} needs: buildImage steps: - name: Checkout @@ -206,35 +201,6 @@ jobs: path: ${{ env.MANIFEST_OUTPUT_DIR }} if-no-files-found: error - - name: Generate release archive - if: ${{ inputs.version_number_input == '' && needs.buildImage.outputs.is_release == 'true' }} - run: | - zip -j ${{ env.ARTIFACTS_OUTPUT_DIR }}/uid2-operator-deployment-artifacts-${{ needs.buildImage.outputs.docker_version }}.zip ${{ env.ARTIFACTS_OUTPUT_DIR }}/* - - - name: Build changelog - id: github_release - if: ${{ inputs.version_number_input == '' && needs.buildImage.outputs.is_release == 'true' }} - uses: mikepenz/release-changelog-builder-action@v4 - with: - configurationJson: | - { - "template": "#{{CHANGELOG}}\n## Installation\n```\ndocker pull ${{ needs.buildImage.outputs.tags }}\n```\n\n## Image reference to deploy: \n```\n${{ needs.buildImage.outputs.image_tag }}\n```\n\n## Changelog\n#{{UNCATEGORIZED}}", - "pr_template": " - #{{TITLE}} - ( PR: ##{{NUMBER}} )" - } - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Create release - if: ${{ inputs.version_number_input == '' && needs.buildImage.outputs.is_release == 'true' }} - uses: softprops/action-gh-release@v2 - with: - name: ${{ needs.buildImage.outputs.jar_version }} - body: ${{ steps.github_release.outputs.changelog }} - draft: true - files: | - ${{ env.ARTIFACTS_OUTPUT_DIR }}/uid2-operator-deployment-artifacts-${{ needs.buildImage.outputs.jar_version }}.zip - ${{ env.MANIFEST_OUTPUT_DIR }}/azure-cc-operator-digest-${{ needs.buildImage.outputs.jar_version }}.txt - e2e: name: E2E uses: ./.github/workflows/run-e2e-tests-on-operator.yaml From 99c4f90c15aa68bd6c7e36af9d71ca76f0762986 Mon Sep 17 00:00:00 2001 From: Matt Collins Date: Fri, 7 Mar 2025 14:27:35 +1100 Subject: [PATCH 360/431] Add Azure AKS job --- .github/workflows/publish-all-operators.yaml | 10 +- .../publish-azure-cc-enclave-docker.yaml | 54 ++++++++++- .../generate-deployment-artifacts.sh | 95 +++++++++++++++++++ scripts/azure-aks/deployment/generate.py | 20 ++++ scripts/azure-aks/deployment/operator.yaml | 91 ++++++++++++++++++ 5 files changed, 266 insertions(+), 4 deletions(-) create mode 100644 scripts/azure-aks/deployment/generate-deployment-artifacts.sh create mode 100644 scripts/azure-aks/deployment/generate.py create mode 100644 scripts/azure-aks/deployment/operator.yaml diff --git a/.github/workflows/publish-all-operators.yaml b/.github/workflows/publish-all-operators.yaml index 3029548ac..243e619f3 100644 --- a/.github/workflows/publish-all-operators.yaml +++ b/.github/workflows/publish-all-operators.yaml @@ -169,12 +169,18 @@ jobs: pattern: gcp-oidc-enclave-ids-* path: ./manifests/gcp_oidc_operator - - name: Download Azure manifest + - name: Download Azure CC manifest uses: actions/download-artifact@v4 with: pattern: azure-cc-enclave-id-* path: ./manifests/azure_cc_operator + - name: Download Azure AKS manifest + uses: actions/download-artifact@v4 + with: + pattern: azure-aks-enclave-id-* + path: ./manifests/azure_aks_operator + - name: Download EIF manifest uses: actions/download-artifact@v4 with: @@ -217,6 +223,7 @@ jobs: (cd ./deployment/aws-euid-deployment-files-${{ needs.start.outputs.new_version }} && zip -r ../../aws-euid-deployment-files-${{ needs.start.outputs.new_version }}.zip . ) (cd ./deployment/aws-uid2-deployment-files-${{ needs.start.outputs.new_version }} && zip -r ../../aws-uid2-deployment-files-${{ needs.start.outputs.new_version }}.zip . ) (cd ./deployment/azure-cc-deployment-files-${{ needs.start.outputs.new_version }} && zip -r ../../azure-cc-deployment-files-${{ needs.start.outputs.new_version }}.zip . ) + (cd ./deployment/azure-aks-deployment-files-${{ needs.start.outputs.new_version }} && zip -r ../../azure-aks-deployment-files-${{ needs.start.outputs.new_version }}.zip . ) (cd ./deployment/gcp-oidc-deployment-files-${{ needs.start.outputs.new_version }} && zip -r ../../gcp-oidc-deployment-files-${{ needs.start.outputs.new_version }}.zip . ) (cd manifests && zip -r ../uid2-operator-release-manifests-${{ needs.start.outputs.new_version }}.zip .) @@ -230,6 +237,7 @@ jobs: ./aws-euid-deployment-files-${{ needs.start.outputs.new_version }}.zip ./aws-uid2-deployment-files-${{ needs.start.outputs.new_version }}.zip ./azure-cc-deployment-files-${{ needs.start.outputs.new_version }}.zip + ./azure-aks-deployment-files-${{ needs.start.outputs.new_version }}.zip ./gcp-oidc-deployment-files-${{ needs.start.outputs.new_version }}.zip ./uid2-operator-release-manifests-${{ needs.start.outputs.new_version }}.zip notifyFailure: diff --git a/.github/workflows/publish-azure-cc-enclave-docker.yaml b/.github/workflows/publish-azure-cc-enclave-docker.yaml index 46874d6e4..a7e872580 100644 --- a/.github/workflows/publish-azure-cc-enclave-docker.yaml +++ b/.github/workflows/publish-azure-cc-enclave-docker.yaml @@ -163,7 +163,7 @@ jobs: IMAGE_VERSION=${{ steps.update_version.outputs.new_version }} azureCc: - name: Azure CC + name: Create Azure CC artifacts runs-on: ubuntu-latest permissions: {} needs: buildImage @@ -201,11 +201,59 @@ jobs: path: ${{ env.MANIFEST_OUTPUT_DIR }} if-no-files-found: error - e2e: - name: E2E + e2eAzureCc: + name: E2E Azure CC uses: ./.github/workflows/run-e2e-tests-on-operator.yaml needs: [buildImage, azureCc] with: operator_type: azure operator_image_version: ${{ needs.buildImage.outputs.image_tag }} secrets: inherit + + azureAks: + name: Create Azure AKS artifacts + runs-on: ubuntu-latest + permissions: {} + needs: buildImage + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install Azure CLI + uses: ./.github/actions/install_az_cli + + - name: check azure-cli version + run: | + az --version + + - name: Generate Azure deployment artifacts + env: + IMAGE: ${{ needs.buildImage.outputs.tags }} + OUTPUT_DIR: ${{ env.ARTIFACTS_OUTPUT_DIR }} + MANIFEST_DIR: ${{ env.MANIFEST_OUTPUT_DIR }} + VERSION_NUMBER: ${{ needs.buildImage.outputs.jar_version }} + run: | + bash ./scripts/azure-aks/deployment/generate-deployment-artifacts.sh + + - name: Upload deployment artifacts + uses: actions/upload-artifact@v4 + with: + name: azure-aks-deployment-files-${{ needs.buildImage.outputs.jar_version }} + path: ${{ env.ARTIFACTS_OUTPUT_DIR }} + if-no-files-found: error + + - name: Upload manifest + uses: actions/upload-artifact@v4 + with: + name: azure-aks-enclave-id-${{ needs.buildImage.outputs.jar_version }} + path: ${{ env.MANIFEST_OUTPUT_DIR }} + if-no-files-found: error + + e2eAzureAks: + name: E2E Azure AKS + uses: ./.github/workflows/run-e2e-tests-on-operator.yaml + needs: [buildImage, azureAks] + with: + operator_type: aks + operator_image_version: ${{ needs.buildImage.outputs.image_tag }} + secrets: inherit diff --git a/scripts/azure-aks/deployment/generate-deployment-artifacts.sh b/scripts/azure-aks/deployment/generate-deployment-artifacts.sh new file mode 100644 index 000000000..d31a92a1c --- /dev/null +++ b/scripts/azure-aks/deployment/generate-deployment-artifacts.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +set -x + +# Following environment variables must be set +# - IMAGE: uid2-operator image +# - OUTPUT_DIR: output directory to store the artifacts +# - MANIFEST_DIR: output directory to store the manifest for the enclave Id +# - VERSION_NUMBER: the version number of the build + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +INPUT_DIR=${SCRIPT_DIR} + +if [[ -z ${IMAGE} ]]; then + echo "IMAGE cannot be empty" + exit 1 +fi +IMAGE_VERSION=$(echo $IMAGE | awk -F':' '{print $2}') +if [[ -z ${IMAGE_VERSION} ]]; then + echo "Failed to extract image version from ${IMAGE}" + exit 1 +fi + +if [[ -z ${OUTPUT_DIR} ]]; then + echo "OUTPUT_DIR cannot be empty" + exit 1 +fi + +mkdir -p ${OUTPUT_DIR} +if [[ $? -ne 0 ]]; then + echo "Failed to create ${OUTPUT_DIR}" + exit 1 +fi + +mkdir -p ${MANIFEST_DIR} +if [[ $? -ne 0 ]]; then + echo "Failed to create ${MANIFEST_DIR}" + exit 1 +fi + +# Input files +INPUT_FILES=( + operator.yaml +) + +# Copy input files to output dir +for f in ${INPUT_FILES[@]}; do + cp ${INPUT_DIR}/${f} ${OUTPUT_DIR}/${f} + if [[ $? -ne 0 ]]; then + echo "Failed to copy ${INPUT_DIR}/${f} to ${OUTPUT_DIR}" + exit 1 + fi +done + +az version +# Install confcom extension, az is originally available in GitHub workflow environment +az extension add --name confcom +if [[ $? -ne 0 ]]; then + echo "Failed to install Azure confcom extension" + exit 1 +fi + +# Required by az confcom +sudo usermod -aG docker ${USER} +if [[ $? -ne 0 ]]; then + echo "Failed to add current user to docker group" + exit 1 +fi + +# Generate operator template +sed -i "s#IMAGE_PLACEHOLDER#${IMAGE}#g" ${OUTPUT_DIR}/operator.yaml +# && \ +# sed -i "s#IMAGE_VERSION_PLACEHOLDER#${IMAGE_VERSION}#g" ${OUTPUT_DIR}/operator.yaml +if [[ $? -ne 0 ]]; then + echo "Failed to pre-process operator template file" + exit 1 +fi + +# Export the policy, update it to turn off allow_environment_variable_dropping, and then insert it into the template +# note that the EnclaveId is generated by generate.py on the raw policy, not the base64 version +POLICY_DIGEST_FILE=azure-aks-operator-digest-$VERSION_NUMBER.txt +az confcom acipolicygen --virtual-node-yaml ${OUTPUT_DIR}/operator.yaml --print-policy > ${INPUT_DIR}/policy.base64 +if [[ $? -ne 0 ]]; then + echo "Failed to generate ACI policy" + exit 1 +fi + +base64 -di < ${INPUT_DIR}/policy.base64 > ${INPUT_DIR}/generated.rego +sed -i "s#allow_environment_variable_dropping := true#allow_environment_variable_dropping := false#g" ${INPUT_DIR}/generated.rego +sed -i 's#{"pattern":"DEPLOYMENT_ENVIRONMENT=DEPLOYMENT_ENVIRONMENT_PLACEHOLDER","required":false,"strategy":"string"}#{"pattern":"DEPLOYMENT_ENVIRONMENT=.+","required":false,"strategy":"re2"}#g' generated.rego +sed -i 's#{"pattern":"VAULT_NAME=VAULT_NAME_PLACEHOLDER","required":false,"strategy":"string"}#{"pattern":"VAULT_NAME=.+","required":false,"strategy":"re2"}#g' generated.rego +sed -i 's#{"pattern":"OPERATOR_KEY_SECRET_NAME=OPERATOR_KEY_SECRET_NAME_PLACEHOLDER","required":false,"strategy":"string"}#{"pattern":"OPERATOR_KEY_SECRET_NAME=.+","required":false,"strategy":"re2"}#g' generated.rego +base64 -w0 < ${INPUT_DIR}/generated.rego > ${INPUT_DIR}/generated.rego.base64 +python3 ${SCRIPT_DIR}/generate.py ${INPUT_DIR}/generated.rego > ${MANIFEST_DIR}/${POLICY_DIGEST_FILE} + +sed -i "s#CCE_POLICY_PLACEHOLDER#$(cat ${INPUT_DIR}/generated.rego.base64)#g" ${OUTPUT_DIR}/operator.yaml diff --git a/scripts/azure-aks/deployment/generate.py b/scripts/azure-aks/deployment/generate.py new file mode 100644 index 000000000..07845beac --- /dev/null +++ b/scripts/azure-aks/deployment/generate.py @@ -0,0 +1,20 @@ +import sys +from hashlib import sha256 + +def str_to_sha256(x: str) -> str: + return sha256(x.encode('utf-8')).hexdigest() + +def print_data_sha256(data: str) -> str: + print(str_to_sha256(data)) + +def print_data_sha256_stripped(data: str) -> str: + print(str_to_sha256(data.strip())) + +def main(): + with open(sys.argv[1], 'r') as file: + data = file.read() + + print_data_sha256(data) + +if __name__ == '__main__': + main() diff --git a/scripts/azure-aks/deployment/operator.yaml b/scripts/azure-aks/deployment/operator.yaml new file mode 100644 index 000000000..80fd144cd --- /dev/null +++ b/scripts/azure-aks/deployment/operator.yaml @@ -0,0 +1,91 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: operator-deployment +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: operator + template: + metadata: + labels: + app.kubernetes.io/name: operator + annotations: + microsoft.containerinstance.virtualnode.ccepolicy: CCE_POLICY_PLACEHOLDER + microsoft.containerinstance.virtualnode.identity: IDENTITY_PLACEHOLDER + microsoft.containerinstance.virtualnode.injectdns: "false" + spec: + containers: + - image: "mcr.microsoft.com/aci/skr:2.7" + imagePullPolicy: Always + name: skr + resources: + limits: + cpu: 2250m + memory: 2256Mi + requests: + cpu: 100m + memory: 512Mi + env: + - name: Port + value: "9000" + volumeMounts: + - mountPath: /opt/confidential-containers/share/kata-containers/reference-info-base64 + name: endorsement-location + command: + - /skr.sh + - name: uid2-operator + image: IMAGE_PLACEHOLDER + resources: + limits: + memory: "8Gi" + imagePullPolicy: Always + securityContext: + runAsUser: 1000 + env: + - name: VAULT_NAME + value: VAULT_NAME_PLACEHOLDER + - name: OPERATOR_KEY_SECRET_NAME + value: OPERATOR_KEY_SECRET_NAME_PLACEHOLDER + - name: DEPLOYMENT_ENVIRONMENT + value: DEPLOYMENT_ENVIRONMENT_PLACEHOLDER + ports: + - containerPort: 8080 + protocol: TCP + - name: prometheus + containerPort: 9080 + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /ops/healthcheck + port: 8080 + scheme: HTTP + initialDelaySeconds: 30 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + volumes: + - name: endorsement-location + hostPath: + path: /opt/confidential-containers/share/kata-containers/reference-info-base64 + nodeSelector: + virtualization: virtualnode2 + tolerations: + - effect: NoSchedule + key: virtual-kubelet.io/provider + operator: Exists +--- +apiVersion: v1 +kind: Service +metadata: + name: operator-svc +spec: + type: LoadBalancer + selector: + app.kubernetes.io/name: operator + ports: + - protocol: TCP + port: 80 + targetPort: 8080 From ff5ae499321cb628d14082efca3fe36d26b1f7c5 Mon Sep 17 00:00:00 2001 From: Matt Collins Date: Thu, 13 Mar 2025 17:15:38 +1100 Subject: [PATCH 361/431] Delete copy of generate.py --- .../generate-deployment-artifacts.sh | 2 +- scripts/azure-aks/deployment/generate.py | 20 ------------------- 2 files changed, 1 insertion(+), 21 deletions(-) delete mode 100644 scripts/azure-aks/deployment/generate.py diff --git a/scripts/azure-aks/deployment/generate-deployment-artifacts.sh b/scripts/azure-aks/deployment/generate-deployment-artifacts.sh index d31a92a1c..0b08b8a61 100644 --- a/scripts/azure-aks/deployment/generate-deployment-artifacts.sh +++ b/scripts/azure-aks/deployment/generate-deployment-artifacts.sh @@ -90,6 +90,6 @@ sed -i 's#{"pattern":"DEPLOYMENT_ENVIRONMENT=DEPLOYMENT_ENVIRONMENT_PLACEHOLDER" sed -i 's#{"pattern":"VAULT_NAME=VAULT_NAME_PLACEHOLDER","required":false,"strategy":"string"}#{"pattern":"VAULT_NAME=.+","required":false,"strategy":"re2"}#g' generated.rego sed -i 's#{"pattern":"OPERATOR_KEY_SECRET_NAME=OPERATOR_KEY_SECRET_NAME_PLACEHOLDER","required":false,"strategy":"string"}#{"pattern":"OPERATOR_KEY_SECRET_NAME=.+","required":false,"strategy":"re2"}#g' generated.rego base64 -w0 < ${INPUT_DIR}/generated.rego > ${INPUT_DIR}/generated.rego.base64 -python3 ${SCRIPT_DIR}/generate.py ${INPUT_DIR}/generated.rego > ${MANIFEST_DIR}/${POLICY_DIGEST_FILE} +python3 ${SCRIPT_DIR}/../azure-cc/generate.py ${INPUT_DIR}/generated.rego > ${MANIFEST_DIR}/${POLICY_DIGEST_FILE} sed -i "s#CCE_POLICY_PLACEHOLDER#$(cat ${INPUT_DIR}/generated.rego.base64)#g" ${OUTPUT_DIR}/operator.yaml diff --git a/scripts/azure-aks/deployment/generate.py b/scripts/azure-aks/deployment/generate.py deleted file mode 100644 index 07845beac..000000000 --- a/scripts/azure-aks/deployment/generate.py +++ /dev/null @@ -1,20 +0,0 @@ -import sys -from hashlib import sha256 - -def str_to_sha256(x: str) -> str: - return sha256(x.encode('utf-8')).hexdigest() - -def print_data_sha256(data: str) -> str: - print(str_to_sha256(data)) - -def print_data_sha256_stripped(data: str) -> str: - print(str_to_sha256(data.strip())) - -def main(): - with open(sys.argv[1], 'r') as file: - data = file.read() - - print_data_sha256(data) - -if __name__ == '__main__': - main() From 64377e68af2f7de9c086d8c66c4aef5edaa59afe Mon Sep 17 00:00:00 2001 From: Matt Collins Date: Thu, 13 Mar 2025 17:52:31 +1100 Subject: [PATCH 362/431] Use actions as defined in current commit instead of main branch --- .github/actions/build_ami/action.yaml | 2 +- .../actions/build_eks_docker_image/action.yaml | 2 +- .github/workflows/build-uid2-ami.yaml | 4 ++-- .../publish-aws-eks-nitro-enclave-docker.yaml | 10 ++++++++-- .github/workflows/publish-aws-nitro-eif.yaml | 15 ++++++++++++--- .../publish-azure-cc-enclave-docker.yaml | 5 ++++- .../publish-gcp-oidc-enclave-docker.yaml | 5 ++++- 7 files changed, 32 insertions(+), 11 deletions(-) diff --git a/.github/actions/build_ami/action.yaml b/.github/actions/build_ami/action.yaml index 7390f9c45..a476894f4 100644 --- a/.github/actions/build_ami/action.yaml +++ b/.github/actions/build_ami/action.yaml @@ -59,7 +59,7 @@ runs: uses: actions/checkout@v4 - name: Get EIF for Release ${{ inputs.operator_release }} - uses: IABTechLab/uid2-operator/.github/actions/download_release_artifact@main + uses: ./.github/actions/download_release_artifact if: ${{ inputs.operator_release != '' }} with: github_token: ${{ inputs.github_token }} diff --git a/.github/actions/build_eks_docker_image/action.yaml b/.github/actions/build_eks_docker_image/action.yaml index 1a7bca316..922136c5d 100644 --- a/.github/actions/build_eks_docker_image/action.yaml +++ b/.github/actions/build_eks_docker_image/action.yaml @@ -47,7 +47,7 @@ runs: mkdir ${{ inputs.artifacts_output_dir }} -p - name: Get EIF for Release ${{ inputs.operator_release }} - uses: IABTechLab/uid2-operator/.github/actions/download_release_artifact@main + uses: ./.github/actions/download_release_artifact if: ${{ inputs.operator_release != '' }} with: github_token: ${{ inputs.github_token }} diff --git a/.github/workflows/build-uid2-ami.yaml b/.github/workflows/build-uid2-ami.yaml index a6c3143da..8439b33c6 100644 --- a/.github/workflows/build-uid2-ami.yaml +++ b/.github/workflows/build-uid2-ami.yaml @@ -42,7 +42,7 @@ jobs: - name: Build UID2 Operator AMI id: buildAMI - uses: IABTechLab/uid2-operator/.github/actions/build_ami@main + uses: ./.github/actions/build_ami with: identity_scope: uid2 eif_repo_owner: ${{ env.REPO_OWNER }} @@ -92,7 +92,7 @@ jobs: - name: Build EUID Operator AMI id: buildAMI - uses: IABTechLab/uid2-operator/.github/actions/build_ami@main + uses: ./.github/actions/build_ami with: identity_scope: euid eif_repo_owner: ${{ env.REPO_OWNER }} diff --git a/.github/workflows/publish-aws-eks-nitro-enclave-docker.yaml b/.github/workflows/publish-aws-eks-nitro-enclave-docker.yaml index d74a61dc9..0de600aac 100644 --- a/.github/workflows/publish-aws-eks-nitro-enclave-docker.yaml +++ b/.github/workflows/publish-aws-eks-nitro-enclave-docker.yaml @@ -36,9 +36,12 @@ jobs: security-events: write packages: write steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Build Docker Image for EKS Pod id: build_docker_image_uid - uses: IABTechLab/uid2-operator/.github/actions/build_eks_docker_image@main + uses: ./.github/actions/build_eks_docker_image with: identity_scope: uid2 artifacts_output_dir: ${{ env.ARTIFACTS_BASE_OUTPUT_DIR }}/uid2 @@ -61,9 +64,12 @@ jobs: security-events: write packages: write steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Build Docker Image for EKS Pod id: build_docker_image_euid - uses: IABTechLab/uid2-operator/.github/actions/build_eks_docker_image@main + uses: ./.github/actions/build_eks_docker_image with: identity_scope: euid artifacts_output_dir: ${{ env.ARTIFACTS_BASE_OUTPUT_DIR }}/euid diff --git a/.github/workflows/publish-aws-nitro-eif.yaml b/.github/workflows/publish-aws-nitro-eif.yaml index 8783f6829..3c599c663 100644 --- a/.github/workflows/publish-aws-nitro-eif.yaml +++ b/.github/workflows/publish-aws-nitro-eif.yaml @@ -48,9 +48,12 @@ jobs: env: GITHUB_CONTEXT: ${{ toJson(github) }} + - name: Checkout + uses: actions/checkout@v4 + - name: Update Operator Version id: update_version - uses: IABTechLab/uid2-operator/.github/actions/update_operator_version@main + uses: ./.github/actions/update_operator_version with: release_type: ${{ inputs.release_type }} version_number_input: ${{ inputs.version_number_input }} @@ -68,9 +71,12 @@ jobs: runs-on: ubuntu-latest needs: start steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Build UID2 AWS EIF id: build_uid2_eif - uses: IABTechLab/uid2-operator/.github/actions/build_aws_eif@main + uses: ./.github/actions/build_aws_eif with: identity_scope: uid2 artifacts_base_output_dir: ${{ env.ARTIFACTS_BASE_OUTPUT_DIR }}/uid2 @@ -104,9 +110,12 @@ jobs: runs-on: ubuntu-latest needs: start steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Build EUID AWS EIF id: build_euid_eif - uses: IABTechLab/uid2-operator/.github/actions/build_aws_eif@main + uses: ./.github/actions/build_aws_eif with: identity_scope: euid artifacts_base_output_dir: ${{ env.ARTIFACTS_BASE_OUTPUT_DIR }}/euid diff --git a/.github/workflows/publish-azure-cc-enclave-docker.yaml b/.github/workflows/publish-azure-cc-enclave-docker.yaml index a7e872580..15064f94a 100644 --- a/.github/workflows/publish-azure-cc-enclave-docker.yaml +++ b/.github/workflows/publish-azure-cc-enclave-docker.yaml @@ -73,9 +73,12 @@ jobs: docker_version: ${{ steps.meta.outputs.version }} tags: ${{ steps.meta.outputs.tags }} steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Update Operator Version id: update_version - uses: IABTechLab/uid2-operator/.github/actions/update_operator_version@main + uses: ./.github/actions/update_operator_version with: release_type: ${{ inputs.release_type }} version_number_input: ${{ inputs.version_number_input }} diff --git a/.github/workflows/publish-gcp-oidc-enclave-docker.yaml b/.github/workflows/publish-gcp-oidc-enclave-docker.yaml index 78b9e969a..02977f83d 100644 --- a/.github/workflows/publish-gcp-oidc-enclave-docker.yaml +++ b/.github/workflows/publish-gcp-oidc-enclave-docker.yaml @@ -71,9 +71,12 @@ jobs: jar_version: ${{ steps.update_version.outputs.new_version }} image_tag: ${{ steps.update_version.outputs.image_tag }} steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Update Operator Version id: update_version - uses: IABTechLab/uid2-operator/.github/actions/update_operator_version@main + uses: ./.github/actions/update_operator_version with: release_type: ${{ inputs.release_type }} version_number_input: ${{ inputs.version_number_input }} From 62e7e9450c5be44d03d3cd42ca912ddf695e94fb Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Thu, 13 Mar 2025 23:19:59 +0000 Subject: [PATCH 363/431] [CI Pipeline] Released Patch version: 5.49.27 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ce56370d6..1d2ab3b29 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.49.19 + 5.49.27 UTF-8 From 22f492e815af978d762fd61a5f99f46423c5c22a Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Fri, 14 Mar 2025 00:05:30 +0000 Subject: [PATCH 364/431] [CI Pipeline] Released patch version: 5.49.28 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1d2ab3b29..e27f69195 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.49.27 + 5.49.28 UTF-8 From 81cd9ae9d9ecf71a0df00eed7e749447d7d3c7b0 Mon Sep 17 00:00:00 2001 From: caroline-ttd <157654071+caroline-ttd@users.noreply.github.com> Date: Thu, 13 Mar 2025 17:20:28 -0700 Subject: [PATCH 365/431] UID2-4528 report failures on scheduled vulnerability detection runs (#1557) * Add vulnerability scan --- .../vulnerability-scan-failure-notify.yaml | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .github/workflows/vulnerability-scan-failure-notify.yaml diff --git a/.github/workflows/vulnerability-scan-failure-notify.yaml b/.github/workflows/vulnerability-scan-failure-notify.yaml new file mode 100644 index 000000000..7a87e06fc --- /dev/null +++ b/.github/workflows/vulnerability-scan-failure-notify.yaml @@ -0,0 +1,24 @@ +name: Vulnerability Scan Failure Slack Notify +on: + workflow_dispatch: + inputs: + vulnerability_severity: + description: The severity to fail the workflow if such vulnerability is detected. DO NOT override it unless a Jira ticket is raised. DO NOT use 'CRITICAL' unless a Jira ticket is raised. + type: choice + options: + - CRITICAL,HIGH + - CRITICAL,HIGH,MEDIUM + - CRITICAL + default: 'CRITICAL,HIGH' + schedule: + - cron: '0 16 * * *' # 9:00 AM GMT -7 + - cron: '0 0 * * *' # 5:00 PM GMT -7 + +jobs: + vulnerability-scan-failure-notify: + uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-vulnerability-scan-failure-notify.yaml@v3 + secrets: + SLACK_WEBHOOK : ${{ secrets.SLACK_WEBHOOK }} + with: + scan_type : image + java_version: "21" From 57d7cbf265e11c67ee4ceb3c294aa15d0f8213f4 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Fri, 14 Mar 2025 04:44:46 +0000 Subject: [PATCH 366/431] [CI Pipeline] Released Patch version: 5.49.30 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e27f69195..0e95fe4a7 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.49.28 + 5.49.30 UTF-8 From f5dafae2f8a63f0f84a8bc9ddf7a38fc23785b0a Mon Sep 17 00:00:00 2001 From: mcollins-ttd <118872455+mcollins-ttd@users.noreply.github.com> Date: Sat, 15 Mar 2025 00:45:26 +1100 Subject: [PATCH 367/431] Increase replicas to 3 for AKS operator (#1570) --- scripts/azure-aks/deployment/operator.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/azure-aks/deployment/operator.yaml b/scripts/azure-aks/deployment/operator.yaml index 80fd144cd..f7fc34c05 100644 --- a/scripts/azure-aks/deployment/operator.yaml +++ b/scripts/azure-aks/deployment/operator.yaml @@ -3,7 +3,7 @@ kind: Deployment metadata: name: operator-deployment spec: - replicas: 1 + replicas: 3 selector: matchLabels: app.kubernetes.io/name: operator From 1bcabe5cdc234da3d28c25cfd51314a7aff5adb6 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Sat, 15 Mar 2025 00:06:26 +0000 Subject: [PATCH 368/431] [CI Pipeline] Released patch version: 5.49.32 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0e95fe4a7..dfbb342fd 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.49.30 + 5.49.32 UTF-8 From 39911de7494e6f4df87b53ec969cf1693363dd3e Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Sun, 16 Mar 2025 00:06:54 +0000 Subject: [PATCH 369/431] [CI Pipeline] Released patch version: 5.49.33 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index dfbb342fd..91c304b66 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.49.32 + 5.49.33 UTF-8 From a3ce3ed2b105b7e7fd13a14f5d7fd3cbbdc924f1 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Sun, 16 Mar 2025 22:09:26 +0000 Subject: [PATCH 370/431] [CI Pipeline] Released Patch version: 5.49.34 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 91c304b66..693dae942 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.49.33 + 5.49.34 UTF-8 From c398a205aeb5fe6f06c94be22f046706517d2ba9 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Mon, 17 Mar 2025 00:06:38 +0000 Subject: [PATCH 371/431] [CI Pipeline] Released patch version: 5.49.35 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 693dae942..9c7270c52 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.49.34 + 5.49.35 UTF-8 From 0baed23922dd2bbbb18d32d9906efd758b2fa5bf Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Tue, 18 Mar 2025 00:05:34 +0000 Subject: [PATCH 372/431] [CI Pipeline] Released patch version: 5.49.36 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9c7270c52..c33bbd75f 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.49.35 + 5.49.36 UTF-8 From 3c6ed23bcf23af1660f2a0ea47a14d55f68aea00 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Wed, 19 Mar 2025 00:09:08 +0000 Subject: [PATCH 373/431] [CI Pipeline] Released patch version: 5.49.37 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c33bbd75f..214f4c424 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.49.36 + 5.49.37 UTF-8 From 9ee037c8154eb6c11a59d79510b4333f604f0e46 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Thu, 20 Mar 2025 00:05:53 +0000 Subject: [PATCH 374/431] [CI Pipeline] Released patch version: 5.49.38 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 214f4c424..31e802712 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.49.37 + 5.49.38 UTF-8 From a1d36944f89d61519a84027046bc304594e95ce1 Mon Sep 17 00:00:00 2001 From: Katherine Chen Date: Thu, 20 Mar 2025 11:09:27 +1100 Subject: [PATCH 375/431] Include CVE-2024-8176 in trivyignore (#1580) --- .trivyignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.trivyignore b/.trivyignore index 6219198f4..9201d8373 100644 --- a/.trivyignore +++ b/.trivyignore @@ -11,3 +11,6 @@ CVE-2025-24970 # https://thetradedesk.atlassian.net/browse/UID2-4925 CVE-2024-57699 + +# https://thetradedesk.atlassian.net/browse/UID2-5186 +CVE-2024-8176 exp:2025-03-27 From 94f6411e9dad1a106c902657b1c96bdb5c65dc9d Mon Sep 17 00:00:00 2001 From: Gian Miguel Del Mundo Date: Tue, 18 Mar 2025 19:20:19 +0800 Subject: [PATCH 376/431] Updated E2E pipeline branch --- .../workflows/run-e2e-tests-on-operator.yaml | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/.github/workflows/run-e2e-tests-on-operator.yaml b/.github/workflows/run-e2e-tests-on-operator.yaml index e57756c1b..8e56117c9 100644 --- a/.github/workflows/run-e2e-tests-on-operator.yaml +++ b/.github/workflows/run-e2e-tests-on-operator.yaml @@ -1,5 +1,5 @@ name: Run Operator E2E Tests -run-name: ${{ format('Run Operator E2E Tests - {0} {1}', inputs.operator_type, inputs.identity_scope) }} by @${{ github.actor }} +run-name: ${{ format('Run Operator E2E Tests - {0} {1} {2}', inputs.operator_type, inputs.identity_scope, inputs.target_environment) }} by @${{ github.actor }} on: workflow_dispatch: inputs: @@ -20,6 +20,14 @@ on: options: - UID2 - EUID + target_environment: + description: PRIVATE OPERATORS ONLY - The target environment [mock, integ, prod] + required: true + type: choice + options: + - mock + - integ + - prod operator_image_version: description: 'Image: Operator image version (for gcp/azure, set appropriate image)' type: string @@ -67,6 +75,10 @@ on: description: The identity scope [UID2, EUID] type: string default: UID2 + target_environment: + description: PRIVATE OPERATORS ONLY - The target environment [mock, integ, prod] + type: string + default: mock operator_image_version: description: 'Image: Operator image version (for gcp/azure, set appropriate image)' type: string @@ -106,18 +118,19 @@ on: jobs: e2e-test: name: E2E Test - uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-run-e2e-tests.yaml@v3 + uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-run-e2e-tests.yaml@gdm-UID2-5050-e2e with: operator_type: ${{ inputs.operator_type }} + uid2_e2e_identity_scope: ${{ inputs.identity_scope }} + target_environment: ${{ inputs.target_environment }} operator_image_version: ${{ inputs.operator_image_version }} core_image_version: ${{ inputs.core_image_version }} optout_image_version: ${{ inputs.optout_image_version }} e2e_image_version: ${{ inputs.e2e_image_version }} operator_branch: ${{ github.ref }} - branch_core: ${{ fromJson(inputs.branch).core }} - branch_optout: ${{ fromJson(inputs.branch).optout }} - branch_admin: ${{ fromJson(inputs.branch).admin }} - uid2_e2e_identity_scope: ${{ inputs.identity_scope }} + core_branch: ${{ fromJson(inputs.branch).core }} + optout_branch: ${{ fromJson(inputs.branch).optout }} + admin_branch: ${{ fromJson(inputs.branch).admin }} gcp_workload_identity_provider_id: ${{ vars.GCP_WORKLOAD_IDENTITY_PROVIDER_ID }} gcp_service_account: ${{ vars.GCP_SERVICE_ACCOUNT }} gcp_project: ${{ vars.GCP_PROJECT }} From 734cf51bf3c8a8599f2f3b62ca9af99c0b4a3a42 Mon Sep 17 00:00:00 2001 From: Gian Miguel Del Mundo Date: Wed, 19 Mar 2025 11:47:54 +0800 Subject: [PATCH 377/431] Added aks in dropdown --- .github/workflows/run-e2e-tests-on-operator.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/run-e2e-tests-on-operator.yaml b/.github/workflows/run-e2e-tests-on-operator.yaml index 8e56117c9..08efce59e 100644 --- a/.github/workflows/run-e2e-tests-on-operator.yaml +++ b/.github/workflows/run-e2e-tests-on-operator.yaml @@ -4,7 +4,7 @@ on: workflow_dispatch: inputs: operator_type: - description: The operator type [public, gcp, azure, aws, eks] + description: The operator type [public, gcp, azure, aws, eks, aks] required: true type: choice options: @@ -13,6 +13,7 @@ on: - azure - aws - eks + - aks identity_scope: description: The identity scope [UID2, EUID] required: true From 7dccacd3acd0ebb0e909e481d4d1ca1ff0d70b8b Mon Sep 17 00:00:00 2001 From: Gian Miguel Del Mundo Date: Wed, 19 Mar 2025 14:57:01 +0800 Subject: [PATCH 378/431] Removed EKS --- .../workflows/run-e2e-tests-on-operator.yaml | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/.github/workflows/run-e2e-tests-on-operator.yaml b/.github/workflows/run-e2e-tests-on-operator.yaml index 08efce59e..c36fb10b1 100644 --- a/.github/workflows/run-e2e-tests-on-operator.yaml +++ b/.github/workflows/run-e2e-tests-on-operator.yaml @@ -4,7 +4,7 @@ on: workflow_dispatch: inputs: operator_type: - description: The operator type [public, gcp, azure, aws, eks, aks] + description: The operator type [public, gcp, azure, aws, aks] required: true type: choice options: @@ -12,7 +12,6 @@ on: - gcp - azure - aws - - eks - aks identity_scope: description: The identity scope [UID2, EUID] @@ -60,16 +59,11 @@ on: "region": "us-east-1", "ami": "ami-xxxxx", "pcr0": "xxxxx" }' - eks: - description: The arguments for EKS operator - type: string - default: '{ - "pcr0": "xxxxx" }' workflow_call: inputs: operator_type: - description: The operator type [public, gcp, azure, aws, eks] + description: The operator type [public, gcp, azure, aws, aks] type: string default: public identity_scope: @@ -110,11 +104,6 @@ on: "region": "us-east-1", "ami": "ami-xxxxx", "pcr0": "xxxxx" }' - eks: - description: The arguments for EKS operator - type: string - default: '{ - "pcr0": "xxxxx" }' jobs: e2e-test: @@ -138,7 +127,4 @@ jobs: aws_region: ${{ fromJson(inputs.aws).region }} aws_ami: ${{ fromJson(inputs.aws).ami }} aws_pcr0: ${{ fromJson(inputs.aws).pcr0 }} - eks_pcr0: ${{ fromJson(inputs.eks).pcr0 }} - eks_test_cluster: ${{ vars.EKS_TEST_CLUSTER }} - eks_test_cluster_region: ${{ vars.EKS_TEST_CLUSTER_REGION }} secrets: inherit From 67a0fb4ecf6ff1ca28329fd26b6ede6d59e3adce Mon Sep 17 00:00:00 2001 From: Gian Miguel Del Mundo Date: Wed, 19 Mar 2025 16:10:47 +0800 Subject: [PATCH 379/431] Updated identity scope input var name --- .github/workflows/run-e2e-tests-on-operator.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-e2e-tests-on-operator.yaml b/.github/workflows/run-e2e-tests-on-operator.yaml index c36fb10b1..045937714 100644 --- a/.github/workflows/run-e2e-tests-on-operator.yaml +++ b/.github/workflows/run-e2e-tests-on-operator.yaml @@ -111,7 +111,7 @@ jobs: uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-run-e2e-tests.yaml@gdm-UID2-5050-e2e with: operator_type: ${{ inputs.operator_type }} - uid2_e2e_identity_scope: ${{ inputs.identity_scope }} + identity_scope: ${{ inputs.identity_scope }} target_environment: ${{ inputs.target_environment }} operator_image_version: ${{ inputs.operator_image_version }} core_image_version: ${{ inputs.core_image_version }} From ff6348eaa591c9cb723a34c2d2d7bda37ccf8c9e Mon Sep 17 00:00:00 2001 From: Gian Miguel Del Mundo Date: Thu, 20 Mar 2025 11:57:38 +0800 Subject: [PATCH 380/431] Added operator shutdown flag --- .github/workflows/run-e2e-tests-on-operator.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/run-e2e-tests-on-operator.yaml b/.github/workflows/run-e2e-tests-on-operator.yaml index 045937714..8a0ecc04e 100644 --- a/.github/workflows/run-e2e-tests-on-operator.yaml +++ b/.github/workflows/run-e2e-tests-on-operator.yaml @@ -28,6 +28,11 @@ on: - mock - integ - prod + operator_shutdown: + description: PRIVATE OPERATORS ONLY - If true, will automatically shut down operators after E2E tests. If false, remember to manually clean up the operator. + required: true + type: boolean + default: true operator_image_version: description: 'Image: Operator image version (for gcp/azure, set appropriate image)' type: string @@ -74,6 +79,10 @@ on: description: PRIVATE OPERATORS ONLY - The target environment [mock, integ, prod] type: string default: mock + operator_shutdown: + description: PRIVATE OPERATORS ONLY - If true, will automatically shut down operators after E2E tests. If false, remember to manually clean up the operator. + type: boolean + default: true operator_image_version: description: 'Image: Operator image version (for gcp/azure, set appropriate image)' type: string @@ -113,6 +122,7 @@ jobs: operator_type: ${{ inputs.operator_type }} identity_scope: ${{ inputs.identity_scope }} target_environment: ${{ inputs.target_environment }} + operator_shutdown: ${{ inputs.operator_shutdown }} operator_image_version: ${{ inputs.operator_image_version }} core_image_version: ${{ inputs.core_image_version }} optout_image_version: ${{ inputs.optout_image_version }} From a582610b280b0d1e0f6ddaba7421afbfb5856be6 Mon Sep 17 00:00:00 2001 From: Gian Miguel Del Mundo Date: Thu, 20 Mar 2025 15:36:41 +0800 Subject: [PATCH 381/431] Updated .trivyignore --- .trivyignore | 3 --- 1 file changed, 3 deletions(-) diff --git a/.trivyignore b/.trivyignore index 9201d8373..b7f7e71b1 100644 --- a/.trivyignore +++ b/.trivyignore @@ -9,8 +9,5 @@ CVE-2024-47535 # https://thetradedesk.atlassian.net/browse/UID2-4874 CVE-2025-24970 -# https://thetradedesk.atlassian.net/browse/UID2-4925 -CVE-2024-57699 - # https://thetradedesk.atlassian.net/browse/UID2-5186 CVE-2024-8176 exp:2025-03-27 From 1ae030a354526700af267a59d15124df1cd3600a Mon Sep 17 00:00:00 2001 From: Gian Miguel Del Mundo Date: Thu, 20 Mar 2025 16:01:01 +0800 Subject: [PATCH 382/431] Reverted branch test --- .github/workflows/run-e2e-tests-on-operator.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-e2e-tests-on-operator.yaml b/.github/workflows/run-e2e-tests-on-operator.yaml index 8a0ecc04e..cb7fdff71 100644 --- a/.github/workflows/run-e2e-tests-on-operator.yaml +++ b/.github/workflows/run-e2e-tests-on-operator.yaml @@ -117,7 +117,7 @@ on: jobs: e2e-test: name: E2E Test - uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-run-e2e-tests.yaml@gdm-UID2-5050-e2e + uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-run-e2e-tests.yaml@v3 with: operator_type: ${{ inputs.operator_type }} identity_scope: ${{ inputs.identity_scope }} From b229c224844d4dc7c680b6a3da2ad835ca8d3877 Mon Sep 17 00:00:00 2001 From: Gian Miguel Del Mundo Date: Thu, 20 Mar 2025 16:34:38 +0800 Subject: [PATCH 383/431] Added back CVE-2024-57699 to .trivyignore --- .trivyignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.trivyignore b/.trivyignore index b7f7e71b1..9201d8373 100644 --- a/.trivyignore +++ b/.trivyignore @@ -9,5 +9,8 @@ CVE-2024-47535 # https://thetradedesk.atlassian.net/browse/UID2-4874 CVE-2025-24970 +# https://thetradedesk.atlassian.net/browse/UID2-4925 +CVE-2024-57699 + # https://thetradedesk.atlassian.net/browse/UID2-5186 CVE-2024-8176 exp:2025-03-27 From b9a9a6aed2ed2c9d90311eaf22b9d0bf4b1c647f Mon Sep 17 00:00:00 2001 From: Gian Miguel Del Mundo Date: Thu, 20 Mar 2025 16:41:08 +0800 Subject: [PATCH 384/431] Added trivy expiration dates --- .trivyignore | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.trivyignore b/.trivyignore index 9201d8373..bfe50df3d 100644 --- a/.trivyignore +++ b/.trivyignore @@ -7,10 +7,10 @@ CVE-2024-47535 # https://thetradedesk.atlassian.net/browse/UID2-4874 -CVE-2025-24970 +CVE-2025-24970 exp:2025-03-27 # https://thetradedesk.atlassian.net/browse/UID2-4925 -CVE-2024-57699 +CVE-2024-57699 exp:2025-03-27 # https://thetradedesk.atlassian.net/browse/UID2-5186 CVE-2024-8176 exp:2025-03-27 From d21df1fd16bd72747a006cf820f35c5709d4da9c Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Fri, 21 Mar 2025 00:05:45 +0000 Subject: [PATCH 385/431] [CI Pipeline] Released patch version: 5.49.51 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 31e802712..8984917d2 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.49.38 + 5.49.51 UTF-8 From 70e88fd6e6933ca4ca82520bb2225b6787f2f435 Mon Sep 17 00:00:00 2001 From: Gian Miguel Del Mundo Date: Fri, 21 Mar 2025 11:43:24 +0800 Subject: [PATCH 386/431] Updated approvers list --- .github/actions/update_operator_version/action.yaml | 2 +- .github/workflows/publish-all-operators.yaml | 2 +- .github/workflows/publish-public-operator-docker-image.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/update_operator_version/action.yaml b/.github/actions/update_operator_version/action.yaml index 1c66838e8..91cd54ff0 100644 --- a/.github/actions/update_operator_version/action.yaml +++ b/.github/actions/update_operator_version/action.yaml @@ -43,7 +43,7 @@ runs: uses: trstringer/manual-approval@v1 with: secret: ${{ github.token }} - approvers: thomasm-ttd,atarassov-ttd,cody-constine-ttd + approvers: atarassov-ttd,vishalegbert-ttd,sunnywu,cody-constine-ttd minimum-approvals: 1 issue-title: Creating Major version of UID2-Operator diff --git a/.github/workflows/publish-all-operators.yaml b/.github/workflows/publish-all-operators.yaml index 243e619f3..6b0198060 100644 --- a/.github/workflows/publish-all-operators.yaml +++ b/.github/workflows/publish-all-operators.yaml @@ -46,7 +46,7 @@ jobs: uses: trstringer/manual-approval@v1 with: secret: ${{ github.token }} - approvers: thomasm-ttd,atarassov-ttd,cody-constine-ttd + approvers: atarassov-ttd,vishalegbert-ttd,sunnywu,cody-constine-ttd minimum-approvals: 1 issue-title: Creating Major version of UID2-Operator diff --git a/.github/workflows/publish-public-operator-docker-image.yaml b/.github/workflows/publish-public-operator-docker-image.yaml index d55806c6b..db3c527c8 100644 --- a/.github/workflows/publish-public-operator-docker-image.yaml +++ b/.github/workflows/publish-public-operator-docker-image.yaml @@ -53,7 +53,7 @@ jobs: uses: trstringer/manual-approval@v1 with: secret: ${{ github.token }} - approvers: thomasm-ttd,atarassov-ttd,cody-constine-ttd + approvers: atarassov-ttd,vishalegbert-ttd,sunnywu,cody-constine-ttd minimum-approvals: 1 issue-title: Creating Major version of UID2-Operator From 8a00a52982ce5cf4073b3321b6cc4de7eda3dead Mon Sep 17 00:00:00 2001 From: Gian Miguel Del Mundo Date: Fri, 21 Mar 2025 13:09:36 +0800 Subject: [PATCH 387/431] Updated trivyignore --- .trivyignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.trivyignore b/.trivyignore index bfe50df3d..a4a3dcbd3 100644 --- a/.trivyignore +++ b/.trivyignore @@ -1,7 +1,6 @@ # List any vulnerability that are to be accepted # See https://aquasecurity.github.io/trivy/v0.35/docs/vulnerability/examples/filter/ # for more details -# e.g. # https://thetradedesk.atlassian.net/browse/UID2-4460 CVE-2024-47535 From 5cd39ee615c3e9d37ba1c3613afce3b1d824c806 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Fri, 21 Mar 2025 06:07:37 +0000 Subject: [PATCH 388/431] [CI Pipeline] Released Patch version: 5.49.56 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 8984917d2..6e8e37166 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.49.51 + 5.49.56 UTF-8 From cddcb94b567c426f1df47d6b3154c2496f48c1bd Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Sat, 22 Mar 2025 00:05:38 +0000 Subject: [PATCH 389/431] [CI Pipeline] Released patch version: 5.49.57 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 6e8e37166..7bb0d8135 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.49.56 + 5.49.57 UTF-8 From d6fdb486a203ba0b52ca867a424cf7d487f1cf9c Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Sun, 23 Mar 2025 00:06:10 +0000 Subject: [PATCH 390/431] [CI Pipeline] Released patch version: 5.49.58 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 7bb0d8135..3c74dffeb 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.49.57 + 5.49.58 UTF-8 From 48c50189b89e85101a956f23a4d78b48fbf0c9d4 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Mon, 24 Mar 2025 00:06:20 +0000 Subject: [PATCH 391/431] [CI Pipeline] Released patch version: 5.49.59 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3c74dffeb..28225b224 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.49.58 + 5.49.59 UTF-8 From 662f8205708a068e7dbf4e150ca1f78892f1f366 Mon Sep 17 00:00:00 2001 From: Gian Miguel Del Mundo Date: Mon, 24 Mar 2025 10:23:06 +0800 Subject: [PATCH 392/431] Updated Azure attestation lib version to 2.1.13 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 6e8e37166..91388facd 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ 1.12.2 2.1.6 2.1.0 - 2.1.0 + 2.1.13 2.1.0 8.1.25 ${project.version} From 39a1e5cf06674b4b06e4881bec22514f2a71fbb7 Mon Sep 17 00:00:00 2001 From: Gian Miguel Del Mundo Date: Mon, 24 Mar 2025 10:39:15 +0800 Subject: [PATCH 393/431] Removed CVE-2024-57699 from trivyignore --- .trivyignore | 3 --- 1 file changed, 3 deletions(-) diff --git a/.trivyignore b/.trivyignore index a4a3dcbd3..2a67adbc2 100644 --- a/.trivyignore +++ b/.trivyignore @@ -8,8 +8,5 @@ CVE-2024-47535 # https://thetradedesk.atlassian.net/browse/UID2-4874 CVE-2025-24970 exp:2025-03-27 -# https://thetradedesk.atlassian.net/browse/UID2-4925 -CVE-2024-57699 exp:2025-03-27 - # https://thetradedesk.atlassian.net/browse/UID2-5186 CVE-2024-8176 exp:2025-03-27 From ef54f9614924e8c6d1e599101af8e55484173bca Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Mon, 24 Mar 2025 05:04:21 +0000 Subject: [PATCH 394/431] [CI Pipeline] Released Patch version: 5.49.61 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 850b2b7f4..f55feac7c 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.49.59 + 5.49.61 UTF-8 From 506f9ee45af068f5c57ee0e147c7d2fc8110e421 Mon Sep 17 00:00:00 2001 From: Gian Miguel Del Mundo Date: Mon, 24 Mar 2025 18:46:25 +0800 Subject: [PATCH 395/431] Updated E2E test branch --- .github/workflows/run-e2e-tests-on-operator.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-e2e-tests-on-operator.yaml b/.github/workflows/run-e2e-tests-on-operator.yaml index cb7fdff71..3570f20ef 100644 --- a/.github/workflows/run-e2e-tests-on-operator.yaml +++ b/.github/workflows/run-e2e-tests-on-operator.yaml @@ -117,7 +117,7 @@ on: jobs: e2e-test: name: E2E Test - uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-run-e2e-tests.yaml@v3 + uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-run-e2e-tests.yaml@gdm-UID2-5040-e2e with: operator_type: ${{ inputs.operator_type }} identity_scope: ${{ inputs.identity_scope }} From eacea24b6408774ba1758e10adc92a2026323b16 Mon Sep 17 00:00:00 2001 From: Ashley Smith Date: Mon, 24 Mar 2025 12:11:00 -0600 Subject: [PATCH 396/431] removing all old sdk tracking code except for version 2 --- .../com/uid2/operator/vertx/Endpoints.java | 1 - .../operator/vertx/UIDOperatorVerticle.java | 30 +- static/js/euid-sdk-1.0.0.js | 344 ------------------ static/js/openid-sdk-1.0.js | 156 -------- static/js/uid2-esp-0.0.1a.js | 25 -- static/js/uid2-sdk-0.0.1a.js | 78 ---- static/js/uid2-sdk-0.0.1b.js | 99 ----- static/js/uid2-sdk-1.0.0.js | 338 ----------------- static/js/uid2-sdk-2.0.0.js | 4 +- 9 files changed, 3 insertions(+), 1072 deletions(-) delete mode 100644 static/js/euid-sdk-1.0.0.js delete mode 100644 static/js/openid-sdk-1.0.js delete mode 100644 static/js/uid2-esp-0.0.1a.js delete mode 100644 static/js/uid2-sdk-0.0.1a.js delete mode 100644 static/js/uid2-sdk-0.0.1b.js delete mode 100644 static/js/uid2-sdk-1.0.0.js diff --git a/src/main/java/com/uid2/operator/vertx/Endpoints.java b/src/main/java/com/uid2/operator/vertx/Endpoints.java index f778f7c72..2e33a7e34 100644 --- a/src/main/java/com/uid2/operator/vertx/Endpoints.java +++ b/src/main/java/com/uid2/operator/vertx/Endpoints.java @@ -6,7 +6,6 @@ public enum Endpoints { OPS_HEALTHCHECK("/ops/healthcheck"), - OLD_SDK_LOG("/ops/logSdk"), V0_KEY_LATEST("/key/latest"), V0_TOKEN_GENERATE("/token/generate"), V0_TOKEN_REFRESH("/token/refresh"), diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index 07035e569..ad6f916bc 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -249,8 +249,6 @@ private Router createRoutesSetup() throws IOException { // Static and health check router.get(OPS_HEALTHCHECK.toString()).handler(this::handleHealthCheck); - // Temporary Old SDK logging - router.get(OLD_SDK_LOG.toString()).handler(this::handleOldSDKVersionLogging); if (this.allowLegacyAPI) { // V1 APIs @@ -817,31 +815,6 @@ private void handleHealthCheck(RoutingContext rc) { } } - // remove in UID2-4990 - private final Map, Counter> _clientVersionCounters = new HashMap<>(); - private void handleOldSDKVersionLogging(RoutingContext rc) { - String clientVersion = rc.request().headers().get(com.uid2.shared.Const.Http.ClientVersionHeader); - if (!clientVersion.equals("openid-sdk-1.0") && !clientVersion.equals("uid2-esp-0.0.1a")) { - clientVersion = null; - } - String host = !rc.queryParam("host").isEmpty() ? rc.queryParam("host").get(0) : null; - if (host != null) { - try { - URI.create(host).toURL(); - } catch (IllegalArgumentException | MalformedURLException e) { - host = null; - } - } - if (clientVersion != null && host != null) { - _clientVersionCounters.computeIfAbsent(new Tuple.Tuple2<>(host, clientVersion), tuple -> Counter - .builder("uid2.old_client_sdk_versions") - .description("counter for how many http requests are processed per each very old client sdk version") - .tags("host", tuple.getItem1(), "client_version", tuple.getItem2()) - .register(Metrics.globalRegistry)).increment();; - } - rc.response().end("OK"); - } - private void handleTokenRefreshV1(RoutingContext rc) { final List tokenList = rc.queryParam("refresh_token"); TokenResponseStatsCollector.PlatformType platformType = getPlatformType(rc); @@ -870,7 +843,6 @@ private void handleTokenRefreshV1(RoutingContext rc) { try { final RefreshResponse r = this.refreshIdentity(rc, refreshToken); siteId = rc.get(Const.RoutingContextData.SiteId); - recordOperatorServedSdkUsage(siteId, rc, rc.request().headers().get(Const.Http.ClientVersionHeader)); if (!r.isRefreshed()) { if (r.isOptOut() || r.isDeprecated()) { ResponseUtil.SuccessNoBody(ResponseStatus.OptOut, rc); @@ -895,6 +867,7 @@ private void handleTokenRefreshV1(RoutingContext rc) { } } + private final Map, Counter> _clientVersionCounters = new HashMap<>(); public void recordOperatorServedSdkUsage(Integer siteId, RoutingContext rc, String clientVersion) { if (siteId != null && clientVersion != null) { _clientVersionCounters.computeIfAbsent(new Tuple.Tuple2<>(Integer.toString(siteId), clientVersion), tuple -> Counter @@ -1181,7 +1154,6 @@ private void handleTokenRefresh(RoutingContext rc) { sendJsonResponse(rc, toJson(r.getTokens())); siteId = rc.get(Const.RoutingContextData.SiteId); - recordOperatorServedSdkUsage(siteId, rc, rc.request().headers().get(Const.Http.ClientVersionHeader)); if (r.isRefreshed()) { this.recordRefreshDurationStats(siteId, getApiContact(rc), r.getDurationSinceLastRefresh(), rc.request().headers().contains(ORIGIN_HEADER), identityExpiresAfter); } diff --git a/static/js/euid-sdk-1.0.0.js b/static/js/euid-sdk-1.0.0.js deleted file mode 100644 index d89f0a712..000000000 --- a/static/js/euid-sdk-1.0.0.js +++ /dev/null @@ -1,344 +0,0 @@ -class EUID { - static get VERSION() { - return "1.0.0"; - } - static get COOKIE_NAME() { - return "__euid"; - } - static get DEFAULT_REFRESH_RETRY_PERIOD_MS() { - return 5000; - } - - constructor() { - // PUBLIC METHODS - - this.init = (opts) => { - if (_initCalled) { - throw new TypeError('Calling init() more than once is not allowed'); - } - - if (typeof opts !== 'object' || opts === null) { - throw new TypeError('opts must be an object'); - } else if (typeof opts.callback !== 'function') { - throw new TypeError('opts.callback must be a function'); - } else if (typeof opts.refreshRetryPeriod !== 'undefined') { - if (typeof opts.refreshRetryPeriod !== 'number') - throw new TypeError('opts.refreshRetryPeriod must be a number'); - else if (opts.refreshRetryPeriod < 1000) - throw new RangeError('opts.refreshRetryPeriod must be >= 1000'); - } - - _initCalled = true; - _opts = opts; - let identity = _opts.identity ? _opts.identity : loadIdentity() - applyIdentity(identity); - }; - this.getAdvertisingToken = () => { - return _identity && !temporarilyUnavailable() ? _identity.advertising_token : undefined; - }; - this.getAdvertisingTokenAsync = () => { - if (!initialised()) { - return new Promise((resolve, reject) => { - _promises.push({ resolve: resolve, reject: reject }); - }); - } else if (_identity) { - return temporarilyUnavailable() - ? Promise.reject(new Error('temporarily unavailable')) - : Promise.resolve(_identity.advertising_token); - } else { - return Promise.reject(new Error('identity not available')); - } - }; - this.isLoginRequired = () => { - return initialised() ? !_identity : undefined; - }; - this.disconnect = () => { - this.abort(); - removeCookie(EUID.COOKIE_NAME); - _identity = undefined; - _lastStatus = EUID.IdentityStatus.INVALID; - - const promises = _promises; - _promises = []; - promises.forEach(p => p.reject(new Error("disconnect()"))); - }; - this.abort = () => { - _initCalled = true; - if (typeof _refreshTimerId !== 'undefined') { - clearTimeout(_refreshTimerId); - _refreshTimerId = undefined; - } - if (_refreshReq) { - _refreshReq.abort(); - _refreshReq = undefined; - } - }; - - // PRIVATE STATE - - let _initCalled = false; - let _opts; - let _identity; - let _lastStatus; - let _refreshTimerId; - let _refreshReq; - let _promises = []; - - // PRIVATE METHODS - - const initialised = () => typeof _lastStatus !== 'undefined'; - const temporarilyUnavailable = () => _lastStatus === EUID.IdentityStatus.EXPIRED; - - const getOptionOrDefault = (value, defaultValue) => { - return typeof value === 'undefined' ? defaultValue : value; - }; - - const setCookie = (name, identity) => { - const value = JSON.stringify(identity); - const expires = new Date(identity.refresh_expires); - const path = getOptionOrDefault(_opts.cookiePath, "/"); - let cookie = name + "=" + encodeURIComponent(value) + " ;path=" + path + ";expires=" + expires.toUTCString(); - if (typeof _opts.cookieDomain !== 'undefined') { - cookie += ";domain=" + _opts.cookieDomain; - } - document.cookie = cookie; - }; - const removeCookie = (name) => { - document.cookie = name + "=;expires=Tue, 1 Jan 1980 23:59:59 GMT"; - }; - const getCookie = (name) => { - const docCookie = document.cookie; - if (docCookie) { - const payload = docCookie.split('; ').find(row => row.startsWith(name+'=')); - if (payload) { - return decodeURIComponent(payload.split('=')[1]); - } - } - }; - - const updateStatus = (status, statusText) => { - _lastStatus = status; - - const promises = _promises; - _promises = []; - - const advertisingToken = this.getAdvertisingToken(); - - const result = { - advertisingToken: advertisingToken, - advertising_token: advertisingToken, - status: status, - statusText: statusText - }; - _opts.callback(result); - - if (advertisingToken) { - promises.forEach(p => p.resolve(advertisingToken)); - } else { - promises.forEach(p => p.reject(new Error(statusText))); - } - }; - const setValidIdentity = (identity, status, statusText) => { - _identity = identity; - setCookie(EUID.COOKIE_NAME, identity); - setRefreshTimer(); - updateStatus(status, statusText); - }; - const setFailedIdentity = (status, statusText) => { - _identity = undefined; - this.abort(); - removeCookie(EUID.COOKIE_NAME); - updateStatus(status, statusText); - }; - const checkIdentity = (identity) => { - if (!identity.advertising_token) { - throw new InvalidIdentityError("advertising_token is not available or is not valid"); - } else if (!identity.refresh_token) { - throw new InvalidIdentityError("refresh_token is not available or is not valid"); - } else if (!identity.refresh_from) { - throw new InvalidIdentityError("refresh_from is not available or is not valid"); - } else if (!identity.identity_expires) { - throw new InvalidIdentityError("identity_expires is not available or is not valid"); - } else if (!identity.refresh_expires) { - throw new InvalidIdentityError("refresh_expires is not available or is not valid"); - } - }; - const tryCheckIdentity = (identity) => { - try { - checkIdentity(identity); - return true; - } catch (err) { - if (err instanceof InvalidIdentityError) { - setFailedIdentity(EUID.IdentityStatus.INVALID, err.message); - return false; - } else { - throw err; - } - } - }; - const setIdentity = (identity, status, statusText) => { - if (tryCheckIdentity(identity)) { - setValidIdentity(identity, status, statusText); - } - }; - const loadIdentity = () => { - const payload = getCookie(EUID.COOKIE_NAME); - if (payload) { - return JSON.parse(payload); - } - }; - - const applyIdentity = (identity) => { - if (!identity) { - setFailedIdentity(EUID.IdentityStatus.NO_IDENTITY, "Identity not available"); - return; - } - - if (!tryCheckIdentity(identity)) { - // failed identity already set - return; - } - - const now = Date.now(); - if (identity.refresh_expires < now) { - setFailedIdentity(EUID.IdentityStatus.REFRESH_EXPIRED, "Identity expired, refresh expired"); - return; - } - if (identity.refresh_from <= now) { - refreshToken(identity); - return; - } - - if (typeof _identity === 'undefined') { - setIdentity(identity, EUID.IdentityStatus.ESTABLISHED, "Identity established"); - } else if (identity.advertising_token !== _identity.advertising_token) { - // identity must have been refreshed from another tab - setIdentity(identity, EUID.IdentityStatus.REFRESH, "Identity refreshed"); - } else { - setRefreshTimer(); - } - } - - const createArrayBuffer = (text) => { - let arrayBuffer = new Uint8Array(text.length); - for (let i = 0; i < text.length; i++) { - arrayBuffer[i] = text.charCodeAt(i); - } - return arrayBuffer; - } - - const refreshToken = (identity) => { - const baseUrl = getOptionOrDefault(_opts.baseUrl, "https://prod.euid.eu"); - const url = baseUrl + "/v2/token/refresh"; - const req = new XMLHttpRequest(); - _refreshReq = req; - req.overrideMimeType("text/plain"); - req.open("POST", url, true); - req.setRequestHeader('X-UID2-Client-Version', 'euid-sdk-' + EUID.VERSION); - req.onreadystatechange = () => { - _refreshReq = undefined; - if (req.readyState !== req.DONE) return; - try { - if(req.status !== 200) { - const response = JSON.parse(req.responseText); - if (!checkResponseStatus(identity, response)) return; - setIdentity(response.body, EUID.IdentityStatus.REFRESHED, "Identity refreshed"); - } else { - let encodeResp = createArrayBuffer(atob(req.responseText)); - window.crypto.subtle.importKey("raw", createArrayBuffer(atob(identity.refresh_response_key)), - { name: "AES-GCM" }, false, ["decrypt"] - ).then((key) => { - //returns the symmetric key - window.crypto.subtle.decrypt({ - name: "AES-GCM", - iv: encodeResp.slice(0, 12), //The initialization vector you used to encrypt - tagLength: 128, //The tagLength you used to encrypt (if any) - }, - key, - encodeResp.slice(12) - ).then((decrypted) => { - const decryptedResponse = String.fromCharCode.apply(String, new Uint8Array(decrypted)); - const response = JSON.parse(decryptedResponse); - if (!checkResponseStatus(identity, response)) return; - setIdentity(response.body, EUID.IdentityStatus.REFRESHED, "Identity refreshed"); - }) - }) - } - } catch (err) { - handleRefreshFailure(identity, err.message); - } - }; - req.send(identity.refresh_token); - }; - const checkResponseStatus = (identity, response) => { - if (typeof response !== 'object' || response === null) { - throw new TypeError("refresh response is not an object"); - } - if (response.status === "optout") { - setFailedIdentity(EUID.IdentityStatus.OPTOUT, "User opted out"); - return false; - } else if (response.status === "expired_token") { - setFailedIdentity(EUID.IdentityStatus.REFRESH_EXPIRED, "Refresh token expired"); - return false; - } else if (response.status === "success") { - if (typeof response.body === 'object' && response.body !== null) { - return true; - } - throw new TypeError("refresh response object does not have a body"); - } else { - throw new TypeError("unexpected response status: " + response.status); - } - }; - const handleRefreshFailure = (identity, errorMessage) => { - const now = Date.now(); - if (identity.refresh_expires <= now) { - setFailedIdentity(EUID.IdentityStatus.REFRESH_EXPIRED, "Refresh expired; token refresh failed: " + errorMessage); - } else if (identity.identity_expires <= now && !temporarilyUnavailable()) { - setValidIdentity(identity, EUID.IdentityStatus.EXPIRED, "Token refresh failed for expired identity: " + errorMessage); - } else if (initialised()) { - setRefreshTimer(); // silently retry later - } else { - setIdentity(identity, EUID.IdentityStatus.ESTABLISHED, "Identity established; token refresh failed: " + errorMessage) - } - }; - const setRefreshTimer = () => { - const timeout = getOptionOrDefault(_opts.refreshRetryPeriod, EUID.DEFAULT_REFRESH_RETRY_PERIOD_MS); - _refreshTimerId = setTimeout(() => { - if (this.isLoginRequired()) return; - applyIdentity(loadIdentity()); - }, timeout); - }; - - // PRIVATE ERRORS - - class InvalidIdentityError extends Error { - constructor(message) { - super(message); - this.name = "InvalidIdentityError"; - } - } - } -} - -(function (EUID) { - let IdentityStatus; // enum - (function (IdentityStatus) { - // identity available - IdentityStatus[IdentityStatus["ESTABLISHED"] = 0] = "ESTABLISHED"; - IdentityStatus[IdentityStatus["REFRESHED"] = 1] = "REFRESHED"; - // identity temporarily not available - IdentityStatus[IdentityStatus["EXPIRED"] = 100] = "EXPIRED"; - // identity not available - IdentityStatus[IdentityStatus["NO_IDENTITY"] = -1] = "NO_IDENTITY"; - IdentityStatus[IdentityStatus["INVALID"] = -2] = "INVALID"; - IdentityStatus[IdentityStatus["REFRESH_EXPIRED"] = -3] = "REFRESH_EXPIRED"; - IdentityStatus[IdentityStatus["OPTOUT"] = -4] = "OPTOUT"; - })(IdentityStatus = EUID.IdentityStatus || (EUID.IdentityStatus = {})); -})(EUID || (EUID = {})); - -window.__euid = new EUID(); - -if (typeof exports !== 'undefined') { - exports.EUID = EUID; - exports.window = window; -} diff --git a/static/js/openid-sdk-1.0.js b/static/js/openid-sdk-1.0.js deleted file mode 100644 index 14c7173d8..000000000 --- a/static/js/openid-sdk-1.0.js +++ /dev/null @@ -1,156 +0,0 @@ -var __openId = { - - init : function(opts) { - const url = "https://prod.uidapi.com/ops/logSdk?host=" + document.location.origin; - const req = new XMLHttpRequest(); - req.open("GET", url, false); - req.setRequestHeader("X-UID2-Client-Version", "openid-sdk-1.0"); - req.send(); - - this.opts = opts; - if (!this.opts["events"]) { - this.opts["events"] = {} - } - if (this.opts["events"]["init"]) { - this.printDebug("Calling init callback"); - this.opts["events"]["init"](this); - } - if (this.opts["start"]) { - if (this.opts["identity"]) { - this.setIdentity(this.opts["identity"]); - } else if (this.opts["email"]) { - this.startVerificationFlow(); - } else { - if (!this.detectFromUrl()) { - this.refreshIfNeededIdentity(); - return; - } - } - - this.establishIdentity(); - } - - }, - - getTDID : function() { - var cookie = Cookies.get("__open_id") - if (cookie) { - var payload = JSON.parse(decodeURIComponent(cookie)); - return payload["tdid"]; - } - }, - - detectFromUrl : function() { - const urlParams = new URLSearchParams(window.location.search); - const payload = urlParams.get("__oidt"); - console.log("Payload = "); - console.log(payload); - - if (payload && payload != "") { - this.setIdentity(payload); - return true; - } else { - return false; - } - }, - - sendCode : function() { - - $("#verification-entry").show(); - - }, - verifyCode: function() { - - console.log("Submit Value"); - - var email = this.opts["email"]; - var setIdentitfy = this.setIdentity; - var establish = this.establishIdentity; - - $.ajax({ - url: "https://www.openid2.com:444/identity/verification/submit?email="+email+ - "&privacy_bits=1&code=1234&token=abasca", - }) - .done(function( data ) { - var d = JSON.stringify(data); - - window.__openId.opts["identity"] = d; - window.__openId.setIdentity(); - window.__openId.establishIdentity(); - }); - - }, - startVerificationFlow : function() { - $("#open-id-container").show(); - $("#verification-email").val(this.opts["email"]); - $("#send-code").on("click", this.sendCode); - $("#verify-code").on("click", function() { window.__openId.verifyCode() }); - }, - - setIdentity : function(tokens) { - Cookies.set("__open_id", tokens); - }, - - - establishIdentity : function() { - var cookie = Cookies.get("__open_id") - if (cookie) { - var payload = JSON.parse(decodeURIComponent(cookie)); - console.log("Cookie Payload = "); - console.log(cookie); - this.opts["events"]["established"](payload["advertisement_token"]); - } else { - console.log("here"); - if (this.opts["events"]["not_established"]) { - this.opts["events"]["not_established"](); - } - } - }, - - disconnect : function() { - Cookies.remove("__open_id"); - this.establishIdentity(); - }, - - needsRereshing : function(paylod) { - var refreshToken = paylod["refresh_token"]; - // FIXME TODO check for Reresh and continue the Lifecycle - return true; - }, - - refreshIfNeededIdentity : function() { - var cookie = Cookies.get("__open_id") - if (cookie) { - var payload = JSON.parse(decodeURIComponent(cookie)); - console.log("Cookie Payload = "); - console.log(cookie); - if (this.needsRereshing(payload)) { - - $.ajax({ - url: "https://www.openid2.com:444/token/refresh?refresh_token="+encodeURIComponent(payload["refresh_token"]) - }) - .done(function( data ) { - console.log("Token = "); - console.log(data); - if (data && data["advertisement_token"] && data["advertisement_token"] != "") { - var d = encodeURIComponent(JSON.stringify(data)); - window.__openId.setIdentity(d); - } else { - window.__openId.disconnect(); - } - window.__openId.establishIdentity(); - }); - } - } else { - window.__openId.establishIdentity(); - } - this.printDebug("Reresh Token here"); - }, - - printDebug : function(m) { - console.log("__open_id: " + m); - - } -} -window.__openId = __openId; -console.log("OepnID SDK Loaded"); diff --git a/static/js/uid2-esp-0.0.1a.js b/static/js/uid2-esp-0.0.1a.js deleted file mode 100644 index 40cf8f14a..000000000 --- a/static/js/uid2-esp-0.0.1a.js +++ /dev/null @@ -1,25 +0,0 @@ -function __esp_getUID2Async(cb) { - const url = "https://prod.uidapi.com/ops/logSdk?host=" + document.location.origin; - const req = new XMLHttpRequest(); - req.open("GET", url, false); - req.setRequestHeader("X-UID2-Client-Version", "uid2-esp-0.0.1a"); - req.send(); - return new Promise(function(cb) { - if (window.__uid2 && window.__uid2.getAdvertisingToken) { - cb(__uid2.getAdvertisingToken()); - } else { - throw new "UID2 SDK not present"; - } - }); -} - -if (typeof (googletag) !== "undefined" && googletag) { - - googletag.encryptedSignalProviders.push({ - id: 'uidapi.com', - collectorFunction: () => { - return __esp_getUID2Async().then((signals) => signals); - } - }); - -} \ No newline at end of file diff --git a/static/js/uid2-sdk-0.0.1a.js b/static/js/uid2-sdk-0.0.1a.js deleted file mode 100644 index 66e068bc1..000000000 --- a/static/js/uid2-sdk-0.0.1a.js +++ /dev/null @@ -1,78 +0,0 @@ -class UID2 { - constructor() { - this.init = (opts) => { - const identity = opts["identity"]; - if (identity) { - this.setIdentity(identity); - } - else { - this.refreshIfNeeded(); - } - }; - this.refreshIfNeeded = () => { - const identity = this.getIdentity(); - if (identity) { - const url = "https://prod.uidapi.com/token/refresh?refresh_token=" + encodeURIComponent(identity["refresh_token"]); - const req = new XMLHttpRequest(); - req.overrideMimeType("application/json"); - var cb = this.handleRefreshResponse; - req.open("GET", url, false); - req.setRequestHeader("X-UID2-Client-Version", "uid2-sdk-0.0.1a"); - req.onload = function () { - cb(req.responseText); - }; - req.send(); - } - }; - this.handleRefreshResponse = (body) => { - this.setIdentity(body); - }; - this.getIdentity = () => { - const payload = this.getCookie("__uid_2"); - if (payload) { - return JSON.parse(payload); - } - }; - this.getAdvertisingToken = () => { - const identity = this.getIdentity(); - if (identity) { - return identity["advertisement_token"]; - } - }; - this.setIdentity = (value) => { - var payload; - if (typeof (value) === "object") { - payload = JSON.stringify(value); - } - else { - payload = value; - } - this.setCookie("__uid_2", payload); - }; - this.setCookie = (name, value) => { - var days = 7; - var date = new Date(); - date.setTime(date.getTime() + (days * 24 * 60 * 60 * 1000)); - document.cookie = name + "=" + encodeURIComponent(value) + " ;path=/;expires=" + date.toUTCString(); - }; - this.getCookie = (name) => { - const docCookie = document.cookie; - if (docCookie) { - var payload = docCookie.split('; ').find(row => row.startsWith(name)); - if (payload) { - return decodeURIComponent(payload.split('=')[1]); - } - } - else { - return undefined; - } - }; - this.removeCookie = (name) => { - document.cookie = name + "=;path=/;expires=Tue, 1 Jan 1980 23:59:59 GMT"; - }; - this.disconnect = () => { - this.removeCookie("__uid_2"); - }; - } -} -window.__uid2 = new UID2(); diff --git a/static/js/uid2-sdk-0.0.1b.js b/static/js/uid2-sdk-0.0.1b.js deleted file mode 100644 index c62cd85f8..000000000 --- a/static/js/uid2-sdk-0.0.1b.js +++ /dev/null @@ -1,99 +0,0 @@ -function __esp_getUID2Async(cb) { - return new Promise(function(cb) { - if (window.__uid2 && window.__uid2.getAdvertisingToken) { - cb(__uid2.getAdvertisingToken()); - } else { - throw new "UID2 SDK not present"; - } - }); -} - -if (typeof (googletag) !== "undefined" && googletag && googletag.encryptedSignalProviders) { - - googletag.encryptedSignalProviders.push({ - id: 'uidapi.com', - collectorFunction: () => { - return __esp_getUID2Async().then((signals) => signals); - } - }); - -} - -class UID2 { - constructor() { - this.init = (opts) => { - const identity = opts["identity"]; - if (identity) { - this.setIdentity(identity); - } - else { - this.refreshIfNeeded(); - } - }; - this.refreshIfNeeded = () => { - const identity = this.getIdentity(); - if (identity) { - const url = "https://prod.uidapi.com/token/refresh?refresh_token=" + encodeURIComponent(identity["refresh_token"]); - const req = new XMLHttpRequest(); - req.overrideMimeType("application/json"); - var cb = this.handleRefreshResponse; - req.open("GET", url, false); - req.setRequestHeader("X-UID2-Client-Version", "uid2-sdk-0.0.1b"); - req.onload = function () { - cb(req.responseText); - }; - req.send(); - } - }; - this.handleRefreshResponse = (body) => { - this.setIdentity(body); - }; - this.getIdentity = () => { - const payload = this.getCookie("__uid_2"); - if (payload) { - return JSON.parse(payload); - } - }; - this.getAdvertisingToken = () => { - const identity = this.getIdentity(); - if (identity) { - return identity["advertisement_token"]; - } - }; - this.setIdentity = (value) => { - var payload; - if (typeof (value) === "object") { - payload = JSON.stringify(value); - } - else { - payload = value; - } - this.setCookie("__uid_2", payload); - }; - this.setCookie = (name, value) => { - var days = 7; - var date = new Date(); - date.setTime(date.getTime() + (days * 24 * 60 * 60 * 1000)); - document.cookie = name + "=" + encodeURIComponent(value) + " ;path=/;expires=" + date.toUTCString(); - }; - this.getCookie = (name) => { - const docCookie = document.cookie; - if (docCookie) { - var payload = docCookie.split('; ').find(row => row.startsWith(name)); - if (payload) { - return decodeURIComponent(payload.split('=')[1]); - } - } - else { - return undefined; - } - }; - this.removeCookie = (name) => { - document.cookie = name + "=;path=/;expires=Tue, 1 Jan 1980 23:59:59 GMT"; - }; - this.disconnect = () => { - this.removeCookie("__uid_2"); - }; - } -} -window.__uid2 = new UID2(); diff --git a/static/js/uid2-sdk-1.0.0.js b/static/js/uid2-sdk-1.0.0.js deleted file mode 100644 index 542ab4447..000000000 --- a/static/js/uid2-sdk-1.0.0.js +++ /dev/null @@ -1,338 +0,0 @@ - -class UID2 { - static get VERSION() { - return "1.0.2"; - } - static get COOKIE_NAME() { - return "__uid_2"; - } - static get DEFAULT_REFRESH_RETRY_PERIOD_MS() { - return 5000; - } - - static setupGoogleTag() { - if (!window.googletag) { - window.googletag = {}; - } - if (!googletag.encryptedSignalProviders) { - googletag.encryptedSignalProviders = []; - } - googletag.encryptedSignalProviders.push({ - id: "uidapi.com", - collectorFunction: () => { - if (window.__uid2 && window.__uid2.getAdvertisingTokenAsync) { - return __uid2.getAdvertisingTokenAsync(); - } else { - return Promise.reject(new Error("UID2 SDK not present")); - } - }, - }); - } - - constructor() { - // PUBLIC METHODS - - this.init = (opts) => { - if (_initCalled) { - throw new TypeError('Calling init() more than once is not allowed'); - } - - if (typeof opts !== 'object' || opts === null) { - throw new TypeError('opts must be an object'); - } else if (typeof opts.callback !== 'function') { - throw new TypeError('opts.callback must be a function'); - } else if (typeof opts.refreshRetryPeriod !== 'undefined') { - if (typeof opts.refreshRetryPeriod !== 'number') - throw new TypeError('opts.refreshRetryPeriod must be a number'); - else if (opts.refreshRetryPeriod < 1000) - throw new RangeError('opts.refreshRetryPeriod must be >= 1000'); - } - - _initCalled = true; - _opts = opts; - applyIdentity(_opts.identity ? _opts.identity : loadIdentity()); - }; - this.getAdvertisingToken = () => { - return _identity && !temporarilyUnavailable() ? _identity.advertising_token : undefined; - }; - this.getAdvertisingTokenAsync = () => { - if (!initialised()) { - return new Promise((resolve, reject) => { - _promises.push({ resolve: resolve, reject: reject }); - }); - } else if (_identity) { - return temporarilyUnavailable() - ? Promise.reject(new Error('temporarily unavailable')) - : Promise.resolve(_identity.advertising_token); - } else { - return Promise.reject(new Error('identity not available')); - } - }; - this.isLoginRequired = () => { - return initialised() ? !_identity : undefined; - }; - this.disconnect = () => { - this.abort(); - removeCookie(UID2.COOKIE_NAME); - _identity = undefined; - _lastStatus = UID2.IdentityStatus.INVALID; - - const promises = _promises; - _promises = []; - promises.forEach(p => p.reject(new Error("disconnect()"))); - }; - this.abort = () => { - _initCalled = true; - if (typeof _refreshTimerId !== 'undefined') { - clearTimeout(_refreshTimerId); - _refreshTimerId = undefined; - } - if (_refreshReq) { - _refreshReq.abort(); - _refreshReq = undefined; - } - }; - - // PRIVATE STATE - - let _initCalled = false; - let _opts; - let _identity; - let _lastStatus; - let _refreshTimerId; - let _refreshReq; - let _promises = []; - - // PRIVATE METHODS - - const initialised = () => typeof _lastStatus !== 'undefined'; - const temporarilyUnavailable = () => _lastStatus === UID2.IdentityStatus.EXPIRED; - - const getOptionOrDefault = (value, defaultValue) => { - return typeof value === 'undefined' ? defaultValue : value; - }; - - const setCookie = (name, identity) => { - const value = JSON.stringify(identity); - const expires = new Date(identity.refresh_expires); - const path = getOptionOrDefault(_opts.cookiePath, "/"); - let cookie = name + "=" + encodeURIComponent(value) + " ;path=" + path + ";expires=" + expires.toUTCString(); - if (typeof _opts.cookieDomain !== 'undefined') { - cookie += ";domain=" + _opts.cookieDomain; - } - document.cookie = cookie; - }; - const removeCookie = (name) => { - document.cookie = name + "=;expires=Tue, 1 Jan 1980 23:59:59 GMT"; - }; - const getCookie = (name) => { - const docCookie = document.cookie; - if (docCookie) { - const payload = docCookie.split('; ').find(row => row.startsWith(name+'=')); - if (payload) { - return decodeURIComponent(payload.split('=')[1]); - } - } - }; - - const updateStatus = (status, statusText) => { - _lastStatus = status; - - const promises = _promises; - _promises = []; - - const advertisingToken = this.getAdvertisingToken(); - - const result = { - advertisingToken: advertisingToken, - advertising_token: advertisingToken, - status: status, - statusText: statusText - }; - _opts.callback(result); - - if (advertisingToken) { - promises.forEach(p => p.resolve(advertisingToken)); - } else { - promises.forEach(p => p.reject(new Error(statusText))); - } - }; - const setValidIdentity = (identity, status, statusText) => { - _identity = identity; - setCookie(UID2.COOKIE_NAME, identity); - setRefreshTimer(); - updateStatus(status, statusText); - }; - const setFailedIdentity = (status, statusText) => { - _identity = undefined; - this.abort(); - removeCookie(UID2.COOKIE_NAME); - updateStatus(status, statusText); - }; - const checkIdentity = (identity) => { - if (!identity.advertising_token) { - throw new InvalidIdentityError("advertising_token is not available or is not valid"); - } else if (!identity.refresh_token) { - throw new InvalidIdentityError("refresh_token is not available or is not valid"); - } - }; - const tryCheckIdentity = (identity) => { - try { - checkIdentity(identity); - return true; - } catch (err) { - if (err instanceof InvalidIdentityError) { - setFailedIdentity(UID2.IdentityStatus.INVALID, err.message); - return false; - } else { - throw err; - } - } - }; - const setIdentity = (identity, status, statusText) => { - if (tryCheckIdentity(identity)) { - setValidIdentity(identity, status, statusText); - } - }; - const loadIdentity = () => { - const payload = getCookie(UID2.COOKIE_NAME); - if (payload) { - return JSON.parse(payload); - } - }; - - const enrichIdentity = (identity, now) => { - return { - refresh_from: now, - refresh_expires: now + 7 * 86400 * 1000, // 7 days - identity_expires: now + 4 * 3600 * 1000, // 4 hours - ...identity, - }; - }; - const applyIdentity = (identity) => { - if (!identity) { - setFailedIdentity(UID2.IdentityStatus.NO_IDENTITY, "Identity not available"); - return; - } - - if (!tryCheckIdentity(identity)) { - // failed identity already set - return; - } - - const now = Date.now(); - identity = enrichIdentity(identity, now); - if (identity.refresh_expires < now) { - setFailedIdentity(UID2.IdentityStatus.REFRESH_EXPIRED, "Identity expired, refresh expired"); - return; - } - if (identity.refresh_from <= now) { - refreshToken(identity); - return; - } - - if (typeof _identity === 'undefined') { - setIdentity(identity, UID2.IdentityStatus.ESTABLISHED, "Identity established"); - } else if (identity.advertising_token !== _identity.advertising_token) { - // identity must have been refreshed from another tab - setIdentity(identity, UID2.IdentityStatus.REFRESH, "Identity refreshed"); - } else { - setRefreshTimer(); - } - } - const refreshToken = (identity) => { - const baseUrl = getOptionOrDefault(_opts.baseUrl, "https://prod.uidapi.com"); - const url = baseUrl + "/v1/token/refresh?refresh_token=" + encodeURIComponent(identity.refresh_token); - const req = new XMLHttpRequest(); - _refreshReq = req; - req.overrideMimeType("application/json"); - req.open("GET", url, true); - req.setRequestHeader('X-UID2-Client-Version', 'uid2-sdk-' + UID2.VERSION); - req.onreadystatechange = () => { - _refreshReq = undefined; - if (req.readyState !== req.DONE) return; - try { - const response = JSON.parse(req.responseText); - if (!checkResponseStatus(identity, response)) return; - checkIdentity(response.body); - setIdentity(response.body, UID2.IdentityStatus.REFRESHED, "Identity refreshed"); - } catch (err) { - handleRefreshFailure(identity, err.message); - } - }; - req.send(); - }; - const checkResponseStatus = (identity, response) => { - if (typeof response !== 'object' || response === null) { - throw new TypeError("refresh response is not an object"); - } - if (response.status === "optout") { - setFailedIdentity(UID2.IdentityStatus.OPTOUT, "User opted out"); - return false; - } else if (response.status === "expired_token") { - setFailedIdentity(UID2.IdentityStatus.REFRESH_EXPIRED, "Refresh token expired"); - return false; - } else if (response.status === "success") { - if (typeof response.body === 'object' && response.body !== null) { - return true; - } - throw new TypeError("refresh response object does not have a body"); - } else { - throw new TypeError("unexpected response status: " + response.status); - } - }; - const handleRefreshFailure = (identity, errorMessage) => { - const now = Date.now(); - if (identity.refresh_expires <= now) { - setFailedIdentity(UID2.IdentityStatus.REFRESH_EXPIRED, "Refresh expired; token refresh failed: " + errorMessage); - } else if (identity.identity_expires <= now && !temporarilyUnavailable()) { - setValidIdentity(identity, UID2.IdentityStatus.EXPIRED, "Token refresh failed for expired identity: " + errorMessage); - } else if (initialised()) { - setRefreshTimer(); // silently retry later - } else { - setIdentity(identity, UID2.IdentityStatus.ESTABLISHED, "Identity established; token refresh failed: " + errorMessage) - } - }; - const setRefreshTimer = () => { - const timeout = getOptionOrDefault(_opts.refreshRetryPeriod, UID2.DEFAULT_REFRESH_RETRY_PERIOD_MS); - _refreshTimerId = setTimeout(() => { - if (this.isLoginRequired()) return; - applyIdentity(loadIdentity()); - }, timeout); - }; - - // PRIVATE ERRORS - - class InvalidIdentityError extends Error { - constructor(message) { - super(message); - this.name = "InvalidIdentityError"; - } - } - } -} - -(function (UID2) { - let IdentityStatus; // enum - (function (IdentityStatus) { - // identity available - IdentityStatus[IdentityStatus["ESTABLISHED"] = 0] = "ESTABLISHED"; - IdentityStatus[IdentityStatus["REFRESHED"] = 1] = "REFRESHED"; - // identity temporarily not available - IdentityStatus[IdentityStatus["EXPIRED"] = 100] = "EXPIRED"; - // identity not available - IdentityStatus[IdentityStatus["NO_IDENTITY"] = -1] = "NO_IDENTITY"; - IdentityStatus[IdentityStatus["INVALID"] = -2] = "INVALID"; - IdentityStatus[IdentityStatus["REFRESH_EXPIRED"] = -3] = "REFRESH_EXPIRED"; - IdentityStatus[IdentityStatus["OPTOUT"] = -4] = "OPTOUT"; - })(IdentityStatus = UID2.IdentityStatus || (UID2.IdentityStatus = {})); -})(UID2 || (UID2 = {})); - -window.__uid2 = new UID2(); - -UID2.setupGoogleTag(); - -if (typeof exports !== 'undefined') { - exports.UID2 = UID2; - exports.window = window; -} diff --git a/static/js/uid2-sdk-2.0.0.js b/static/js/uid2-sdk-2.0.0.js index bd9973f16..b3be1b38f 100644 --- a/static/js/uid2-sdk-2.0.0.js +++ b/static/js/uid2-sdk-2.0.0.js @@ -254,8 +254,8 @@ class UID2 { return arrayBuffer; } - const refreshToken = (identity) => { - const baseUrl = getOptionOrDefault(_opts.baseUrl, "https://prod.uidapi.com"); + this.refreshToken = (identity) => { + const baseUrl = "http://localhost:8080" const url = baseUrl + "/v2/token/refresh"; const req = new XMLHttpRequest(); _refreshReq = req; From 30931321a1b5f492bbf39b3a0b7c1aa58c7c0036 Mon Sep 17 00:00:00 2001 From: Ashley Smith Date: Mon, 24 Mar 2025 12:12:38 -0600 Subject: [PATCH 397/431] no changes for v2 file --- static/js/uid2-sdk-2.0.0.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/static/js/uid2-sdk-2.0.0.js b/static/js/uid2-sdk-2.0.0.js index b3be1b38f..bd9973f16 100644 --- a/static/js/uid2-sdk-2.0.0.js +++ b/static/js/uid2-sdk-2.0.0.js @@ -254,8 +254,8 @@ class UID2 { return arrayBuffer; } - this.refreshToken = (identity) => { - const baseUrl = "http://localhost:8080" + const refreshToken = (identity) => { + const baseUrl = getOptionOrDefault(_opts.baseUrl, "https://prod.uidapi.com"); const url = baseUrl + "/v2/token/refresh"; const req = new XMLHttpRequest(); _refreshReq = req; From e79cc5761dbd79d26781c05b0454048f00d1d660 Mon Sep 17 00:00:00 2001 From: abuabraham-ttd <130511456+abuabraham-ttd@users.noreply.github.com> Date: Mon, 24 Mar 2025 13:04:58 -0700 Subject: [PATCH 398/431] Fix debug logs (#1598) * Fix debug logs /usr/sbin/syslog-ng --verbose is only needed when operator has debug mode enabled * update prod urls --- scripts/aws/UID_CloudFormation.template.yml | 4 ++-- scripts/aws/entrypoint.sh | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/scripts/aws/UID_CloudFormation.template.yml b/scripts/aws/UID_CloudFormation.template.yml index e2422c1fe..e1431159e 100644 --- a/scripts/aws/UID_CloudFormation.template.yml +++ b/scripts/aws/UID_CloudFormation.template.yml @@ -190,9 +190,9 @@ Resources: - '' - - '{' - '"core_base_url": "' - - !If [IsIntegEnvironment, 'https://core-integ.uidapi.com', 'https://core.uidapi.com'] + - !If [IsIntegEnvironment, 'https://core-integ.uidapi.com', 'https://core-prod.uidapi.com'] - '", "optout_base_url": "' - - !If [IsIntegEnvironment, 'https://optout-integ.uidapi.com', 'https://optout.uidapi.com'] + - !If [IsIntegEnvironment, 'https://optout-integ.uidapi.com', 'https://optout-prod.uidapi.com'] - '", "operator_key": "' - Ref: APIToken - '"' diff --git a/scripts/aws/entrypoint.sh b/scripts/aws/entrypoint.sh index 4d67a14fa..ec8bb4d2f 100755 --- a/scripts/aws/entrypoint.sh +++ b/scripts/aws/entrypoint.sh @@ -21,8 +21,6 @@ ifconfig lo 127.0.0.1 echo "Starting vsock proxy..." /app/vsockpx --config /app/proxies.nitro.yaml --daemon --workers $(( ( $(nproc) + 3 ) / 4 )) --log-level 3 -/usr/sbin/syslog-ng --verbose - build_parameterized_config() { curl -s -f -o "${PARAMETERIZED_CONFIG}" -x socks5h://127.0.0.1:3305 http://127.0.0.1:27015/getConfig REQUIRED_KEYS=("optout_base_url" "core_base_url" "core_api_token" "optout_api_token" "environment") @@ -85,6 +83,7 @@ LOGBACK_CONF="./conf/logback.xml" if [[ "$DEBUG_MODE" == "true" ]]; then LOGBACK_CONF="./conf/logback-debug.xml" + /usr/sbin/syslog-ng --verbose fi # -- set pwd to /app so we can find default configs From 6ff0ff15dfe9794b025f69b5f8c3892c36efc4e5 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Mon, 24 Mar 2025 20:07:31 +0000 Subject: [PATCH 399/431] [CI Pipeline] Released Minor version: 5.50.0 --- pom.xml | 2 +- version.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index f55feac7c..b5653728f 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.49.61 + 5.50.0 UTF-8 diff --git a/version.json b/version.json index 71a45ab14..45826baba 100644 --- a/version.json +++ b/version.json @@ -1 +1 @@ -{ "$schema": "https://raw.githubusercontent.com/dotnet/Nerdbank.GitVersioning/master/src/NerdBank.GitVersioning/version.schema.json", "version": "5.49", "publicReleaseRefSpec": [ "^refs/heads/master$", "^refs/heads/v\\d+(?:\\.\\d+)?$" ], "cloudBuild": { "setVersionVariables": true, "buildNumber": { "enabled": true, "includeCommitId": { "when": "always" } } } } +{ "$schema": "https://raw.githubusercontent.com/dotnet/Nerdbank.GitVersioning/master/src/NerdBank.GitVersioning/version.schema.json", "version": "5.50", "publicReleaseRefSpec": [ "^refs/heads/master$", "^refs/heads/v\\d+(?:\\.\\d+)?$" ], "cloudBuild": { "setVersionVariables": true, "buildNumber": { "enabled": true, "includeCommitId": { "when": "always" } } } } From cdcc852f5a6390c673232f07702f3a9270728646 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Tue, 25 Mar 2025 00:05:48 +0000 Subject: [PATCH 400/431] [CI Pipeline] Released patch version: 5.50.1 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index b5653728f..aa74dbf9f 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.50.0 + 5.50.1 UTF-8 From cfb3bdcc9c3c05ceabd1aeda86a35c7640bd9eaa Mon Sep 17 00:00:00 2001 From: abuabraham-ttd <130511456+abuabraham-ttd@users.noreply.github.com> Date: Mon, 24 Mar 2025 22:09:13 -0700 Subject: [PATCH 401/431] revert add logs everytime (#1606) --- scripts/aws/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/aws/entrypoint.sh b/scripts/aws/entrypoint.sh index ec8bb4d2f..c4edba6b1 100755 --- a/scripts/aws/entrypoint.sh +++ b/scripts/aws/entrypoint.sh @@ -16,6 +16,7 @@ ulimit -n 65536 # -- setup loopback device echo "Setting up loopback device..." ifconfig lo 127.0.0.1 +/usr/sbin/syslog-ng --verbose # -- start vsock proxy echo "Starting vsock proxy..." @@ -83,7 +84,6 @@ LOGBACK_CONF="./conf/logback.xml" if [[ "$DEBUG_MODE" == "true" ]]; then LOGBACK_CONF="./conf/logback-debug.xml" - /usr/sbin/syslog-ng --verbose fi # -- set pwd to /app so we can find default configs From 7344e5a523b1eb42b71cc7c329db675db315c407 Mon Sep 17 00:00:00 2001 From: Gian Miguel Del Mundo Date: Tue, 25 Mar 2025 16:03:19 +0800 Subject: [PATCH 402/431] Updated operator shutdown var --- .github/workflows/run-e2e-tests-on-operator.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/run-e2e-tests-on-operator.yaml b/.github/workflows/run-e2e-tests-on-operator.yaml index 3570f20ef..52132f47e 100644 --- a/.github/workflows/run-e2e-tests-on-operator.yaml +++ b/.github/workflows/run-e2e-tests-on-operator.yaml @@ -29,10 +29,10 @@ on: - integ - prod operator_shutdown: - description: PRIVATE OPERATORS ONLY - If true, will automatically shut down operators after E2E tests. If false, remember to manually clean up the operator. + description: PRIVATE OPERATORS ONLY - If true, will delay operator shutdown by 24 hours. required: true type: boolean - default: true + default: false operator_image_version: description: 'Image: Operator image version (for gcp/azure, set appropriate image)' type: string @@ -79,10 +79,10 @@ on: description: PRIVATE OPERATORS ONLY - The target environment [mock, integ, prod] type: string default: mock - operator_shutdown: - description: PRIVATE OPERATORS ONLY - If true, will automatically shut down operators after E2E tests. If false, remember to manually clean up the operator. + delay_operator_shutdown: + description: PRIVATE OPERATORS ONLY - If true, will delay operator shutdown by 24 hours. type: boolean - default: true + default: false operator_image_version: description: 'Image: Operator image version (for gcp/azure, set appropriate image)' type: string @@ -122,7 +122,7 @@ jobs: operator_type: ${{ inputs.operator_type }} identity_scope: ${{ inputs.identity_scope }} target_environment: ${{ inputs.target_environment }} - operator_shutdown: ${{ inputs.operator_shutdown }} + delay_operator_shutdown: ${{ inputs.delay_operator_shutdown }} operator_image_version: ${{ inputs.operator_image_version }} core_image_version: ${{ inputs.core_image_version }} optout_image_version: ${{ inputs.optout_image_version }} From 645211a334d17b94134573ab6f2dfa1d9f2491db Mon Sep 17 00:00:00 2001 From: Gian Miguel Del Mundo Date: Tue, 25 Mar 2025 16:05:06 +0800 Subject: [PATCH 403/431] Fixed input --- .github/workflows/run-e2e-tests-on-operator.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-e2e-tests-on-operator.yaml b/.github/workflows/run-e2e-tests-on-operator.yaml index 52132f47e..d8ad68e97 100644 --- a/.github/workflows/run-e2e-tests-on-operator.yaml +++ b/.github/workflows/run-e2e-tests-on-operator.yaml @@ -28,7 +28,7 @@ on: - mock - integ - prod - operator_shutdown: + delay_operator_shutdown: description: PRIVATE OPERATORS ONLY - If true, will delay operator shutdown by 24 hours. required: true type: boolean From cba7abf6d277881d14c46ca02cb7e6fc8dc68814 Mon Sep 17 00:00:00 2001 From: Gian Miguel Del Mundo Date: Tue, 25 Mar 2025 18:23:03 +0800 Subject: [PATCH 404/431] Reverted e2e test workflow branch --- .github/workflows/run-e2e-tests-on-operator.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-e2e-tests-on-operator.yaml b/.github/workflows/run-e2e-tests-on-operator.yaml index d8ad68e97..462a992e1 100644 --- a/.github/workflows/run-e2e-tests-on-operator.yaml +++ b/.github/workflows/run-e2e-tests-on-operator.yaml @@ -117,7 +117,7 @@ on: jobs: e2e-test: name: E2E Test - uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-run-e2e-tests.yaml@gdm-UID2-5040-e2e + uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-run-e2e-tests.yaml@v3 with: operator_type: ${{ inputs.operator_type }} identity_scope: ${{ inputs.identity_scope }} From 83cec23d8e44b6a64a9099aab35c8b2a08b61d11 Mon Sep 17 00:00:00 2001 From: Katherine Chen Date: Wed, 26 Mar 2025 08:37:47 +1100 Subject: [PATCH 405/431] UID2-5211 Add disable_optout_token feature flag (#1605) * Add disable_optout_token feature flag * Add comment for removing the FF test * Remove disable_optout_token param from private operator configs --- conf/docker-config.json | 3 +- conf/integ-config.json | 3 +- conf/local-config.json | 3 +- conf/local-e2e-docker-public-config.json | 3 +- conf/local-e2e-public-config.json | 3 +- ...dator-latest-e2e-docker-public-config.json | 3 +- src/main/java/com/uid2/operator/Const.java | 1 + .../operator/vertx/UIDOperatorVerticle.java | 4 +- .../operator/UIDOperatorVerticleTest.java | 51 +++++++++++++++++++ 9 files changed, 67 insertions(+), 7 deletions(-) diff --git a/conf/docker-config.json b/conf/docker-config.json index 17266d2bd..818d5f61e 100644 --- a/conf/docker-config.json +++ b/conf/docker-config.json @@ -45,5 +45,6 @@ "format": "json" }, "config_scan_period_ms": 5000 - } + }, + "disable_optout_token": false } diff --git a/conf/integ-config.json b/conf/integ-config.json index 87940a527..0527049f4 100644 --- a/conf/integ-config.json +++ b/conf/integ-config.json @@ -21,5 +21,6 @@ "url": "http://localhost:8088/operator/config" }, "config_scan_period_ms": 300000 - } + }, + "disable_optout_token": false } \ No newline at end of file diff --git a/conf/local-config.json b/conf/local-config.json index 6a91eecb2..a2d715cd6 100644 --- a/conf/local-config.json +++ b/conf/local-config.json @@ -44,5 +44,6 @@ "format": "json" }, "config_scan_period_ms": 5000 - } + }, + "disable_optout_token": false } diff --git a/conf/local-e2e-docker-public-config.json b/conf/local-e2e-docker-public-config.json index 1c3ea4f84..108089ab3 100644 --- a/conf/local-e2e-docker-public-config.json +++ b/conf/local-e2e-docker-public-config.json @@ -40,5 +40,6 @@ "url": "http://core:8088/operator/config" }, "config_scan_period_ms": 300000 - } + }, + "disable_optout_token": false } diff --git a/conf/local-e2e-public-config.json b/conf/local-e2e-public-config.json index 1d4b2a12e..f6182de13 100644 --- a/conf/local-e2e-public-config.json +++ b/conf/local-e2e-public-config.json @@ -46,5 +46,6 @@ "url": "http://localhost:8088/operator/config" }, "config_scan_period_ms": 300000 - } + }, + "disable_optout_token": false } diff --git a/conf/validator-latest-e2e-docker-public-config.json b/conf/validator-latest-e2e-docker-public-config.json index 2c9b11442..4ec7544c7 100644 --- a/conf/validator-latest-e2e-docker-public-config.json +++ b/conf/validator-latest-e2e-docker-public-config.json @@ -39,5 +39,6 @@ "url": "http://core:8088/operator/config" }, "config_scan_period_ms": 300000 - } + }, + "disable_optout_token": false } diff --git a/src/main/java/com/uid2/operator/Const.java b/src/main/java/com/uid2/operator/Const.java index c7d7f3dda..9d6563750 100644 --- a/src/main/java/com/uid2/operator/Const.java +++ b/src/main/java/com/uid2/operator/Const.java @@ -32,5 +32,6 @@ public class Config extends com.uid2.shared.Const.Config { public static final String ConfigScanPeriodMsProp = "config_scan_period_ms"; public static final String IdentityV3Prop = "identity_v3"; + public static final String DisableOptoutTokenProp = "disable_optout_token"; } } diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index 07035e569..05dc776cf 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -100,6 +100,7 @@ public class UIDOperatorVerticle extends AbstractVerticle { private final Clock clock; private final boolean allowLegacyAPI; private final boolean identityV3Enabled; + private final boolean disableOptoutToken; protected IUIDOperatorService idService; private final Map _identityMapMetricSummaries = new HashMap<>(); private final Map, DistributionSummary> _refreshDurationMetricSummaries = new HashMap<>(); @@ -185,6 +186,7 @@ public UIDOperatorVerticle(IConfigService configService, this.optOutStatusMaxRequestSize = config.getInteger(Const.Config.OptOutStatusMaxRequestSize, 5000); this.allowLegacyAPI = config.getBoolean(Const.Config.AllowLegacyAPIProp, true); this.identityV3Enabled = config.getBoolean(IdentityV3Prop, false); + this.disableOptoutToken = config.getBoolean(DisableOptoutTokenProp, false); } @Override @@ -1087,7 +1089,7 @@ private void handleTokenGenerateV2(RoutingContext rc) { identityExpiresAfter); if (t.isEmptyToken()) { - if (optoutCheckPolicy.getItem1() == OptoutCheckPolicy.DoNotRespect) { // only legacy can use this policy + if (optoutCheckPolicy.getItem1() == OptoutCheckPolicy.DoNotRespect && !this.disableOptoutToken) { // only legacy can use this policy final InputUtil.InputVal optOutTokenInput = input.getIdentityType() == IdentityType.Email ? InputUtil.InputVal.validEmail(OptOutTokenIdentityForEmail, OptOutTokenIdentityForEmail) : InputUtil.InputVal.validPhone(OptOutTokenIdentityForPhone, OptOutTokenIdentityForPhone); diff --git a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java index e226b29a6..b2d154b86 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java @@ -129,6 +129,11 @@ public void deployVerticle(Vertx vertx, VertxTestContext testContext, TestInfo t when(this.secureLinkValidatorService.validateRequest(any(RoutingContext.class), any(JsonObject.class), any(Role.class))).thenReturn(true); setupConfig(config); + // TODO: Remove this when we remove tokenGenerateOptOutTokenWithDisableOptoutTokenFF test + if(testInfo.getTestMethod().isPresent() && + testInfo.getTestMethod().get().getName().equals("tokenGenerateOptOutTokenWithDisableOptoutTokenFF")) { + config.put(Const.Config.DisableOptoutTokenProp, true); + } if(testInfo.getDisplayName().equals("cstgNoPhoneSupport(Vertx, VertxTestContext)")) { config.put("enable_phone_support", false); } @@ -165,6 +170,7 @@ private void setupConfig(JsonObject config) { config.put(Const.Config.AllowClockSkewSecondsProp, 3600); config.put(Const.Config.OptOutStatusApiEnabled, true); config.put(Const.Config.OptOutStatusMaxRequestSize, optOutStatusMaxRequestSize); + config.put(Const.Config.DisableOptoutTokenProp, false); } private static byte[] makeAesKey(String prefix) { @@ -1207,6 +1213,51 @@ void tokenGenerateOptOutToken(String policyParameterKey, String identity, Identi }); } + @ParameterizedTest // TODO: remove test after optout check phase 3 + @CsvSource({"policy,someoptout@example.com,Email", + "policy,+01234567890,Phone", + "optout_check,someoptout@example.com,Email", + "optout_check,+01234567890,Phone"}) + void tokenGenerateOptOutTokenWithDisableOptoutTokenFF(String policyParameterKey, String identity, IdentityType identityType, + Vertx vertx, VertxTestContext testContext) { + ClientKey oldClientKey = new ClientKey( + null, + null, + Utils.toBase64String(clientSecret), + "test-contact", + newClientCreationDateTime.minusSeconds(5), + Set.of(Role.GENERATOR), + 201, + null + ); + when(clientKeyProvider.get(any())).thenReturn(oldClientKey); + when(clientKeyProvider.getClientKey(any())).thenReturn(oldClientKey); + when(clientKeyProvider.getOldestClientKey(201)).thenReturn(oldClientKey); + when(this.optOutStore.getLatestEntry(any())).thenReturn(Instant.now()); + setupSalts(); + setupKeys(); + + JsonObject v2Payload = new JsonObject(); + v2Payload.put(identityType.name().toLowerCase(), identity); + v2Payload.put(policyParameterKey, OptoutCheckPolicy.DoNotRespect.policy); + + sendTokenGenerate("v2", vertx, + "", v2Payload, 200, + json -> { + assertEquals("optout", json.getString("status")); + + decodeV2RefreshToken(json); + + assertTokenStatusMetrics( + 201, + TokenResponseStatsCollector.Endpoint.GenerateV2, + TokenResponseStatsCollector.ResponseStatus.OptOut, + TokenResponseStatsCollector.PlatformType.Other); + + testContext.completeNow(); + }); + } + @ParameterizedTest @ValueSource(strings = {"v1", "v2"}) void tokenGenerateForEmail(String apiVersion, Vertx vertx, VertxTestContext testContext) { From fe7af52b2ea0927caee3f355c3410c0b0f3f204e Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Wed, 26 Mar 2025 00:05:58 +0000 Subject: [PATCH 406/431] [CI Pipeline] Released patch version: 5.50.4 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index aa74dbf9f..5855409a5 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.50.1 + 5.50.4 UTF-8 From fb7bb94aa88dc7a666f7a49051cc926903646134 Mon Sep 17 00:00:00 2001 From: Katherine Chen Date: Wed, 26 Mar 2025 14:07:19 +1100 Subject: [PATCH 407/431] Set default AllowLegacyAPIProp value to false (#1609) * Set default AllowLegacyAPIProp value to false * Fix unit test --- src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java | 2 +- src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index 05dc776cf..434e2bc94 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -184,7 +184,7 @@ public UIDOperatorVerticle(IConfigService configService, this.saltRetrievalResponseHandler = saltRetrievalResponseHandler; this.optOutStatusApiEnabled = config.getBoolean(Const.Config.OptOutStatusApiEnabled, true); this.optOutStatusMaxRequestSize = config.getInteger(Const.Config.OptOutStatusMaxRequestSize, 5000); - this.allowLegacyAPI = config.getBoolean(Const.Config.AllowLegacyAPIProp, true); + this.allowLegacyAPI = config.getBoolean(Const.Config.AllowLegacyAPIProp, false); this.identityV3Enabled = config.getBoolean(IdentityV3Prop, false); this.disableOptoutToken = config.getBoolean(DisableOptoutTokenProp, false); } diff --git a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java index b2d154b86..fa08d6b6b 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java @@ -137,6 +137,8 @@ public void deployVerticle(Vertx vertx, VertxTestContext testContext, TestInfo t if(testInfo.getDisplayName().equals("cstgNoPhoneSupport(Vertx, VertxTestContext)")) { config.put("enable_phone_support", false); } + // TODO: Remove this when we remove allow_legacy_api FF + config.put("allow_legacy_api", true); when(configService.getConfig()).thenReturn(config); this.uidOperatorVerticle = new ExtendedUIDOperatorVerticle(configService, config, config.getBoolean("client_side_token_generate"), siteProvider, clientKeyProvider, clientSideKeypairProvider, new KeyManager(keysetKeyStore, keysetProvider), saltProvider, optOutStore, clock, statsCollectorQueue, secureLinkValidatorService, shutdownHandler::handleSaltRetrievalResponse); From 597c33e6e88ba5f29afc02c309a637f57f7cceba Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Wed, 26 Mar 2025 03:09:41 +0000 Subject: [PATCH 408/431] [CI Pipeline] Released Patch version: 5.50.6 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5855409a5..489c584d5 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.50.4 + 5.50.6 UTF-8 From 7179d6d008196a65f6ee02e45575a69f0cdd0904 Mon Sep 17 00:00:00 2001 From: Katherine Chen Date: Wed, 26 Mar 2025 16:04:27 +1100 Subject: [PATCH 409/431] UID2-5061 Revert feature flag and remote config (#1578) * Remove dynamic feature flag * Add remote_config to confs * Update test * Remove config service * Rename feature flag name * Use one config service * Clean up the code * Remove unused variables * Fix typo --- conf/default-config.json | 3 +- conf/docker-config.json | 3 +- conf/feat-flag/feat-flag.json | 5 -- conf/integ-config.json | 3 +- conf/local-config.json | 3 +- conf/local-e2e-docker-private-config.json | 3 +- conf/local-e2e-docker-public-config.json | 3 +- conf/local-e2e-private-config.json | 3 +- conf/local-e2e-public-config.json | 3 +- ...dator-latest-e2e-docker-public-config.json | 3 +- scripts/aws/conf/feat-flag/feat-flag.json | 5 -- scripts/azure-cc/conf/default-config.json | 3 +- .../azure-cc/conf/feat-flag/feat-flag.json | 5 -- .../gcp-oidc/conf/feat-flag/feat-flag.json | 5 -- src/main/java/com/uid2/operator/Const.java | 1 + src/main/java/com/uid2/operator/Main.java | 58 +++++++++---------- .../service/ConfigServiceManager.java | 56 ------------------ .../service/DelegatingConfigService.java | 22 ------- .../operator/ConfigServiceManagerTest.java | 58 ------------------- 19 files changed, 49 insertions(+), 196 deletions(-) delete mode 100644 conf/feat-flag/feat-flag.json delete mode 100644 scripts/aws/conf/feat-flag/feat-flag.json delete mode 100644 scripts/azure-cc/conf/feat-flag/feat-flag.json delete mode 100644 scripts/gcp-oidc/conf/feat-flag/feat-flag.json delete mode 100644 src/main/java/com/uid2/operator/service/ConfigServiceManager.java delete mode 100644 src/main/java/com/uid2/operator/service/DelegatingConfigService.java delete mode 100644 src/test/java/com/uid2/operator/ConfigServiceManagerTest.java diff --git a/conf/default-config.json b/conf/default-config.json index a57739a7f..d6a8e8a41 100644 --- a/conf/default-config.json +++ b/conf/default-config.json @@ -35,5 +35,6 @@ "enclave_platform": null, "failure_shutdown_wait_hours": 120, "sharing_token_expiry_seconds": 2592000, - "operator_type": "public" + "operator_type": "public", + "enable_remote_config": false } diff --git a/conf/docker-config.json b/conf/docker-config.json index 818d5f61e..3402673e5 100644 --- a/conf/docker-config.json +++ b/conf/docker-config.json @@ -46,5 +46,6 @@ }, "config_scan_period_ms": 5000 }, - "disable_optout_token": false + "disable_optout_token": false, + "enable_remote_config": false } diff --git a/conf/feat-flag/feat-flag.json b/conf/feat-flag/feat-flag.json deleted file mode 100644 index 6e816fb99..000000000 --- a/conf/feat-flag/feat-flag.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "remote_config": { - "enabled": true - } -} \ No newline at end of file diff --git a/conf/integ-config.json b/conf/integ-config.json index 0527049f4..c29bfd4f0 100644 --- a/conf/integ-config.json +++ b/conf/integ-config.json @@ -22,5 +22,6 @@ }, "config_scan_period_ms": 300000 }, - "disable_optout_token": false + "disable_optout_token": false, + "enable_remote_config": false } \ No newline at end of file diff --git a/conf/local-config.json b/conf/local-config.json index a2d715cd6..8cb729651 100644 --- a/conf/local-config.json +++ b/conf/local-config.json @@ -45,5 +45,6 @@ }, "config_scan_period_ms": 5000 }, - "disable_optout_token": false + "disable_optout_token": false, + "enable_remote_config": false } diff --git a/conf/local-e2e-docker-private-config.json b/conf/local-e2e-docker-private-config.json index 9751e1f39..d739b8798 100644 --- a/conf/local-e2e-docker-private-config.json +++ b/conf/local-e2e-docker-private-config.json @@ -34,5 +34,6 @@ "url": "http://core:8088/operator/config" }, "config_scan_period_ms": 300000 - } + }, + "enable_remote_config": false } diff --git a/conf/local-e2e-docker-public-config.json b/conf/local-e2e-docker-public-config.json index 108089ab3..03cbb6597 100644 --- a/conf/local-e2e-docker-public-config.json +++ b/conf/local-e2e-docker-public-config.json @@ -41,5 +41,6 @@ }, "config_scan_period_ms": 300000 }, - "disable_optout_token": false + "disable_optout_token": false, + "enable_remote_config": false } diff --git a/conf/local-e2e-private-config.json b/conf/local-e2e-private-config.json index 8c59036b7..02f277df9 100644 --- a/conf/local-e2e-private-config.json +++ b/conf/local-e2e-private-config.json @@ -45,5 +45,6 @@ "url": "http://localhost:8088/operator/config" }, "config_scan_period_ms": 300000 - } + }, + "enable_remote_config": false } diff --git a/conf/local-e2e-public-config.json b/conf/local-e2e-public-config.json index f6182de13..f8ee5a44c 100644 --- a/conf/local-e2e-public-config.json +++ b/conf/local-e2e-public-config.json @@ -47,5 +47,6 @@ }, "config_scan_period_ms": 300000 }, - "disable_optout_token": false + "disable_optout_token": false, + "enable_remote_config": false } diff --git a/conf/validator-latest-e2e-docker-public-config.json b/conf/validator-latest-e2e-docker-public-config.json index 4ec7544c7..270143f07 100644 --- a/conf/validator-latest-e2e-docker-public-config.json +++ b/conf/validator-latest-e2e-docker-public-config.json @@ -40,5 +40,6 @@ }, "config_scan_period_ms": 300000 }, - "disable_optout_token": false + "disable_optout_token": false, + "enable_remote_config": false } diff --git a/scripts/aws/conf/feat-flag/feat-flag.json b/scripts/aws/conf/feat-flag/feat-flag.json deleted file mode 100644 index a296546fb..000000000 --- a/scripts/aws/conf/feat-flag/feat-flag.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "remote_config": { - "enabled": false - } -} \ No newline at end of file diff --git a/scripts/azure-cc/conf/default-config.json b/scripts/azure-cc/conf/default-config.json index 4870b2fda..b63f7420c 100644 --- a/scripts/azure-cc/conf/default-config.json +++ b/scripts/azure-cc/conf/default-config.json @@ -38,5 +38,6 @@ "failure_shutdown_wait_hours": 120, "sharing_token_expiry_seconds": 2592000, "validate_service_links": false, - "operator_type": "private" + "operator_type": "private", + "enable_remote_config": false } diff --git a/scripts/azure-cc/conf/feat-flag/feat-flag.json b/scripts/azure-cc/conf/feat-flag/feat-flag.json deleted file mode 100644 index a296546fb..000000000 --- a/scripts/azure-cc/conf/feat-flag/feat-flag.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "remote_config": { - "enabled": false - } -} \ No newline at end of file diff --git a/scripts/gcp-oidc/conf/feat-flag/feat-flag.json b/scripts/gcp-oidc/conf/feat-flag/feat-flag.json deleted file mode 100644 index a296546fb..000000000 --- a/scripts/gcp-oidc/conf/feat-flag/feat-flag.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "remote_config": { - "enabled": false - } -} \ No newline at end of file diff --git a/src/main/java/com/uid2/operator/Const.java b/src/main/java/com/uid2/operator/Const.java index 9d6563750..bd2ba2987 100644 --- a/src/main/java/com/uid2/operator/Const.java +++ b/src/main/java/com/uid2/operator/Const.java @@ -33,5 +33,6 @@ public class Config extends com.uid2.shared.Const.Config { public static final String ConfigScanPeriodMsProp = "config_scan_period_ms"; public static final String IdentityV3Prop = "identity_v3"; public static final String DisableOptoutTokenProp = "disable_optout_token"; + public static final String EnableRemoteConfigProp = "enable_remote_config"; } } diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index f25d2d1fb..80702c1fa 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -38,8 +38,6 @@ import io.micrometer.prometheus.PrometheusMeterRegistry; import io.micrometer.prometheus.PrometheusRenameFilter; import io.vertx.config.ConfigRetriever; -import io.vertx.config.ConfigRetrieverOptions; -import io.vertx.config.ConfigStoreOptions; import io.vertx.core.*; import io.vertx.core.http.HttpServerOptions; import io.vertx.core.json.JsonObject; @@ -59,6 +57,8 @@ import java.util.*; import java.util.function.Supplier; +import static com.uid2.operator.Const.Config.ConfigScanPeriodMsProp; +import static com.uid2.operator.Const.Config.EnableRemoteConfigProp; import static io.micrometer.core.instrument.Metrics.globalRegistry; public class Main { @@ -268,20 +268,32 @@ private ICloudStorage wrapCloudStorageForOptOut(ICloudStorage cloudStorage) { } } - private Future initialiseConfigService() { - // Read runtime config values from this.config. - final ConfigStoreOptions configStoreOptions = new ConfigStoreOptions() - .setType("json") - .setConfig(config); - - final ConfigRetrieverOptions configRetrieverOptions = new ConfigRetrieverOptions() - .addStore(configStoreOptions) - // Don't scan as config values won't change. - .setScanPeriod(-1); - - final ConfigRetriever configRetriever = ConfigRetriever.create(vertx, configRetrieverOptions); - - return ConfigService.create(configRetriever).map(x -> (IConfigService) x); + private Future initialiseConfigService() throws Exception { + boolean enableRemoteConfigFeatureFlag = config.getBoolean(EnableRemoteConfigProp, false); + ConfigRetriever configRetriever; + + if (enableRemoteConfigFeatureFlag) { + configRetriever = ConfigRetrieverFactory.create( + vertx, + config.getJsonObject("runtime_config_store"), + this.createOperatorKeyRetriever().retrieve() + ); + } else { + configRetriever = ConfigRetrieverFactory.create( + vertx, + new JsonObject() + .put("type", "json") + .put("config", config) + .put(ConfigScanPeriodMsProp, -1), + "" + ); + } + + return ConfigService.create(configRetriever) + .map(configService -> (IConfigService) configService) + .onFailure(e -> { + LOGGER.error("Failed to initialise ConfigService", e); + }); } private void run() throws Exception { @@ -330,20 +342,6 @@ private void run() throws Exception { }); } - private void setupFeatureFlagListener(ConfigServiceManager manager, ConfigRetriever retriever) { - retriever.listen(change -> { - JsonObject newConfig = change.getNewConfiguration(); - boolean useDynamicConfig = newConfig.getJsonObject("remote_config", new JsonObject()).getBoolean("enabled", false); - manager.updateConfigService(useDynamicConfig).onComplete(update -> { - if (update.succeeded()) { - LOGGER.info("Remote config feature flag toggled successfully"); - } else { - LOGGER.error("Failed to toggle remote config feature flag: ", update.cause()); - } - }); - }); - } - private Future createStoreVerticles() throws Exception { // load metadatas for the first time if (clientSideTokenGenerate) { diff --git a/src/main/java/com/uid2/operator/service/ConfigServiceManager.java b/src/main/java/com/uid2/operator/service/ConfigServiceManager.java deleted file mode 100644 index 5fa3ffac2..000000000 --- a/src/main/java/com/uid2/operator/service/ConfigServiceManager.java +++ /dev/null @@ -1,56 +0,0 @@ -package com.uid2.operator.service; - -import io.vertx.core.Future; -import io.vertx.core.Promise; -import io.vertx.core.Vertx; -import io.vertx.core.shareddata.Lock; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class ConfigServiceManager { - private final Vertx vertx; - private final DelegatingConfigService delegatingConfigService; - private final IConfigService dynamicConfigService; - private final IConfigService staticConfigService; - private static final Logger logger = LoggerFactory.getLogger(ConfigServiceManager.class); - - public ConfigServiceManager(Vertx vertx, IConfigService dynamicConfigService, IConfigService staticConfigService, boolean useDynamicConfig) { - this.vertx = vertx; - this.dynamicConfigService = dynamicConfigService; - this.staticConfigService = staticConfigService; - this.delegatingConfigService = new DelegatingConfigService(useDynamicConfig ? dynamicConfigService : staticConfigService); - } - - public Future updateConfigService(boolean useDynamicConfig) { - Promise promise = Promise.promise(); - vertx.sharedData().getLocalLock("updateConfigServiceLock", lockAsyncResult -> { - if (lockAsyncResult.succeeded()) { - Lock lock = lockAsyncResult.result(); - try { - if (useDynamicConfig) { - logger.info("Switching to DynamicConfigService"); - delegatingConfigService.updateConfigService(dynamicConfigService); - } else { - logger.info("Switching to StaticConfigService"); - delegatingConfigService.updateConfigService(staticConfigService); - } - promise.complete(); - } catch (Exception e) { - promise.fail(e); - } finally { - lock.release(); - } - } else { - logger.error("Failed to acquire lock for updating active ConfigService", lockAsyncResult.cause()); - promise.fail(lockAsyncResult.cause()); - } - }); - - return promise.future(); - } - - public IConfigService getDelegatingConfigService() { - return delegatingConfigService; - } - -} diff --git a/src/main/java/com/uid2/operator/service/DelegatingConfigService.java b/src/main/java/com/uid2/operator/service/DelegatingConfigService.java deleted file mode 100644 index caf5016a4..000000000 --- a/src/main/java/com/uid2/operator/service/DelegatingConfigService.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.uid2.operator.service; - -import io.vertx.core.json.JsonObject; - -import java.util.concurrent.atomic.AtomicReference; - -public class DelegatingConfigService implements IConfigService { - private final AtomicReference activeConfigService; - - public DelegatingConfigService(IConfigService initialConfigService) { - this.activeConfigService = new AtomicReference<>(initialConfigService); - } - - public void updateConfigService(IConfigService newConfigService) { - this.activeConfigService.set(newConfigService); - } - - @Override - public JsonObject getConfig() { - return activeConfigService.get().getConfig(); - } -} diff --git a/src/test/java/com/uid2/operator/ConfigServiceManagerTest.java b/src/test/java/com/uid2/operator/ConfigServiceManagerTest.java deleted file mode 100644 index b481bdac3..000000000 --- a/src/test/java/com/uid2/operator/ConfigServiceManagerTest.java +++ /dev/null @@ -1,58 +0,0 @@ -package com.uid2.operator; - -import com.uid2.operator.service.ConfigServiceManager; -import com.uid2.operator.service.IConfigService; -import io.vertx.core.Vertx; -import io.vertx.core.json.JsonObject; -import io.vertx.junit5.VertxExtension; -import io.vertx.junit5.VertxTestContext; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; - -import static com.uid2.operator.Const.Config.*; -import static org.junit.jupiter.api.Assertions.*; -import static com.uid2.operator.service.UIDOperatorService.*; -import static org.mockito.Mockito.*; - -@ExtendWith(VertxExtension.class) -public class ConfigServiceManagerTest { - private JsonObject bootstrapConfig; - private JsonObject staticConfig; - private ConfigServiceManager configServiceManager; - - @BeforeEach - void setUp(Vertx vertx) { - bootstrapConfig = new JsonObject() - .put(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, 3600) - .put(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, 7200) - .put(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS, 1800) - .put(MaxBidstreamLifetimeSecondsProp, 7200); - staticConfig = bootstrapConfig.copy() - .put(MaxBidstreamLifetimeSecondsProp, 7201); - - IConfigService dynamicConfigService = mock(IConfigService.class); - when(dynamicConfigService.getConfig()).thenReturn(bootstrapConfig); - IConfigService staticConfigService = mock(IConfigService.class); - when(staticConfigService.getConfig()).thenReturn(staticConfig); - - configServiceManager = new ConfigServiceManager(vertx, dynamicConfigService, staticConfigService, true); - } - - @Test - void testRemoteFeatureFlag(VertxTestContext testContext) { - IConfigService delegatingConfigService = configServiceManager.getDelegatingConfigService(); - - configServiceManager.updateConfigService(true) - .compose(updateToDynamic -> { - testContext.verify(() -> assertEquals(bootstrapConfig, delegatingConfigService.getConfig())); - - return configServiceManager.updateConfigService(false); - }) - .onSuccess(updateToStatic -> testContext.verify(() -> { - assertEquals(staticConfig, delegatingConfigService.getConfig()); - testContext.completeNow(); - })) - .onFailure(testContext::failNow); - } -} From fc6fdfbc46aa28b699f6940f8cc8ca2785795f02 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Wed, 26 Mar 2025 05:28:21 +0000 Subject: [PATCH 410/431] [CI Pipeline] Released Patch version: 5.50.9 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 489c584d5..012a463f3 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.50.6 + 5.50.9 UTF-8 From ab4da1543a53558566c41ad8b298854f5e1bdc50 Mon Sep 17 00:00:00 2001 From: Katherine Chen Date: Wed, 26 Mar 2025 16:38:01 +1100 Subject: [PATCH 411/431] Remove feature flag folder (#1613) --- Dockerfile | 1 - Makefile.eif | 3 --- scripts/azure-cc/Dockerfile | 1 - scripts/gcp-oidc/Dockerfile | 1 - 4 files changed, 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0d4a6eac5..db73539e2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,7 +18,6 @@ COPY ./target/${JAR_NAME}-${JAR_VERSION}-static.tar.gz /app/static.tar.gz COPY ./conf/default-config.json /app/conf/ COPY ./conf/*.xml /app/conf/ COPY ./conf/runtime-config-defaults.json /app/conf/ -COPY ./conf/feat-flag/feat-flag.json /app/conf/feat-flag/ RUN tar xzvf /app/static.tar.gz --no-same-owner --no-same-permissions && rm -f /app/static.tar.gz diff --git a/Makefile.eif b/Makefile.eif index 2b65069f5..bddcb0ff8 100644 --- a/Makefile.eif +++ b/Makefile.eif @@ -33,9 +33,6 @@ build_configs: build/conf/default-config.json build/conf/feat-flag.json build/co build/conf/default-config.json: build_artifacts ./scripts/aws/conf/default-config.json cp ./scripts/aws/conf/default-config.json ./build/conf/ -build/conf/feat-flag.json: build_artifacts ./scripts/aws/conf/feat-flag/feat-flag.json - cp ./scripts/aws/conf/feat-flag/feat-flag.json ./build/conf/ - build/conf/euid-integ-config.json: build_artifacts ./scripts/aws/conf/euid-integ-config.json cp ./scripts/aws/conf/euid-integ-config.json ./build/conf/ diff --git a/scripts/azure-cc/Dockerfile b/scripts/azure-cc/Dockerfile index c34661fa1..91b6c8096 100644 --- a/scripts/azure-cc/Dockerfile +++ b/scripts/azure-cc/Dockerfile @@ -33,7 +33,6 @@ COPY ./target/${JAR_NAME}-${JAR_VERSION}-sources.jar /app COPY ./target/${JAR_NAME}-${JAR_VERSION}-static.tar.gz /app/static.tar.gz COPY ./conf/*.json /app/conf/ COPY ./conf/*.xml /app/conf/ -COPY ./conf/feat-flag/feat-flag.json /app/conf/feat-flag/ # Extract and clean up tar.gz RUN tar xzvf /app/static.tar.gz --no-same-owner --no-same-permissions && \ diff --git a/scripts/gcp-oidc/Dockerfile b/scripts/gcp-oidc/Dockerfile index aa3ead08f..5320b7223 100644 --- a/scripts/gcp-oidc/Dockerfile +++ b/scripts/gcp-oidc/Dockerfile @@ -28,7 +28,6 @@ COPY ./target/${JAR_NAME}-${JAR_VERSION}-sources.jar /app COPY ./target/${JAR_NAME}-${JAR_VERSION}-static.tar.gz /app/static.tar.gz COPY ./conf/*.json /app/conf/ COPY ./conf/*.xml /app/conf/ -COPY ./conf/feat-flag/feat-flag.json /app/conf/feat-flag/ RUN tar xzvf /app/static.tar.gz --no-same-owner --no-same-permissions && rm -f /app/static.tar.gz From db2c7ad53e35a64f2c99541b59dda4cc7f8c19a8 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Wed, 26 Mar 2025 05:40:16 +0000 Subject: [PATCH 412/431] [CI Pipeline] Released Patch version: 5.50.12 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 012a463f3..1e526aaa9 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.50.9 + 5.50.12 UTF-8 From e3d94f6d9925c80f4ed83dd678a49f92f65265e7 Mon Sep 17 00:00:00 2001 From: Gian Miguel Del Mundo Date: Wed, 26 Mar 2025 17:43:53 +0800 Subject: [PATCH 413/431] Updated run E2E pipeline target description --- .github/workflows/run-e2e-tests-on-operator.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-e2e-tests-on-operator.yaml b/.github/workflows/run-e2e-tests-on-operator.yaml index 462a992e1..0c397a7a3 100644 --- a/.github/workflows/run-e2e-tests-on-operator.yaml +++ b/.github/workflows/run-e2e-tests-on-operator.yaml @@ -21,7 +21,7 @@ on: - UID2 - EUID target_environment: - description: PRIVATE OPERATORS ONLY - The target environment [mock, integ, prod] + description: For gcp, azure, aws (NOT public, aks) - The target environment [mock, integ, prod] required: true type: choice options: From 420f746dd7b49a7d777055a3836cbf8f50ac8252 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Wed, 26 Mar 2025 09:48:38 +0000 Subject: [PATCH 414/431] [CI Pipeline] Released Patch version: 5.50.17 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1e526aaa9..309d7174a 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.50.12 + 5.50.17 UTF-8 From a72250610927cce29c2532883de1012bb2699780 Mon Sep 17 00:00:00 2001 From: Gian Miguel Del Mundo Date: Wed, 26 Mar 2025 18:02:18 +0800 Subject: [PATCH 415/431] Fixed AWS Makefile --- Makefile.eif | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.eif b/Makefile.eif index bddcb0ff8..38e47c13c 100644 --- a/Makefile.eif +++ b/Makefile.eif @@ -28,7 +28,7 @@ euidoperator.eif: build_artifacts build_configs build/proxies.nitro.yaml build/s .PHONY: build_configs -build_configs: build/conf/default-config.json build/conf/feat-flag.json build/conf/euid-integ-config.json build/conf/euid-prod-config.json build/conf/uid2-integ-config.json build/conf/uid2-prod-config.json build/conf/logback.xml build/conf/logback-debug.xml +build_configs: build/conf/default-config.json build/conf/euid-integ-config.json build/conf/euid-prod-config.json build/conf/uid2-integ-config.json build/conf/uid2-prod-config.json build/conf/logback.xml build/conf/logback-debug.xml build/conf/default-config.json: build_artifacts ./scripts/aws/conf/default-config.json cp ./scripts/aws/conf/default-config.json ./build/conf/ From 59267185d6f2131eaf40b795ff3e29f8c65672b0 Mon Sep 17 00:00:00 2001 From: Gian Miguel Del Mundo Date: Wed, 26 Mar 2025 18:05:26 +0800 Subject: [PATCH 416/431] Removed feat-flag.json from AWS Dockerfile --- scripts/aws/Dockerfile | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/scripts/aws/Dockerfile b/scripts/aws/Dockerfile index 88e1a2242..67aa17368 100644 --- a/scripts/aws/Dockerfile +++ b/scripts/aws/Dockerfile @@ -33,11 +33,10 @@ COPY ./libjnsm.so /app/lib/ COPY ./vsockpx /app/ COPY ./entrypoint.sh /app/ COPY ./proxies.nitro.yaml /app/ -COPY ./conf/default-config.json /app/conf/ -COPY ./conf/*.json /app/conf/ -COPY ./conf/*.xml /app/conf/ -COPY ./syslog-ng-client.conf /etc/syslog-ng/syslog-ng.conf -COPY ./conf/feat-flag.json /app/conf/feat-flag/ +COPY ./conf/default-config.json /app/conf/ +COPY ./conf/*.json /app/conf/ +COPY ./conf/*.xml /app/conf/ +COPY ./syslog-ng-client.conf /etc/syslog-ng/syslog-ng.conf RUN chmod +x /app/vsockpx && chmod +x /app/entrypoint.sh From e9f98e0a23e2ca25f07c33e9e5afe2e021d4ef08 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Wed, 26 Mar 2025 10:07:42 +0000 Subject: [PATCH 417/431] [CI Pipeline] Released Patch version: 5.50.20 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 309d7174a..4e639f024 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.50.17 + 5.50.20 UTF-8 From 51e390f1b6f88cea7a4c8b36dd52a29818fc070b Mon Sep 17 00:00:00 2001 From: Gian Miguel Del Mundo Date: Wed, 26 Mar 2025 23:22:59 +0800 Subject: [PATCH 418/431] Reverted target_environment description change --- .github/workflows/run-e2e-tests-on-operator.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-e2e-tests-on-operator.yaml b/.github/workflows/run-e2e-tests-on-operator.yaml index 0c397a7a3..462a992e1 100644 --- a/.github/workflows/run-e2e-tests-on-operator.yaml +++ b/.github/workflows/run-e2e-tests-on-operator.yaml @@ -21,7 +21,7 @@ on: - UID2 - EUID target_environment: - description: For gcp, azure, aws (NOT public, aks) - The target environment [mock, integ, prod] + description: PRIVATE OPERATORS ONLY - The target environment [mock, integ, prod] required: true type: choice options: From 21908b809ad2fd0b30ee6981451bba43d23ad374 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Wed, 26 Mar 2025 18:43:25 +0000 Subject: [PATCH 419/431] [CI Pipeline] Released Patch version: 5.50.22 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 4e639f024..af4b1c780 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.50.20 + 5.50.22 UTF-8 From 0fc567f674cfc6b2164852f74ff56002377e90cc Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Thu, 27 Mar 2025 00:05:44 +0000 Subject: [PATCH 420/431] [CI Pipeline] Released patch version: 5.50.23 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index af4b1c780..d5a037c1a 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.50.22 + 5.50.23 UTF-8 From 1bb5f1ee89c4e7b8c19875c4e0229c871766b461 Mon Sep 17 00:00:00 2001 From: sophia chen Date: Thu, 27 Mar 2025 14:10:37 +1100 Subject: [PATCH 421/431] extended trivyignore for CVE-2024-8176 and CVE-2025-24970 --- .trivyignore | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.trivyignore b/.trivyignore index 2a67adbc2..dcb1e8d4b 100644 --- a/.trivyignore +++ b/.trivyignore @@ -6,7 +6,7 @@ CVE-2024-47535 # https://thetradedesk.atlassian.net/browse/UID2-4874 -CVE-2025-24970 exp:2025-03-27 +CVE-2025-24970 exp:2025-04-03 # https://thetradedesk.atlassian.net/browse/UID2-5186 -CVE-2024-8176 exp:2025-03-27 +CVE-2024-8176 exp:2025-04-03 From abc4ca3288d2e39f3ae69315fdf1d7a9e11f0ec8 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Fri, 28 Mar 2025 00:06:15 +0000 Subject: [PATCH 422/431] [CI Pipeline] Released patch version: 5.50.26 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d5a037c1a..59b81c870 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.50.23 + 5.50.26 UTF-8 From 81561a2530521de3ef831daf9668c5d78b0c440e Mon Sep 17 00:00:00 2001 From: Matt Collins Date: Fri, 28 Mar 2025 11:30:45 +1100 Subject: [PATCH 423/431] Fix AKS enclave ID generation - Add error checking. - Fix path to generate.py. - Some sed commands should have been on ${INPUT_DIR}/generated.rego. - Collapse multiple sed commands into one for simpler error checking. --- .../generate-deployment-artifacts.sh | 38 ++++++++++++++++--- 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/scripts/azure-aks/deployment/generate-deployment-artifacts.sh b/scripts/azure-aks/deployment/generate-deployment-artifacts.sh index 0b08b8a61..7a3b7db10 100644 --- a/scripts/azure-aks/deployment/generate-deployment-artifacts.sh +++ b/scripts/azure-aks/deployment/generate-deployment-artifacts.sh @@ -85,11 +85,37 @@ if [[ $? -ne 0 ]]; then fi base64 -di < ${INPUT_DIR}/policy.base64 > ${INPUT_DIR}/generated.rego -sed -i "s#allow_environment_variable_dropping := true#allow_environment_variable_dropping := false#g" ${INPUT_DIR}/generated.rego -sed -i 's#{"pattern":"DEPLOYMENT_ENVIRONMENT=DEPLOYMENT_ENVIRONMENT_PLACEHOLDER","required":false,"strategy":"string"}#{"pattern":"DEPLOYMENT_ENVIRONMENT=.+","required":false,"strategy":"re2"}#g' generated.rego -sed -i 's#{"pattern":"VAULT_NAME=VAULT_NAME_PLACEHOLDER","required":false,"strategy":"string"}#{"pattern":"VAULT_NAME=.+","required":false,"strategy":"re2"}#g' generated.rego -sed -i 's#{"pattern":"OPERATOR_KEY_SECRET_NAME=OPERATOR_KEY_SECRET_NAME_PLACEHOLDER","required":false,"strategy":"string"}#{"pattern":"OPERATOR_KEY_SECRET_NAME=.+","required":false,"strategy":"re2"}#g' generated.rego +if [[ $? -ne 0 ]]; then + echo "Failed to base64-decode policy" + exit 1 +fi + +sed --in-place \ + -e "s#allow_environment_variable_dropping := true#allow_environment_variable_dropping := false#g" \ + -e 's#{"pattern":"DEPLOYMENT_ENVIRONMENT=DEPLOYMENT_ENVIRONMENT_PLACEHOLDER","required":false,"strategy":"string"}#{"pattern":"DEPLOYMENT_ENVIRONMENT=.+","required":false,"strategy":"re2"}#g' \ + -e 's#{"pattern":"VAULT_NAME=VAULT_NAME_PLACEHOLDER","required":false,"strategy":"string"}#{"pattern":"VAULT_NAME=.+","required":false,"strategy":"re2"}#g' \ + -e 's#{"pattern":"OPERATOR_KEY_SECRET_NAME=OPERATOR_KEY_SECRET_NAME_PLACEHOLDER","required":false,"strategy":"string"}#{"pattern":"OPERATOR_KEY_SECRET_NAME=.+","required":false,"strategy":"re2"}#g' \ + ${INPUT_DIR}/generated.rego +if [[ $? -ne 0 ]]; then + echo "Failed to replace placeholders in policy file" + exit 1 +fi + base64 -w0 < ${INPUT_DIR}/generated.rego > ${INPUT_DIR}/generated.rego.base64 -python3 ${SCRIPT_DIR}/../azure-cc/generate.py ${INPUT_DIR}/generated.rego > ${MANIFEST_DIR}/${POLICY_DIGEST_FILE} +if [[ $? -ne 0 ]]; then + echo "Failed to base64-encode policy file" + exit 1 +fi + +python3 ${SCRIPT_DIR}/../../azure-cc/deployment/generate.py ${INPUT_DIR}/generated.rego > ${MANIFEST_DIR}/${POLICY_DIGEST_FILE} +if [[ $? -ne 0 ]]; then + echo "Failed to generate digest from policy file" + exit 1 +fi + +sed --in-place "s#CCE_POLICY_PLACEHOLDER#$(cat ${INPUT_DIR}/generated.rego.base64)#g" ${OUTPUT_DIR}/operator.yaml +if [[ $? -ne 0 ]]; then + echo "Failed to replace placeholder in operator.yaml" + exit 1 +fi -sed -i "s#CCE_POLICY_PLACEHOLDER#$(cat ${INPUT_DIR}/generated.rego.base64)#g" ${OUTPUT_DIR}/operator.yaml From e2d09ca1308f60edc39e5592d2421d6b9e9e6529 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Fri, 28 Mar 2025 01:01:18 +0000 Subject: [PATCH 424/431] [CI Pipeline] Released Snapshot version: 5.50.24-alpha-255-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d5a037c1a..f5714c47f 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.50.23 + 5.50.24-alpha-255-SNAPSHOT UTF-8 From bae310b8c6740a680fd628112bd3ad8b901753fc Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Sat, 29 Mar 2025 00:05:39 +0000 Subject: [PATCH 425/431] [CI Pipeline] Released patch version: 5.50.30 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 59b81c870..e4eab8073 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.50.26 + 5.50.30 UTF-8 From 410588c4d2ef258b3fda440036ab08776617bf51 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Sun, 30 Mar 2025 00:07:17 +0000 Subject: [PATCH 426/431] [CI Pipeline] Released patch version: 5.50.31 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e4eab8073..541c69943 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.50.30 + 5.50.31 UTF-8 From d21f2a65f8d8edfcc2ad0f87435851320488c3f8 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Mon, 31 Mar 2025 00:06:49 +0000 Subject: [PATCH 427/431] [CI Pipeline] Released patch version: 5.50.32 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 541c69943..773fabc8c 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.50.31 + 5.50.32 UTF-8 From f1f3095ee2487dcffbd44edf3ca218889d8573cd Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Tue, 1 Apr 2025 00:08:16 +0000 Subject: [PATCH 428/431] [CI Pipeline] Released patch version: 5.50.33 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 773fabc8c..80b8468f5 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.50.32 + 5.50.33 UTF-8 From de95e07b76318495dc26ea1327f1be701d80e6a9 Mon Sep 17 00:00:00 2001 From: abuabraham-ttd <130511456+abuabraham-ttd@users.noreply.github.com> Date: Tue, 1 Apr 2025 08:48:18 -0700 Subject: [PATCH 429/431] Re-adding encryption (#1558) * Re-adding encryption, flag to toggle. ---- Co-authored-by: Release Workflow Co-authored-by: Vishal Egbert --- conf/default-config.json | 3 + conf/docker-config.json | 2 + conf/integ-config.json | 1 + conf/local-config.json | 2 + conf/local-e2e-docker-private-config.json | 2 + conf/local-e2e-docker-public-config.json | 2 + conf/local-e2e-private-config.json | 2 + conf/local-e2e-public-config.json | 2 + ...dator-latest-e2e-docker-public-config.json | 2 + pom.xml | 2 +- scripts/aws/conf/euid-integ-config.json | 1 + scripts/aws/conf/euid-prod-config.json | 1 + scripts/aws/conf/uid2-integ-config.json | 1 + scripts/aws/conf/uid2-prod-config.json | 1 + scripts/azure-cc/conf/integ-uid2-config.json | 1 + scripts/azure-cc/conf/prod-uid2-config.json | 1 + scripts/gcp-oidc/conf/integ-config.json | 1 + scripts/gcp-oidc/conf/prod-config.json | 1 + src/main/java/com/uid2/operator/Const.java | 1 + src/main/java/com/uid2/operator/Main.java | 101 ++++++++++++----- .../uid2/operator/reader/ApiStoreReader.java | 58 ++++++++++ ...RotatingCloudEncryptionKeyApiProvider.java | 34 ++++++ .../cloud_encryption_keys.json | 73 ++++++++++++ .../test/cloud_encryption_keys/metadata.json | 7 ++ .../com/uid2/operator/ApiStoreReaderTest.java | 104 ++++++++++++++++++ ...tingCloudEncryptionKeyApiProviderTest.java | 103 +++++++++++++++++ 26 files changed, 482 insertions(+), 27 deletions(-) create mode 100644 src/main/java/com/uid2/operator/reader/ApiStoreReader.java create mode 100644 src/main/java/com/uid2/operator/reader/RotatingCloudEncryptionKeyApiProvider.java create mode 100644 src/main/resources/com.uid2.core/test/cloud_encryption_keys/cloud_encryption_keys.json create mode 100644 src/main/resources/com.uid2.core/test/cloud_encryption_keys/metadata.json create mode 100644 src/test/java/com/uid2/operator/ApiStoreReaderTest.java create mode 100644 src/test/java/com/uid2/operator/RotatingCloudEncryptionKeyApiProviderTest.java diff --git a/conf/default-config.json b/conf/default-config.json index d6a8e8a41..0683103a5 100644 --- a/conf/default-config.json +++ b/conf/default-config.json @@ -30,6 +30,9 @@ "salts_metadata_path": "salts/metadata.json", "services_metadata_path": "services/metadata.json", "service_links_metadata_path": "service_links/metadata.json", + "cloud_encryption_keys_metadata_path": "cloud_encryption_keys/metadata.json", + "encrypted_files": false, + "cloud_encryption_keys_refresh_ms": 300000, "optout_metadata_path": null, "optout_inmem_cache": false, "enclave_platform": null, diff --git a/conf/docker-config.json b/conf/docker-config.json index 3402673e5..6c376b5ea 100644 --- a/conf/docker-config.json +++ b/conf/docker-config.json @@ -31,6 +31,8 @@ "salts_metadata_path": "/com.uid2.core/test/salts/metadata.json", "services_metadata_path": "/com.uid2.core/test/services/metadata.json", "service_links_metadata_path": "/com.uid2.core/test/service_links/metadata.json", + "cloud_encryption_keys_metadata_path": "/com.uid2.core/test/cloud_encryption_keys/metadata.json", + "encrypted_files": true, "identity_token_expires_after_seconds": 3600, "optout_metadata_path": null, "optout_inmem_cache": false, diff --git a/conf/integ-config.json b/conf/integ-config.json index c29bfd4f0..b741cf2a3 100644 --- a/conf/integ-config.json +++ b/conf/integ-config.json @@ -13,6 +13,7 @@ "core_api_token": "trusted-partner-key", "optout_api_token": "test-operator-key", "optout_api_uri": "http://localhost:8081/optout/replicate", + "cloud_encryption_keys_metadata_path": "http://localhost:8088/cloud_encryption_keys/retrieve", "salts_expired_shutdown_hours": 12, "operator_type": "public", "runtime_config_store": { diff --git a/conf/local-config.json b/conf/local-config.json index 8cb729651..7c61e2cad 100644 --- a/conf/local-config.json +++ b/conf/local-config.json @@ -9,6 +9,7 @@ "salts_metadata_path": "/com.uid2.core/test/salts/metadata.json", "services_metadata_path": "/com.uid2.core/test/services/metadata.json", "service_links_metadata_path": "/com.uid2.core/test/service_links/metadata.json", + "cloud_encryption_keys_metadata_path": "/com.uid2.core/test/cloud_encryption_keys/metadata.json", "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, @@ -37,6 +38,7 @@ "client_side_token_generate_log_invalid_http_origins": true, "salts_expired_shutdown_hours": 12, "operator_type": "public", + "encrypted_files": true, "runtime_config_store": { "type": "file", "config" : { diff --git a/conf/local-e2e-docker-private-config.json b/conf/local-e2e-docker-private-config.json index d739b8798..87e5cd7a8 100644 --- a/conf/local-e2e-docker-private-config.json +++ b/conf/local-e2e-docker-private-config.json @@ -11,6 +11,8 @@ "keysets_metadata_path": "http://core:8088/key/keyset/refresh", "keyset_keys_metadata_path": "http://core:8088/key/keyset-keys/refresh", "salts_metadata_path": "http://core:8088/salt/refresh", + "cloud_encryption_keys_metadata_path": "http://core:8088/cloud_encryption_keys/retrieve", + "encrypted_files": false, "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, diff --git a/conf/local-e2e-docker-public-config.json b/conf/local-e2e-docker-public-config.json index 03cbb6597..60d5e287a 100644 --- a/conf/local-e2e-docker-public-config.json +++ b/conf/local-e2e-docker-public-config.json @@ -13,6 +13,8 @@ "salts_metadata_path": "http://core:8088/salt/refresh", "services_metadata_path": "http://core:8088/services/refresh", "service_links_metadata_path": "http://core:8088/service_links/refresh", + "cloud_encryption_keys_metadata_path": "http://core:8088/cloud_encryption_keys/retrieve", + "encrypted_files": false, "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, diff --git a/conf/local-e2e-private-config.json b/conf/local-e2e-private-config.json index 02f277df9..06b1ddb3a 100644 --- a/conf/local-e2e-private-config.json +++ b/conf/local-e2e-private-config.json @@ -13,6 +13,8 @@ "salts_metadata_path": "http://localhost:8088/salt/refresh", "services_metadata_path": "http://localhost:8088/services/refresh", "service_links_metadata_path": "http://localhost:8088/service_links/refresh", + "cloud_encryption_keys_metadata_path": "http://localhost:8088/cloud_encryption_keys/retrieve", + "encrypted_files": false, "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, diff --git a/conf/local-e2e-public-config.json b/conf/local-e2e-public-config.json index f8ee5a44c..6b5e0fc03 100644 --- a/conf/local-e2e-public-config.json +++ b/conf/local-e2e-public-config.json @@ -13,6 +13,8 @@ "salts_metadata_path": "http://localhost:8088/salt/refresh", "services_metadata_path": "http://localhost:8088/services/refresh", "service_links_metadata_path": "http://localhost:8088/service_links/refresh", + "cloud_encryption_keys_metadata_path": "http://localhost:8088/cloud_encryption_keys/retrieve", + "encrypted_files": false, "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, diff --git a/conf/validator-latest-e2e-docker-public-config.json b/conf/validator-latest-e2e-docker-public-config.json index 270143f07..38b6c2b28 100644 --- a/conf/validator-latest-e2e-docker-public-config.json +++ b/conf/validator-latest-e2e-docker-public-config.json @@ -14,6 +14,8 @@ "salts_metadata_path": "http://core:8088/salt/refresh", "services_metadata_path": "http://core:8088/services/refresh", "service_links_metadata_path": "http://core:8088/service_links/refresh", + "cloud_encryption_keys_metadata_path": "http://core:8088/cloud_encryption_keys/retrieve", + "encrypted_files": true, "identity_token_expires_after_seconds": 3600, "refresh_token_expires_after_seconds": 86400, "refresh_identity_token_after_seconds": 900, diff --git a/pom.xml b/pom.xml index 80b8468f5..9b8a0f7ac 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ 2.1.0 2.1.13 2.1.0 - 8.1.25 + 9.0.8 ${project.version} 21 21 diff --git a/scripts/aws/conf/euid-integ-config.json b/scripts/aws/conf/euid-integ-config.json index 651046136..0944b74e9 100644 --- a/scripts/aws/conf/euid-integ-config.json +++ b/scripts/aws/conf/euid-integ-config.json @@ -10,6 +10,7 @@ "optout_metadata_path": "https://optout.integ.euid.eu/optout/refresh", "core_attest_url": "https://core.integ.euid.eu/attest", "optout_api_uri": "https://optout.integ.euid.eu/optout/replicate", + "cloud_encryption_keys_metadata_path": "https://core.integ.euid.eu/cloud_encryption_keys/retrieve", "optout_s3_folder": "optout/", "allow_legacy_api": false, "identity_scope": "euid" diff --git a/scripts/aws/conf/euid-prod-config.json b/scripts/aws/conf/euid-prod-config.json index 9d19bc91b..e09b202f3 100644 --- a/scripts/aws/conf/euid-prod-config.json +++ b/scripts/aws/conf/euid-prod-config.json @@ -10,6 +10,7 @@ "service_links_metadata_path": "https://core.prod.euid.eu/service_links/refresh", "optout_metadata_path": "https://optout.prod.euid.eu/optout/refresh", "core_attest_url": "https://core.prod.euid.eu/attest", + "cloud_encryption_keys_metadata_path": "https://core.prod.euid.eu/cloud_encryption_keys/retrieve", "core_api_token": "your-api-token", "optout_s3_path_compat": false, "optout_api_uri": "https://optout.prod.euid.eu/optout/replicate", diff --git a/scripts/aws/conf/uid2-integ-config.json b/scripts/aws/conf/uid2-integ-config.json index a99b56813..3c267a655 100644 --- a/scripts/aws/conf/uid2-integ-config.json +++ b/scripts/aws/conf/uid2-integ-config.json @@ -10,6 +10,7 @@ "services_metadata_path": "https://core-integ.uidapi.com/services/refresh", "service_links_metadata_path": "https://core-integ.uidapi.com/service_links/refresh", "optout_metadata_path": "https://optout-integ.uidapi.com/optout/refresh", + "cloud_encryption_keys_metadata_path": "https://core-integ.uidapi.com/cloud_encryption_keys/retrieve", "optout_s3_folder": "uid-optout-integ/", "identity_scope": "uid2" } diff --git a/scripts/aws/conf/uid2-prod-config.json b/scripts/aws/conf/uid2-prod-config.json index d7a03d679..e143f098e 100644 --- a/scripts/aws/conf/uid2-prod-config.json +++ b/scripts/aws/conf/uid2-prod-config.json @@ -10,6 +10,7 @@ "service_links_metadata_path": "https://core-prod.uidapi.com/service_links/refresh", "optout_metadata_path": "https://optout-prod.uidapi.com/optout/refresh", "core_attest_url": "https://core-prod.uidapi.com/attest", + "cloud_encryption_keys_metadata_path": "https://core-prod.uidapi.com/cloud_encryption_keys/retrieve", "core_api_token": "your-api-token", "optout_s3_path_compat": false, "optout_api_uri": "https://optout-prod.uidapi.com/optout/replicate", diff --git a/scripts/azure-cc/conf/integ-uid2-config.json b/scripts/azure-cc/conf/integ-uid2-config.json index 569304e35..4a47711ee 100644 --- a/scripts/azure-cc/conf/integ-uid2-config.json +++ b/scripts/azure-cc/conf/integ-uid2-config.json @@ -10,6 +10,7 @@ "optout_metadata_path": "https://optout.uidapi.com/optout/refresh", "core_attest_url": "https://core.uidapi.com/attest", "optout_api_uri": "https://optout.uidapi.com/optout/replicate", + "cloud_encryption_keys_metadata_path": "https://core.uidapi.com/cloud_encryption_keys/retrieve", "optout_s3_folder": "uid-optout-integ/", "runtime_config_store": { "type": "http", diff --git a/scripts/azure-cc/conf/prod-uid2-config.json b/scripts/azure-cc/conf/prod-uid2-config.json index 9c4eba18a..232344504 100644 --- a/scripts/azure-cc/conf/prod-uid2-config.json +++ b/scripts/azure-cc/conf/prod-uid2-config.json @@ -9,6 +9,7 @@ "service_links_metadata_path": "https://core.uidapi.com/service_links/refresh", "optout_metadata_path": "https://optout.uidapi.com/optout/refresh", "core_attest_url": "https://core.uidapi.com/attest", + "cloud_encryption_keys_metadata_path": "https://core.uidapi.com/cloud_encryption_keys/retrieve", "optout_api_uri": "https://optout.uidapi.com/optout/replicate", "optout_s3_folder": "optout-v2/", "identity_token_expires_after_seconds": 259200, diff --git a/scripts/gcp-oidc/conf/integ-config.json b/scripts/gcp-oidc/conf/integ-config.json index e07aeea24..5d3882f25 100644 --- a/scripts/gcp-oidc/conf/integ-config.json +++ b/scripts/gcp-oidc/conf/integ-config.json @@ -9,6 +9,7 @@ "service_links_metadata_path": "https://core.uidapi.com/service_links/refresh", "optout_metadata_path": "https://optout.uidapi.com/optout/refresh", "core_attest_url": "https://core.uidapi.com/attest", + "cloud_encryption_keys_metadata_path": "https://core.uidapi.com/cloud_encryption_keys/retrieve", "optout_api_uri": "https://optout.uidapi.com/optout/replicate", "optout_s3_folder": "uid-optout-integ/", "runtime_config_store": { diff --git a/scripts/gcp-oidc/conf/prod-config.json b/scripts/gcp-oidc/conf/prod-config.json index 9c4eba18a..232344504 100644 --- a/scripts/gcp-oidc/conf/prod-config.json +++ b/scripts/gcp-oidc/conf/prod-config.json @@ -9,6 +9,7 @@ "service_links_metadata_path": "https://core.uidapi.com/service_links/refresh", "optout_metadata_path": "https://optout.uidapi.com/optout/refresh", "core_attest_url": "https://core.uidapi.com/attest", + "cloud_encryption_keys_metadata_path": "https://core.uidapi.com/cloud_encryption_keys/retrieve", "optout_api_uri": "https://optout.uidapi.com/optout/replicate", "optout_s3_folder": "optout-v2/", "identity_token_expires_after_seconds": 259200, diff --git a/src/main/java/com/uid2/operator/Const.java b/src/main/java/com/uid2/operator/Const.java index bd2ba2987..d2ed93afd 100644 --- a/src/main/java/com/uid2/operator/Const.java +++ b/src/main/java/com/uid2/operator/Const.java @@ -20,6 +20,7 @@ public class Config extends com.uid2.shared.Const.Config { public static final String ValidateServiceLinks = "validate_service_links"; public static final String OperatorTypeProp = "operator_type"; public static final String EnclavePlatformProp = "enclave_platform"; + public static final String EncryptedFiles = "encrypted_files"; public static final String AzureVaultNameProp = "azure_vault_name"; public static final String AzureSecretNameProp = "azure_secret_name"; diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index 80702c1fa..d1a66856d 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -8,6 +8,7 @@ import com.uid2.operator.monitoring.IStatsCollectorQueue; import com.uid2.operator.monitoring.OperatorMetrics; import com.uid2.operator.monitoring.StatsCollectorVerticle; +import com.uid2.operator.reader.RotatingCloudEncryptionKeyApiProvider; import com.uid2.operator.service.*; import com.uid2.operator.vertx.Endpoints; import com.uid2.operator.vertx.OperatorShutdownHandler; @@ -21,6 +22,7 @@ import com.uid2.shared.jmx.AdminApi; import com.uid2.shared.optout.OptOutCloudSync; import com.uid2.shared.store.CloudPath; +import com.uid2.shared.store.EncryptedRotatingSaltProvider; import com.uid2.shared.store.RotatingSaltProvider; import com.uid2.shared.store.reader.*; import com.uid2.shared.store.scope.GlobalScope; @@ -76,6 +78,7 @@ public class Main { private final RotatingClientSideKeypairStore clientSideKeypairProvider; private final RotatingSaltProvider saltProvider; private final CloudSyncOptOutStore optOutStore; + private final boolean encryptedCloudFilesEnabled; private OperatorShutdownHandler shutdownHandler = null; private final OperatorMetrics metrics; private final boolean clientSideTokenGenerate; @@ -83,6 +86,7 @@ public class Main { private IStatsCollectorQueue _statsCollectorQueue; private RotatingServiceStore serviceProvider; private RotatingServiceLinkStore serviceLinkProvider; + private RotatingCloudEncryptionKeyApiProvider cloudEncryptionKeyProvider; public Main(Vertx vertx, JsonObject config) throws Exception { this.vertx = vertx; @@ -100,6 +104,7 @@ public Main(Vertx vertx, JsonObject config) throws Exception { boolean useStorageMock = config.getBoolean(Const.Config.StorageMockProp, false); this.clientSideTokenGenerate = config.getBoolean(Const.Config.EnableClientSideTokenGenerate, false); this.validateServiceLinks = config.getBoolean(Const.Config.ValidateServiceLinks, false); + this.encryptedCloudFilesEnabled = config.getBoolean(Const.Config.EncryptedFiles, false); this.shutdownHandler = new OperatorShutdownHandler(Duration.ofHours(12), Duration.ofHours(config.getInteger(Const.Config.SaltsExpiredShutdownHours, 12)), Clock.systemUTC(), new ShutdownService()); String coreAttestUrl = this.config.getString(Const.Config.CoreAttestUrlProp); @@ -134,17 +139,46 @@ public Main(Vertx vertx, JsonObject config) throws Exception { this.fsOptOut = configureCloudOptOutStore(); } - String sitesMdPath = this.config.getString(Const.Config.SitesMetadataPathProp); - String keypairMdPath = this.config.getString(Const.Config.ClientSideKeypairsMetadataPathProp); - this.clientSideKeypairProvider = new RotatingClientSideKeypairStore(fsStores, new GlobalScope(new CloudPath(keypairMdPath))); - String clientsMdPath = this.config.getString(Const.Config.ClientsMetadataPathProp); - this.clientKeyProvider = new RotatingClientKeyProvider(fsStores, new GlobalScope(new CloudPath(clientsMdPath))); - String keysetKeysMdPath = this.config.getString(Const.Config.KeysetKeysMetadataPathProp); - this.keysetKeyStore = new RotatingKeysetKeyStore(fsStores, new GlobalScope(new CloudPath(keysetKeysMdPath))); - String keysetMdPath = this.config.getString(Const.Config.KeysetsMetadataPathProp); - this.keysetProvider = new RotatingKeysetProvider(fsStores, new GlobalScope(new CloudPath(keysetMdPath))); - String saltsMdPath = this.config.getString(Const.Config.SaltsMetadataPathProp); - this.saltProvider = new RotatingSaltProvider(fsStores, saltsMdPath); + if (this.encryptedCloudFilesEnabled) { + String cloudEncryptionKeyMdPath = this.config.getString(Const.Config.CloudEncryptionKeysMetadataPathProp); + this.cloudEncryptionKeyProvider = new RotatingCloudEncryptionKeyApiProvider(fsStores, + new GlobalScope(new CloudPath(cloudEncryptionKeyMdPath))); + + String keypairMdPath = this.config.getString(Const.Config.ClientSideKeypairsMetadataPathProp); + this.clientSideKeypairProvider = new RotatingClientSideKeypairStore(fsStores, + new GlobalScope(new CloudPath(keypairMdPath)), cloudEncryptionKeyProvider); + String clientsMdPath = this.config.getString(Const.Config.ClientsMetadataPathProp); + this.clientKeyProvider = new RotatingClientKeyProvider(fsStores, new GlobalScope(new CloudPath(clientsMdPath)), + cloudEncryptionKeyProvider); + String keysetKeysMdPath = this.config.getString(Const.Config.KeysetKeysMetadataPathProp); + this.keysetKeyStore = new RotatingKeysetKeyStore(fsStores, new GlobalScope(new CloudPath(keysetKeysMdPath)), + cloudEncryptionKeyProvider); + String keysetMdPath = this.config.getString(Const.Config.KeysetsMetadataPathProp); + this.keysetProvider = new RotatingKeysetProvider(fsStores, new GlobalScope(new CloudPath(keysetMdPath)), + cloudEncryptionKeyProvider); + String saltsMdPath = this.config.getString(Const.Config.SaltsMetadataPathProp); + this.saltProvider = new EncryptedRotatingSaltProvider(fsStores, cloudEncryptionKeyProvider, + new GlobalScope(new CloudPath(saltsMdPath))); + String sitesMdPath = this.config.getString(Const.Config.SitesMetadataPathProp); + this.siteProvider = clientSideTokenGenerate + ? new RotatingSiteStore(fsStores, new GlobalScope(new CloudPath(sitesMdPath)), + cloudEncryptionKeyProvider) + : null; + } else { + String keypairMdPath = this.config.getString(Const.Config.ClientSideKeypairsMetadataPathProp); + this.clientSideKeypairProvider = new RotatingClientSideKeypairStore(fsStores, new GlobalScope(new CloudPath(keypairMdPath))); + String clientsMdPath = this.config.getString(Const.Config.ClientsMetadataPathProp); + this.clientKeyProvider = new RotatingClientKeyProvider(fsStores, new GlobalScope(new CloudPath(clientsMdPath))); + String keysetKeysMdPath = this.config.getString(Const.Config.KeysetKeysMetadataPathProp); + this.keysetKeyStore = new RotatingKeysetKeyStore(fsStores, new GlobalScope(new CloudPath(keysetKeysMdPath))); + String keysetMdPath = this.config.getString(Const.Config.KeysetsMetadataPathProp); + this.keysetProvider = new RotatingKeysetProvider(fsStores, new GlobalScope(new CloudPath(keysetMdPath))); + String saltsMdPath = this.config.getString(Const.Config.SaltsMetadataPathProp); + this.saltProvider = new RotatingSaltProvider(fsStores, saltsMdPath); + String sitesMdPath = this.config.getString(Const.Config.SitesMetadataPathProp); + this.siteProvider = clientSideTokenGenerate ? new RotatingSiteStore(fsStores, new GlobalScope(new CloudPath(sitesMdPath))) : null; + } + this.optOutStore = new CloudSyncOptOutStore(vertx, fsLocal, this.config, operatorKey, Clock.systemUTC()); if (this.validateServiceLinks) { @@ -154,23 +188,26 @@ public Main(Vertx vertx, JsonObject config) throws Exception { this.serviceLinkProvider = new RotatingServiceLinkStore(fsStores, new GlobalScope(new CloudPath(serviceLinkMdPath))); } - this.siteProvider = clientSideTokenGenerate ? new RotatingSiteStore(fsStores, new GlobalScope(new CloudPath(sitesMdPath))) : null; - if (useStorageMock && coreAttestUrl == null) { if (clientSideTokenGenerate) { this.siteProvider.loadContent(); this.clientSideKeypairProvider.loadContent(); } - this.clientKeyProvider.loadContent(); - this.saltProvider.loadContent(); - this.keysetProvider.loadContent(); - this.keysetKeyStore.loadContent(); if (this.validateServiceLinks) { this.serviceProvider.loadContent(); this.serviceLinkProvider.loadContent(); } + if (this.encryptedCloudFilesEnabled) { + this.cloudEncryptionKeyProvider.loadContent(); + } + + this.clientKeyProvider.loadContent(); + this.saltProvider.loadContent(); + this.keysetProvider.loadContent(); + this.keysetKeyStore.loadContent(); + try { getKeyManager().getMasterKey(); } catch (KeyManager.NoActiveKeyException e) { @@ -348,16 +385,21 @@ private Future createStoreVerticles() throws Exception { siteProvider.getMetadata(); clientSideKeypairProvider.getMetadata(); } - clientKeyProvider.getMetadata(); - keysetKeyStore.getMetadata(); - keysetProvider.getMetadata(); - saltProvider.getMetadata(); if (validateServiceLinks) { serviceProvider.getMetadata(); serviceLinkProvider.getMetadata(); } + if (encryptedCloudFilesEnabled) { + cloudEncryptionKeyProvider.getMetadata(); + } + + clientKeyProvider.getMetadata(); + keysetKeyStore.getMetadata(); + keysetProvider.getMetadata(); + saltProvider.getMetadata(); + // create cloud sync for optout store OptOutCloudSync optOutCloudSync = new OptOutCloudSync(config, false); this.optOutStore.registerCloudSync(optOutCloudSync); @@ -365,10 +407,21 @@ private Future createStoreVerticles() throws Exception { // create rotating store verticles to poll for updates Promise promise = Promise.promise(); List fs = new ArrayList<>(); + if (clientSideTokenGenerate) { fs.add(createAndDeployRotatingStoreVerticle("site", siteProvider, "site_refresh_ms")); fs.add(createAndDeployRotatingStoreVerticle("client_side_keypairs", clientSideKeypairProvider, "client_side_keypairs_refresh_ms")); } + + if (validateServiceLinks) { + fs.add(createAndDeployRotatingStoreVerticle("service", serviceProvider, "service_refresh_ms")); + fs.add(createAndDeployRotatingStoreVerticle("service_link", serviceLinkProvider, "service_link_refresh_ms")); + } + + if (encryptedCloudFilesEnabled) { + fs.add(createAndDeployRotatingStoreVerticle("cloud_encryption_keys", cloudEncryptionKeyProvider, "cloud_encryption_keys_refresh_ms")); + } + fs.add(createAndDeployRotatingStoreVerticle("auth", clientKeyProvider, "auth_refresh_ms")); fs.add(createAndDeployRotatingStoreVerticle("keyset", keysetProvider, "keyset_refresh_ms")); fs.add(createAndDeployRotatingStoreVerticle("keysetkey", keysetKeyStore, "keysetkey_refresh_ms")); @@ -379,10 +432,6 @@ private Future createStoreVerticles() throws Exception { else promise.complete(); }); - if (validateServiceLinks) { - fs.add(createAndDeployRotatingStoreVerticle("service", serviceProvider, "service_refresh_ms")); - fs.add(createAndDeployRotatingStoreVerticle("service_link", serviceLinkProvider, "service_link_refresh_ms")); - } return promise.future(); } @@ -517,7 +566,7 @@ private void createVertxEventLoopsMetric() { private Map.Entry createUidClients(Vertx vertx, String attestationUrl, String clientApiToken, Handler> responseWatcher) throws Exception { AttestationResponseHandler attestationResponseHandler = getAttestationTokenRetriever(vertx, attestationUrl, clientApiToken, responseWatcher); - UidCoreClient coreClient = new UidCoreClient(clientApiToken, CloudUtils.defaultProxy, attestationResponseHandler); + UidCoreClient coreClient = new UidCoreClient(clientApiToken, CloudUtils.defaultProxy, attestationResponseHandler, this.encryptedCloudFilesEnabled); UidOptOutClient optOutClient = new UidOptOutClient(clientApiToken, CloudUtils.defaultProxy, attestationResponseHandler); return new AbstractMap.SimpleEntry<>(coreClient, optOutClient); } diff --git a/src/main/java/com/uid2/operator/reader/ApiStoreReader.java b/src/main/java/com/uid2/operator/reader/ApiStoreReader.java new file mode 100644 index 000000000..7b39c9a83 --- /dev/null +++ b/src/main/java/com/uid2/operator/reader/ApiStoreReader.java @@ -0,0 +1,58 @@ +package com.uid2.operator.reader; + +import com.uid2.shared.cloud.DownloadCloudStorage; +import com.uid2.shared.store.ScopedStoreReader; +import com.uid2.shared.store.parser.Parser; +import com.uid2.shared.store.parser.ParsingResult; +import com.uid2.shared.store.scope.StoreScope; +import io.vertx.core.json.JsonArray; +import io.vertx.core.json.JsonObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; + +public class ApiStoreReader extends ScopedStoreReader { + private static final Logger LOGGER = LoggerFactory.getLogger(ApiStoreReader.class); + + public ApiStoreReader(DownloadCloudStorage fileStreamProvider, StoreScope scope, Parser parser, String dataTypeName) { + super(fileStreamProvider, scope, parser, dataTypeName); + } + + + public long loadContent(JsonObject contents) throws Exception { + return loadContent(contents, dataTypeName); + } + + @Override + public long loadContent(JsonObject contents, String dataType) throws IOException { + if (contents == null) { + throw new IllegalArgumentException(String.format("No contents provided for loading data type %s, cannot load content", dataType)); + } + + try { + JsonArray dataArray = contents.getJsonArray(dataType); + if (dataArray == null) { + throw new IllegalArgumentException(String.format("No array of type: %s, found in the contents", dataType)); + } + + String jsonString = dataArray.toString(); + InputStream inputStream = new ByteArrayInputStream(jsonString.getBytes(StandardCharsets.UTF_8)); + + ParsingResult parsed = parser.deserialize(inputStream); + latestSnapshot.set(parsed.getData()); + + final int count = parsed.getCount(); + latestEntryCount.set(count); + LOGGER.info(String.format("Loaded %d %s", count, dataType)); + return count; + } catch (Exception e) { + LOGGER.error(String.format("Unable to load %s", dataType)); + throw e; + } + } +} + diff --git a/src/main/java/com/uid2/operator/reader/RotatingCloudEncryptionKeyApiProvider.java b/src/main/java/com/uid2/operator/reader/RotatingCloudEncryptionKeyApiProvider.java new file mode 100644 index 000000000..838bd8b0b --- /dev/null +++ b/src/main/java/com/uid2/operator/reader/RotatingCloudEncryptionKeyApiProvider.java @@ -0,0 +1,34 @@ +package com.uid2.operator.reader; + +import com.uid2.shared.cloud.DownloadCloudStorage; +import com.uid2.shared.model.CloudEncryptionKey; +import com.uid2.shared.store.CloudPath; +import com.uid2.shared.store.parser.CloudEncryptionKeyParser; +import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider; +import com.uid2.shared.store.scope.StoreScope; +import io.vertx.core.json.JsonObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Instant; +import java.util.*; + +public class RotatingCloudEncryptionKeyApiProvider extends RotatingCloudEncryptionKeyProvider { + private static final Logger LOGGER = LoggerFactory.getLogger(RotatingCloudEncryptionKeyApiProvider.class); + + public RotatingCloudEncryptionKeyApiProvider(DownloadCloudStorage fileStreamProvider, StoreScope scope) { + super(fileStreamProvider, scope, new ApiStoreReader<>(fileStreamProvider, scope, new CloudEncryptionKeyParser(), "cloud_encryption_keys")); + } + + public RotatingCloudEncryptionKeyApiProvider(DownloadCloudStorage fileStreamProvider, StoreScope scope, ApiStoreReader> reader) { + super(fileStreamProvider, scope, reader); + } + + + @Override + public long getVersion(JsonObject metadata) { + // Since we are pulling from an api not a data file, we use the epoch time we got the keys as the version + return Instant.now().getEpochSecond(); + } +} + diff --git a/src/main/resources/com.uid2.core/test/cloud_encryption_keys/cloud_encryption_keys.json b/src/main/resources/com.uid2.core/test/cloud_encryption_keys/cloud_encryption_keys.json new file mode 100644 index 000000000..a9134f518 --- /dev/null +++ b/src/main/resources/com.uid2.core/test/cloud_encryption_keys/cloud_encryption_keys.json @@ -0,0 +1,73 @@ +[ { + "id" : 1, + "siteId" : 999, + "activates" : 1720641670, + "created" : 1720641670, + "secret" : "mydrCudb2PZOm01Qn0SpthltmexHUAA11Hy1m+uxjVw=" +}, { + "id" : 2, + "siteId" : 999, + "activates" : 1720728070, + "created" : 1720641670, + "secret" : "FtdslrFSsvVXOuhOWGwEI+0QTkCvM8SGZAP3k2u3PgY=" +}, { + "id" : 3, + "siteId" : 999, + "activates" : 1720814470, + "created" : 1720641670, + "secret" : "/7zO6QbKrhZKIV36G+cU9UR4hZUVg5bD+KjbczICjHw=" +}, { + "id" : 4, + "siteId" : 123, + "activates" : 1720641671, + "created" : 1720641671, + "secret" : "XjiqRlWQQJGLr7xfV1qbueKwyzt881GVohuUkQt/ht4=" +}, { + "id" : 5, + "siteId" : 123, + "activates" : 1720728071, + "created" : 1720641671, + "secret" : "QmpIf5NzO+UROjl5XjB/BmF6paefM8n6ub9B2plC9aI=" +}, { + "id" : 6, + "siteId" : 123, + "activates" : 1720814471, + "created" : 1720641671, + "secret" : "40w9UMSYxGm+KldOWOXhBGI8QgjvUUQjivtkP4VpKV8=" +}, { + "id" : 7, + "siteId" : 124, + "activates" : 1720641671, + "created" : 1720641671, + "secret" : "QdwD0kQV1BwmLRD0PH1YpqgaOrgpVTfu08o98mSZ6uE=" +}, { + "id" : 8, + "siteId" : 124, + "activates" : 1720728071, + "created" : 1720641671, + "secret" : "yCVCM/HLf9/6k+aUNrx7w17VbyfSzI8JykLQLSR+CW0=" +}, { + "id" : 9, + "siteId" : 124, + "activates" : 1720814471, + "created" : 1720641671, + "secret" : "JqHl8BrTyx9XpR2lYj/5xvUpzgnibGeomETTwF4rn1U=" +}, { + "id" : 10, + "siteId" : 127, + "activates" : 1720641671, + "created" : 1720641671, + "secret" : "JqiG1b34AvrdO3Aj6cCcjOBJMijrDzTmrR+p9ZtP2es=" +}, { + "id" : 11, + "siteId" : 127, + "activates" : 1720728072, + "created" : 1720641672, + "secret" : "lp1CyHdfc7K0aO5JGpA+Ve5Z/V5LImtGEQwCg/YB0kY=" +}, { + "id" : 12, + "siteId" : 127, + "activates" : 1720814472, + "created" : 1720641672, + "secret" : "G99rFYJF+dnSlk/xG6fuC3WNqQxTLJbDIdVyPMbGQ6s=" +} ] diff --git a/src/main/resources/com.uid2.core/test/cloud_encryption_keys/metadata.json b/src/main/resources/com.uid2.core/test/cloud_encryption_keys/metadata.json new file mode 100644 index 000000000..6ca4c52f0 --- /dev/null +++ b/src/main/resources/com.uid2.core/test/cloud_encryption_keys/metadata.json @@ -0,0 +1,7 @@ +{ + "version": 1, + "generated": 1620253519, + "cloud_encryption_keys": { + "location": "/com.uid2.core/test/cloud_encryption_keys/cloud_encryption_keys.json" + } +} diff --git a/src/test/java/com/uid2/operator/ApiStoreReaderTest.java b/src/test/java/com/uid2/operator/ApiStoreReaderTest.java new file mode 100644 index 000000000..8aba38250 --- /dev/null +++ b/src/test/java/com/uid2/operator/ApiStoreReaderTest.java @@ -0,0 +1,104 @@ +package com.uid2.operator; + +import com.uid2.operator.reader.ApiStoreReader; +import com.uid2.shared.cloud.DownloadCloudStorage; +import com.uid2.shared.store.CloudPath; +import com.uid2.shared.store.parser.Parser; +import com.uid2.shared.store.parser.ParsingResult; +import com.uid2.shared.store.scope.GlobalScope; +import io.vertx.core.json.JsonArray; +import io.vertx.core.json.JsonObject; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +import java.io.InputStream; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + + class ApiStoreReaderTest { + + @Mock + private DownloadCloudStorage mockStorage; + + @Mock + private Parser> mockParser; + + private final CloudPath metadataPath = new CloudPath("test/test-metadata.json"); + private final String dataType = "test-data-type"; + private final GlobalScope scope = new GlobalScope(metadataPath); + + private ApiStoreReader> reader; + + @BeforeEach + void setUp() { + MockitoAnnotations.openMocks(this); + reader = new ApiStoreReader<>(mockStorage, scope, mockParser, dataType); + } + + @Test + void getMetadataPathReturnsPathFromScope() { + CloudPath actual = reader.getMetadataPath(); + assertThat(actual).isEqualTo(metadataPath); + } + + @Test + void loadContentThrowsExceptionWhenContentsAreNull() { + assertThatThrownBy(() -> reader.loadContent(null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("No contents provided for loading data type"); + } + + @Test + void loadContentThrowsExceptionWhenArrayNotFound() { + JsonObject contents = new JsonObject(); + assertThatThrownBy(() -> reader.loadContent(contents)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("No array of type: test-data-type, found in the contents"); + } + + @Test + void loadContentSuccessfullyLoadsData() throws Exception { + JsonObject contents = new JsonObject() + .put(dataType, new JsonArray().add("value1").add("value2")); + + List expectedData = Arrays.asList(new TestData("value1"), new TestData("value2")); + when(mockParser.deserialize(any(InputStream.class))) + .thenReturn(new ParsingResult<>(expectedData, expectedData.size())); + + long count = reader.loadContent(contents); + + assertThat(count).isEqualTo(2); + assertThat(reader.getSnapshot()).isEqualTo(expectedData); + } + + private static class TestData { + private final String value; + + TestData(String value) { + this.value = value; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + TestData testData = (TestData) o; + return value.equals(testData.value); + } + + @Override + public int hashCode() { + return value.hashCode(); + } + } + } + + diff --git a/src/test/java/com/uid2/operator/RotatingCloudEncryptionKeyApiProviderTest.java b/src/test/java/com/uid2/operator/RotatingCloudEncryptionKeyApiProviderTest.java new file mode 100644 index 000000000..bfdd6fa17 --- /dev/null +++ b/src/test/java/com/uid2/operator/RotatingCloudEncryptionKeyApiProviderTest.java @@ -0,0 +1,103 @@ +package com.uid2.operator; + +import com.uid2.operator.reader.ApiStoreReader; +import com.uid2.operator.reader.RotatingCloudEncryptionKeyApiProvider; +import com.uid2.shared.cloud.DownloadCloudStorage; +import com.uid2.shared.model.CloudEncryptionKey; +import com.uid2.shared.store.CloudPath; +import com.uid2.shared.store.scope.StoreScope; +import io.vertx.core.json.JsonObject; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +import java.util.HashMap; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; + +class RotatingCloudEncryptionKeyApiProviderTest { + + @Mock + private DownloadCloudStorage mockFileStreamProvider; + + @Mock + private StoreScope mockScope; + + @Mock + private ApiStoreReader> mockApiStoreReader; + + private RotatingCloudEncryptionKeyApiProvider rotatingCloudEncryptionKeyApiProvider; + + @BeforeEach + void setUp() { + MockitoAnnotations.openMocks(this); + rotatingCloudEncryptionKeyApiProvider = new RotatingCloudEncryptionKeyApiProvider(mockFileStreamProvider, mockScope, mockApiStoreReader); + } + + @Test + void testGetMetadata() throws Exception { + JsonObject expectedMetadata = new JsonObject().put("version", 1L); + when(mockApiStoreReader.getMetadata()).thenReturn(expectedMetadata); + + JsonObject metadata = rotatingCloudEncryptionKeyApiProvider.getMetadata(); + assertEquals(expectedMetadata, metadata); + verify(mockApiStoreReader).getMetadata(); + } + + @Test + void testGetMetadataPath() { + CloudPath expectedPath = new CloudPath("test/path"); + when(mockApiStoreReader.getMetadataPath()).thenReturn(expectedPath); + + CloudPath path = rotatingCloudEncryptionKeyApiProvider.getMetadataPath(); + assertEquals(expectedPath, path); + verify(mockApiStoreReader).getMetadataPath(); + } + + @Test + void testLoadContentWithMetadata() throws Exception { + JsonObject metadata = new JsonObject(); + when(mockApiStoreReader.loadContent(metadata, "cloud_encryption_keys")).thenReturn(1L); + + long version = rotatingCloudEncryptionKeyApiProvider.loadContent(metadata); + assertEquals(1L, version); + verify(mockApiStoreReader).loadContent(metadata, "cloud_encryption_keys"); + } + + @Test + void testGetAll() { + Map expectedKeys = new HashMap<>(); + CloudEncryptionKey key = new CloudEncryptionKey(1, 123, 1687635529, 1687808329, "secret"); + expectedKeys.put(1, key); + when(mockApiStoreReader.getSnapshot()).thenReturn(expectedKeys); + + Map keys = rotatingCloudEncryptionKeyApiProvider.getAll(); + assertEquals(expectedKeys, keys); + verify(mockApiStoreReader).getSnapshot(); + } + + @Test + void testGetAllWithNullSnapshot() { + when(mockApiStoreReader.getSnapshot()).thenReturn(null); + + Map keys = rotatingCloudEncryptionKeyApiProvider.getAll(); + assertNotNull(keys); + assertTrue(keys.isEmpty()); + verify(mockApiStoreReader).getSnapshot(); + } + + @Test + void testLoadContent() throws Exception { + JsonObject metadata = new JsonObject().put("version", 1L); + when(mockApiStoreReader.getMetadata()).thenReturn(metadata); + when(mockApiStoreReader.loadContent(metadata, "cloud_encryption_keys")).thenReturn(1L); + + rotatingCloudEncryptionKeyApiProvider.loadContent(); + verify(mockApiStoreReader).getMetadata(); + verify(mockApiStoreReader).loadContent(metadata, "cloud_encryption_keys"); + } +} + From af092564ff878dbecbbacea5cfdeb5f97d42e533 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Tue, 1 Apr 2025 20:49:52 +0000 Subject: [PATCH 430/431] [CI Pipeline] Released Patch version: 5.50.35 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9b8a0f7ac..1d6aeaefc 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.50.33 + 5.50.35 UTF-8 From fe174ef6066a91cbb2bd5f652c0619640174bfa2 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Wed, 2 Apr 2025 00:05:53 +0000 Subject: [PATCH 431/431] [CI Pipeline] Released patch version: 5.50.36 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1d6aeaefc..15e312bcf 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-operator - 5.50.35 + 5.50.36 UTF-8