diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index be9ba8923..9d7655c95 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -41,6 +41,7 @@ jobs: TEST_OIDC_READ_CLIENT_ID: ${{ secrets.TEST_OIDC_READ_CLIENT_ID }} TEST_OIDC_READ_CLIENT_SECRET: ${{ secrets.TEST_OIDC_READ_CLIENT_SECRET }} TEST_OIDC_READ_TENANT: ${{ secrets.TEST_OIDC_READ_TENANT }} + TEST_OIDC_READ_PROJECT: "publicdata" TEST_CLIENT_ID_BLUEFIELD: ${{ secrets.BLUEFIELD_CLIENT_ID }} TEST_CLIENT_SECRET_BLUEFIELD: ${{ secrets.BLUEFIELD_CLIENT_SECRET }} TEST_AAD_TENANT_BLUEFIELD: "b86328db-09aa-4f0e-9a03-0136f604d20a" diff --git a/build.sbt b/build.sbt index cb74388d5..c74c92db7 100644 --- a/build.sbt +++ b/build.sbt @@ -9,7 +9,7 @@ val sparkVersion = "3.3.3" val circeVersion = "0.14.6" val sttpVersion = "3.5.2" val Specs2Version = "4.20.2" -val cogniteSdkVersion = "2.8.763" +val cogniteSdkVersion = "2.9.764" val prometheusVersion = "0.16.0" val log4sVersion = "1.10.0" diff --git a/src/main/scala/cognite/spark/v1/DefaultSource.scala b/src/main/scala/cognite/spark/v1/DefaultSource.scala index 326214bb9..3e5b55ac6 100644 --- a/src/main/scala/cognite/spark/v1/DefaultSource.scala +++ b/src/main/scala/cognite/spark/v1/DefaultSource.scala @@ -346,14 +346,7 @@ object DefaultSource { } clientId <- parameters.get("clientId") clientSecret <- parameters.get("clientSecret") - project <- parameters.get("project") - clientCredentials = OAuth2.ClientCredentials( - tokenUri, - clientId, - clientSecret, - scopes, - project, - audience) + clientCredentials = OAuth2.ClientCredentials(tokenUri, clientId, clientSecret, scopes, audience) } yield CdfSparkAuth.OAuth2ClientCredentials(clientCredentials) val session = for { diff --git a/src/test/scala/cognite/spark/v1/DefaultSourceTest.scala b/src/test/scala/cognite/spark/v1/DefaultSourceTest.scala index 22794c411..a4ad592a1 100644 --- a/src/test/scala/cognite/spark/v1/DefaultSourceTest.scala +++ b/src/test/scala/cognite/spark/v1/DefaultSourceTest.scala @@ -75,7 +75,6 @@ class DefaultSourceTest extends WordSpec with Matchers { "value-ClientId", "value-ClientSecret", List("value-Scopes"), - "value-Project", Some("value-Audience"))) ) } diff --git a/src/test/scala/cognite/spark/v1/SdkV1RddTest.scala b/src/test/scala/cognite/spark/v1/SdkV1RddTest.scala index 881e059fc..66779ef30 100644 --- a/src/test/scala/cognite/spark/v1/SdkV1RddTest.scala +++ b/src/test/scala/cognite/spark/v1/SdkV1RddTest.scala @@ -33,7 +33,7 @@ class SdkV1RddTest extends FlatSpec with Matchers with ParallelTestExecution wit val sdkRdd = { val relationConfig = getDefaultConfig( CdfSparkAuth.OAuth2ClientCredentials(readOidcCredentials), - readOidcCredentials.cdfProjectName + readProject ) SdkV1Rdd[String, String]( spark.sparkContext, diff --git a/src/test/scala/cognite/spark/v1/SparkTest.scala b/src/test/scala/cognite/spark/v1/SparkTest.scala index 6d7f2a943..5080498ac 100644 --- a/src/test/scala/cognite/spark/v1/SparkTest.scala +++ b/src/test/scala/cognite/spark/v1/SparkTest.scala @@ -58,7 +58,6 @@ trait SparkTest { clientSecret = OIDCWrite.clientSecret, scopes = List(OIDCWrite.scopes), audience = Some(OIDCWrite.audience), - cdfProjectName = OIDCWrite.project, ) implicit val sttpBackend: SttpBackend[IO, Any] = CdpConnector.retryingSttpBackend(15, 30) @@ -66,7 +65,7 @@ trait SparkTest { OAuth2.ClientCredentialsProvider[IO](writeCredentials).unsafeRunTimed(1.second).get val writeClient: GenericClient[IO] = new GenericClient( applicationName = "jetfire-test", - projectName = writeCredentials.cdfProjectName, + projectName = OIDCWrite.project, baseUrl = OIDCWrite.baseUrl, authProvider = writeAuthProvider, apiVersion = None, @@ -100,6 +99,7 @@ trait SparkTest { // readClientSecret has to be renewed every 180 days at https://hub.cognite.com/open-industrial-data-211 private val readClientSecret = System.getenv("TEST_OIDC_READ_CLIENT_SECRET") private val readAadTenant = System.getenv("TEST_OIDC_READ_TENANT") + val readProject = System.getenv("TEST_OIDC_READ_PROJECT") assert( readClientId != null && !readClientId.isEmpty, @@ -110,6 +110,9 @@ trait SparkTest { assert( readAadTenant != null && !readAadTenant.isEmpty, "Environment variable \"TEST_OIDC_READ_TENANT\" was not set") + assert( + readProject != null && !readProject.isEmpty, + "Environment variable \"TEST_OIDC_READ_PROJECT\" was not set") private val readTokenUri = s"https://login.microsoftonline.com/$readAadTenant/oauth2/v2.0/token" @@ -117,8 +120,7 @@ trait SparkTest { tokenUri = uri"$readTokenUri", clientId = readClientId, clientSecret = readClientSecret, - scopes = List("https://api.cognitedata.com/.default"), - cdfProjectName = "publicdata" + scopes = List("https://api.cognitedata.com/.default") ) def dataFrameReaderUsingOidc: DataFrameReader = @@ -127,7 +129,7 @@ trait SparkTest { .option("tokenUri", readTokenUri) .option("clientId", readClientId) .option("clientSecret", readClientSecret) - .option("project", "publicdata") + .option("project", readProject) .option("scopes", "https://api.cognitedata.com/.default") val testDataSetId = 86163806167772L @@ -157,8 +159,7 @@ trait SparkTest { tokenUri = bluefieldTokenUri, clientId = bluefieldClientId, clientSecret = bluefieldClientSecret, - scopes = List("https://bluefield.cognitedata.com/.default"), - cdfProjectName = "extractor-bluefield-testing" + scopes = List("https://bluefield.cognitedata.com/.default") ) val authProvider =