Skip to content

Commit

Permalink
CDF-20144: remove wdl tests and mark deprecated
Browse files Browse the repository at this point in the history
wdl didn't hit GA and has no active usage through Scala SDK
and so no active usage through spark-datasource either.

Let's start by removing tests and marking deprecated.
Removing tests has a benefit of getting closer to being able
to run repo tests in any project (wdl tests can only be run in
dedicated projects with wdl access anyway).

[CDF-20144]
  • Loading branch information
dmivankov committed Oct 31, 2023
1 parent bf4e903 commit 9af82b0
Show file tree
Hide file tree
Showing 11 changed files with 14 additions and 330 deletions.
2 changes: 1 addition & 1 deletion build.sbt
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ lazy val commonSettings = Seq(
organization := "com.cognite.spark.datasource",
organizationName := "Cognite",
organizationHomepage := Some(url("https://cognite.com")),
version := "3.2." + patchVersion,
version := "3.3." + patchVersion,
isSnapshot := patchVersion.endsWith("-SNAPSHOT"),
crossScalaVersions := supportedScalaVersions,
semanticdbEnabled := true,
Expand Down
5 changes: 3 additions & 2 deletions src/main/scala/cognite/spark/v1/DefaultSource.scala
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ class DefaultSource
new SequenceRowsRelation(config, sequenceId)(sqlContext)
}

@deprecated("wdl support is deprecated", since = "0")
private def createWellDataLayer(
parameters: Map[String, String],
config: RelationConfig,
Expand Down Expand Up @@ -174,7 +175,7 @@ class DefaultSource
case FlexibleDataModelRelationFactory.ResourceType =>
createFlexibleDataModelRelation(parameters, config, sqlContext)
case "welldatalayer" =>
createWellDataLayer(parameters, config, sqlContext)
createWellDataLayer(parameters, config, sqlContext): @annotation.nowarn
case _ => sys.error("Unknown resource type: " + resourceType)
}
}
Expand Down Expand Up @@ -241,7 +242,7 @@ class DefaultSource
case FlexibleDataModelRelationFactory.ResourceType =>
createFlexibleDataModelRelation(parameters, config, sqlContext)
case "welldatalayer" =>
createWellDataLayer(parameters, config, sqlContext)
createWellDataLayer(parameters, config, sqlContext): @annotation.nowarn
case _ => sys.error(s"Resource type $resourceType does not support save()")
}
val batchSizeDefault = relation match {
Expand Down
1 change: 1 addition & 0 deletions src/main/scala/cognite/spark/v1/wdl/JsonObjectToRow.scala
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import io.circe.{Json, JsonObject}
import org.apache.spark.sql.Row
import org.apache.spark.sql.types._

@deprecated("wdl support is deprecated", since = "0")
object JsonObjectToRow {
implicit class RequiredOption[T](optionValue: Option[T]) {
def orThrow(structFieldName: String, nullable: Boolean, dataType: DataType): Option[T] =
Expand Down
1 change: 1 addition & 0 deletions src/main/scala/cognite/spark/v1/wdl/RowToJson.scala
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import java.time.{Instant, LocalDate}
import scala.collection.mutable

// sscalastyle:off
@deprecated("wdl support is deprecated", since = "0")
object RowToJson {

/**
Expand Down
2 changes: 2 additions & 0 deletions src/main/scala/cognite/spark/v1/wdl/WdlExceptions.scala
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,12 @@ package cognite.spark.v1.wdl
import cognite.spark.v1.CdfSparkException
import org.apache.spark.sql.types.DataType

@deprecated("wdl support is deprecated", since = "0")
class RequiredFieldIsNullException(val structFieldName: String, val dataType: DataType)
extends CdfSparkException(
s"Required field `${structFieldName}` of type `${dataType.typeName}` should not be NULL.")

@deprecated("wdl support is deprecated", since = "0")
class WrongFieldTypeException(
val structFieldName: String,
val expectedDataType: DataType,
Expand Down
4 changes: 4 additions & 0 deletions src/main/scala/cognite/spark/v1/wdl/WdlModels.scala
Original file line number Diff line number Diff line change
Expand Up @@ -3,24 +3,28 @@ package cognite.spark.v1.wdl
import cognite.spark.v1.CdfSparkException
import io.circe.{Json, JsonObject}

@deprecated("wdl support is deprecated", since = "0")
case class WdlModel(
shortName: String,
ingest: Option[WdlIngestDefinition],
retrieve: WdlRetrieveDefinition
)

@deprecated("wdl support is deprecated", since = "0")
case class WdlIngestDefinition(
schemaName: String,
url: String
)

@deprecated("wdl support is deprecated", since = "0")
case class WdlRetrieveDefinition(
schemaName: String,
url: String,
isGet: Boolean = false,
transformBody: JsonObject => JsonObject = it => it
)

@deprecated("wdl support is deprecated", since = "0")
object WdlModels {
val models: Seq[WdlModel] = Seq(
WdlModel(
Expand Down
1 change: 1 addition & 0 deletions src/main/scala/cognite/spark/v1/wdl/WellDataLayerRDD.scala
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import org.apache.spark.{Partition, SparkContext, TaskContext}

import scala.collection.AbstractIterator

@deprecated("wdl support is deprecated", since = "0")
class WellDataLayerRDD(
@transient override val sparkContext: SparkContext,
val schema: StructType,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import org.apache.spark.sql.sources.TableScan
import org.apache.spark.sql.types.{DataType, StructType}
import org.apache.spark.sql.{Row, SQLContext}

@deprecated("wdl support is deprecated", since = "0")
class WellDataLayerRelation(
config: RelationConfig,
model: String
Expand Down
19 changes: 0 additions & 19 deletions src/test/scala/cognite/spark/v1/wdl/RowEquality.scala

This file was deleted.

243 changes: 0 additions & 243 deletions src/test/scala/cognite/spark/v1/wdl/RowToJsonTest.scala

This file was deleted.

Loading

0 comments on commit 9af82b0

Please sign in to comment.