Skip to content

Commit

Permalink
Merge pull request #564 from NDLANO/max-query-params
Browse files Browse the repository at this point in the history
search-api: Use a case class for query parameters for GET `/search-api/v1/search`
  • Loading branch information
jnatten authored Dec 16, 2024
2 parents b09c2e2 + 2fa48e8 commit 47ef162
Show file tree
Hide file tree
Showing 9 changed files with 171 additions and 114 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ trait SearchService {

resultArray.map(result => {
val matchedLanguage = language match {
case AllLanguages | "*" =>
case AllLanguages =>
searchConverterService.getLanguageFromHit(result).getOrElse(language)
case _ => language
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ trait SearchService {

resultArray.map(result => {
val matchedLanguage = language match {
case Language.AllLanguages | "*" =>
case Language.AllLanguages =>
searchConverterService.getLanguageFromHit(result).getOrElse(language)
case _ => language
}
Expand Down Expand Up @@ -73,12 +73,12 @@ trait SearchService {
sort match {
case Sort.ByTitleAsc =>
language match {
case "*" | Language.AllLanguages => fieldSort("defaultTitle").order(SortOrder.Asc).missing("_last")
case Language.AllLanguages => fieldSort("defaultTitle").order(SortOrder.Asc).missing("_last")
case _ => fieldSort(s"title.$sortLanguage.raw").order(SortOrder.Asc).missing("_last").unmappedType("long")
}
case Sort.ByTitleDesc =>
language match {
case "*" | Language.AllLanguages => fieldSort("defaultTitle").order(SortOrder.Desc).missing("_last")
case Language.AllLanguages => fieldSort("defaultTitle").order(SortOrder.Desc).missing("_last")
case _ => fieldSort(s"title.$sortLanguage.raw").order(SortOrder.Desc).missing("_last").unmappedType("long")
}
case Sort.ByRelevanceAsc => fieldSort("_score").order(SortOrder.Asc)
Expand Down
2 changes: 1 addition & 1 deletion language/src/main/scala/no/ndla/language/Language.scala
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ object Language {
val DefaultLanguage = "nb"
val UnknownLanguage: LanguageTag = LanguageTag("und")
val NoLanguage = ""
val AllLanguages = "*"
final val AllLanguages = "*"
val Nynorsk = "nynorsk"

val languagePriority: Seq[String] = Seq(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import no.ndla.network.NdlaClient
import no.ndla.network.clients.{FeideApiClient, MyNDLAApiClient, RedisClient}
import no.ndla.network.tapir.TapirApplication
import no.ndla.search.{BaseIndexService, Elastic4sClient}
import no.ndla.searchapi.controller.parameters.GetSearchQueryParams
import no.ndla.searchapi.controller.{InternController, SearchController, SwaggerDocControllerConfig}
import no.ndla.searchapi.integration.*
import no.ndla.searchapi.model.api.ErrorHandling
Expand Down Expand Up @@ -47,6 +48,7 @@ class ComponentRegistry(properties: SearchApiProperties)
with MyNDLAApiClient
with SearchService
with SearchController
with GetSearchQueryParams
with FeideApiClient
with RedisClient
with InternController
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ class SearchApiProperties extends BaseProps with StrictLogging {
case _ => Failure(new IllegalArgumentException(s"Unknown index name: $indexName"))
}

def DefaultPageSize = 10
final val DefaultPageSize = 10
def MaxPageSize = 10000
def IndexBulkSize: Int = propOrElse("INDEX_BULK_SIZE", "100").toInt
def ElasticSearchIndexMaxResultWindow = 10000
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ import no.ndla.common.errors.AccessDeniedException
import no.ndla.common.model.NDLADate
import no.ndla.common.model.api.CommaSeparatedList.*
import no.ndla.common.model.domain.draft.DraftStatus
import no.ndla.common.model.domain.{ArticleType, Availability}
import no.ndla.common.model.domain.Availability
import no.ndla.language.Language.AllLanguages
import no.ndla.network.clients.FeideApiClient
import no.ndla.network.tapir.NoNullJsonPrinter.jsonBody
Expand All @@ -23,6 +23,7 @@ import no.ndla.network.tapir.TapirUtil.errorOutputsFor
import no.ndla.network.tapir.auth.Permission.DRAFT_API_WRITE
import no.ndla.searchapi.controller.parameters.{
DraftSearchParamsDTO,
GetSearchQueryParams,
GrepSearchInputDTO,
SearchParamsDTO,
SubjectAggsInputDTO
Expand Down Expand Up @@ -54,7 +55,7 @@ import sttp.tapir.server.ServerEndpoint

trait SearchController {
this: SearchApiClient & MultiSearchService & SearchConverterService & SearchService & MultiDraftSearchService &
FeideApiClient & Props & ErrorHandling & TapirController & GrepSearchService =>
FeideApiClient & Props & ErrorHandling & TapirController & GrepSearchService & GetSearchQueryParams =>
val searchController: SearchController

class SearchController extends TapirController {
Expand All @@ -71,7 +72,6 @@ trait SearchController {
query[String]("language")
.description("The ISO 639-1 language code describing language.")
.default(AllLanguages)
private val license = query[Option[String]]("license").description("Return only results with provided license.")
private val sort = query[Option[String]]("sort").description(s"""The sorting used on results.
The following are supported: ${Sort.all.mkString(", ")}. Default is by -relevance (desc).""".stripMargin)

Expand All @@ -85,11 +85,6 @@ trait SearchController {
)
.default(DefaultPageSize)
.validate(Validator.inRange(0, MaxPageSize))
private val resourceTypes =
listQuery[String]("resource-types")
.description(
"Return only learning resources with specific taxonomy type(s), e.g. 'urn:resourcetype:learningpath'. To provide multiple types, separate by comma (,)."
)
private val learningResourceIds =
listQuery[Long]("ids")
.description(
Expand All @@ -102,12 +97,6 @@ trait SearchController {
private val subjects =
listQuery[String]("subjects")
.description("A comma separated list of subjects the learning resources should be filtered by.")
private val articleTypes =
listQuery[String]("article-types")
.description(
s"A comma separated list of article-types the search should be filtered by. Available values is ${ArticleType.all
.mkString(", ")}"
)
private val contextTypes =
listQuery[String]("context-types")
.description(
Expand All @@ -134,15 +123,6 @@ trait SearchController {
.description("A comma separated list of codes from GREP API the resources should be filtered by.")
private val traits = listQuery[String]("traits")
.description("A comma separated list of traits the resources should be filtered by.")
private val scrollId = query[Option[String]]("search-context")
.description(
s"""A unique string obtained from a search you want to keep scrolling in. To obtain one from a search, provide one of the following values: ${InitialScrollContextKeywords
.mkString("[", ",", "]")}.
|When scrolling, the parameters from the initial search is used, except in the case of '${this.language.name}' and '${this.fallback.name}'.
|This value may change between scrolls. Always use the one in the latest scroll result (The context, if unused, dies after $ElasticSearchScrollKeepAlive).
|If you are not paginating past $ElasticSearchIndexMaxResultWindow hits, you can ignore this and use '${this.pageNo.name}' and '${this.pageSize.name}' instead.
|""".stripMargin
)
private val aggregatePaths = listQuery[String]("aggregate-paths")
.description("List of index-paths that should be term-aggregated and returned in result.")
private val embedResource =
Expand Down Expand Up @@ -363,91 +343,47 @@ trait SearchController {
.errorOut(errorOutputsFor(400, 401, 403))
.out(jsonBody[MultiSearchResultDTO])
.out(EndpointOutput.derived[DynamicHeaders])
.in(pageNo)
.in(pageSize)
.in(articleTypes)
.in(contextTypes)
.in(language)
.in(learningResourceIds)
.in(resourceTypes)
.in(license)
.in(queryParam)
.in(sort)
.in(fallback)
.in(subjects)
.in(languageFilter)
.in(relevanceFilter)
.in(scrollId)
.in(grepCodes)
.in(traits)
.in(aggregatePaths)
.in(embedResource)
.in(embedId)
.in(filterInactive)
.in(GetSearchQueryParams.input)
.in(feideHeader)
.serverLogicPure {
case (
page,
pageSize,
articleTypes,
contextTypes,
language,
learningResourceIds,
resourceTypes,
license,
query,
sortStr,
fallback,
subjects,
languageFilter,
relevanceFilter,
scrollId,
grepCodes,
traits,
aggregatePaths,
embedResource,
embedId,
filterInactive,
feideToken
) =>
scrollWithOr(scrollId, language, multiSearchService) {
val sort = sortStr.flatMap(Sort.valueOf)
val shouldScroll = scrollId.exists(InitialScrollContextKeywords.contains)
getAvailability(feideToken).flatMap(availability => {
val settings = SearchSettings(
query = query,
fallback = fallback,
language = language,
license = license,
page = page,
pageSize = pageSize,
sort = sort.getOrElse(Sort.ByRelevanceDesc),
withIdIn = learningResourceIds.values,
subjects = subjects.values,
resourceTypes = resourceTypes.values,
learningResourceTypes = contextTypes.values.flatMap(LearningResourceType.valueOf),
supportedLanguages = languageFilter.values,
relevanceIds = relevanceFilter.values,
grepCodes = grepCodes.values,
traits = traits.values.flatMap(SearchTrait.valueOf),
shouldScroll = shouldScroll,
filterByNoResourceType = false,
aggregatePaths = aggregatePaths.values,
embedResource = embedResource.values,
embedId = embedId,
availability = availability,
articleTypes = articleTypes.values,
filterInactive = filterInactive
)
multiSearchService.matchingQuery(settings) match {
case Success(searchResult) =>
val result = searchConverterService.toApiMultiSearchResult(searchResult)
val headers = DynamicHeaders.fromMaybeValue("search-context", searchResult.scrollId)
Success((result, headers))
case Failure(ex) => Failure(ex)
}
})
}
.serverLogicPure { case (q, feideToken) =>
scrollWithOr(q.scrollId, q.language, multiSearchService) {
val sort = q.sort.flatMap(Sort.valueOf)
val shouldScroll = q.scrollId.exists(InitialScrollContextKeywords.contains)
getAvailability(feideToken).flatMap(availability => {
val settings = SearchSettings(
query = q.queryParam,
fallback = q.fallback,
language = q.language,
license = q.license,
page = q.page,
pageSize = q.pageSize,
sort = sort.getOrElse(Sort.ByRelevanceDesc),
withIdIn = q.learningResourceIds.values,
subjects = q.subjects.values,
resourceTypes = q.resourceTypes.values,
learningResourceTypes = q.contextTypes.values.flatMap(LearningResourceType.valueOf),
supportedLanguages = q.languageFilter.values,
relevanceIds = q.relevanceFilter.values,
grepCodes = q.grepCodes.values,
shouldScroll = shouldScroll,
filterByNoResourceType = false,
aggregatePaths = q.aggregatePaths.values,
embedResource = q.embedResource.values,
embedId = q.embedId,
availability = availability,
articleTypes = q.articleTypes.values,
filterInactive = q.filterInactive,
traits = q.traits.values.flatMap(SearchTrait.valueOf)
)
multiSearchService.matchingQuery(settings) match {
case Success(searchResult) =>
val result = searchConverterService.toApiMultiSearchResult(searchResult)
val headers = DynamicHeaders.fromMaybeValue("search-context", searchResult.scrollId)
Success((result, headers))
case Failure(ex) => Failure(ex)
}
})
}

}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
/*
* Part of NDLA backend.search-api.main
* Copyright (C) 2024 NDLA
*
* See LICENSE
*
*/

package no.ndla.searchapi.controller.parameters

import no.ndla.common.model.api.CommaSeparatedList.CommaSeparatedList
import no.ndla.language.Language
import no.ndla.network.tapir.NonEmptyString
import no.ndla.searchapi.Props
import sttp.tapir.*
import sttp.tapir.EndpointIO.annotations.query
import sttp.tapir.Schema.annotations.{default, description}
import sttp.tapir.ValidationResult.{Invalid, Valid}

trait GetSearchQueryParams {
this: Props =>

case class GetSearchQueryParams(
@query("page")
@description("The page number of the search hits to display.")
@default(1)
page: Int,
@query("page-size")
@description("The number of search hits to display for each page.")
@default(props.DefaultPageSize)
pageSize: Int,
@query("article-types")
@description("A comma separated list of article-types the search should be filtered by.")
articleTypes: CommaSeparatedList[String],
@query("context-types")
@description("A comma separated list of types the learning resources should be filtered by.")
contextTypes: CommaSeparatedList[String],
@query("language")
@description("The ISO 639-1 language code describing language.")
@default(Language.AllLanguages)
language: String,
@query("ids")
@description(
"Return only learning resources that have one of the provided ids. To provide multiple ids, separate by comma (,)."
)
learningResourceIds: CommaSeparatedList[Long],
@query("resource-types")
@description(
"Return only learning resources with specific taxonomy type(s), e.g. 'urn:resourcetype:learningpath'. To provide multiple types, separate by comma (,)."
)
resourceTypes: CommaSeparatedList[String],
@query("license")
@description("Return only results with provided license.")
license: Option[String],
@query("query")
@description("Return only results with content matching the specified query.")
@default(None)
queryParam: Option[NonEmptyString],
@query("sort")
@description("Sort the search results by the specified field.")
sort: Option[String],
@query("fallback")
@default(false)
@description("Fallback to existing language if language is specified.")
fallback: Boolean,
@query("subjects")
@description("A comma separated list of subjects the learning resources should be filtered by.")
subjects: CommaSeparatedList[String],
@query("language-filter")
@description("A comma separated list of ISO 639-1 language codes that the learning resource can be available in.")
languageFilter: CommaSeparatedList[String],
@query("relevance")
@description(
"A comma separated list of relevances the learning resources should be filtered by. If subjects are specified the learning resource must have specified relevances in relation to a specified subject. If levels are specified the learning resource must have specified relevances in relation to a specified level."
)
relevanceFilter: CommaSeparatedList[String],
@query("search-context")
@description("A unique string obtained from a search you want to keep scrolling in.")
scrollId: Option[String],
@query("grep-codes")
@description("A comma separated list of codes from GREP API the resources should be filtered by.")
grepCodes: CommaSeparatedList[String],
@query("aggregate-paths")
@description("List of index-paths that should be term-aggregated and returned in result.")
aggregatePaths: CommaSeparatedList[String],
@query("embed-resource")
@description(
"Return only results with embed data-resource the specified resource. Can specify multiple with a comma separated list to filter for one of the embed types."
)
embedResource: CommaSeparatedList[String],
@query("embed-id")
@description("Return only results with embed data-resource_id, data-videoid or data-url with the specified id.")
embedId: Option[String],
@query("filter-inactive")
@description("Filter out inactive taxonomy contexts.")
@default(false)
filterInactive: Boolean,
@query("traits")
@description("A comma separated list of traits the resources should be filtered by.")
traits: CommaSeparatedList[String]
)

object GetSearchQueryParams {
implicit val schema: Schema[GetSearchQueryParams] = Schema.derived[GetSearchQueryParams]
implicit val schemaOpt: Schema[Option[GetSearchQueryParams]] = schema.asOption
def input = EndpointInput
.derived[GetSearchQueryParams]
.validate {
Validator.custom {
case q if q.page < 1 => Invalid("page must be greater than 0")
case q if q.pageSize < 1 || q.pageSize > props.MaxPageSize =>
Invalid(s"page-size must be between 1 and ${props.MaxPageSize}")
case _ => Valid
}
}
}
}
Loading

0 comments on commit 47ef162

Please sign in to comment.