diff --git a/audio-api/src/main/scala/no/ndla/audioapi/AudioApiProperties.scala b/audio-api/src/main/scala/no/ndla/audioapi/AudioApiProperties.scala index 7a298818a..d6a9b6360 100644 --- a/audio-api/src/main/scala/no/ndla/audioapi/AudioApiProperties.scala +++ b/audio-api/src/main/scala/no/ndla/audioapi/AudioApiProperties.scala @@ -9,6 +9,7 @@ package no.ndla.audioapi import com.typesafe.scalalogging.StrictLogging +import no.ndla.common.Environment.prop import no.ndla.common.configuration.{BaseProps, HasBaseProps} import no.ndla.database.{DatabaseProps, HasDatabaseProps} import no.ndla.network.{AuthUser, Domains} @@ -36,6 +37,13 @@ class AudioApiProperties extends BaseProps with DatabaseProps with StrictLogging val StorageName: String = propOrElse("AUDIO_FILE_S3_BUCKET", s"$Environment.audio.ndla") val StorageRegion: Option[String] = propOrNone("AUDIO_FILE_S3_BUCKET_REGION") + val TranscribeStorageName: String = propOrElse("TRANSCRIBE_FILE_S3_BUCKET", s"$Environment.transcribe.ndla") + val TranscribeStorageRegion: Option[String] = propOrNone("TRANSCRIBE_FILE_S3_BUCKET_REGION") + + val BrightcoveClientId: String = prop("BRIGHTCOVE_API_CLIENT_ID") + val BrightcoveClientSecret: String = prop("BRIGHTCOVE_API_CLIENT_SECRET") + val BrightcoveAccountId: String = prop("BRIGHTCOVE_ACCOUNT_ID") + val SearchServer: String = propOrElse("SEARCH_SERVER", "http://search-audio-api.ndla-local") val RunWithSignedSearchRequests: Boolean = propOrElse("RUN_WITH_SIGNED_SEARCH_REQUESTS", "true").toBoolean val SearchIndex: String = propOrElse("SEARCH_INDEX_NAME", "audios") diff --git a/audio-api/src/main/scala/no/ndla/audioapi/ComponentRegistry.scala b/audio-api/src/main/scala/no/ndla/audioapi/ComponentRegistry.scala index 3684fcee8..4d055dd79 100644 --- a/audio-api/src/main/scala/no/ndla/audioapi/ComponentRegistry.scala +++ b/audio-api/src/main/scala/no/ndla/audioapi/ComponentRegistry.scala @@ -16,7 +16,8 @@ import no.ndla.audioapi.repository.{AudioRepository, SeriesRepository} import no.ndla.audioapi.service.* import no.ndla.audioapi.service.search.* import no.ndla.common.Clock -import no.ndla.common.aws.NdlaS3Client +import no.ndla.common.aws.{NdlaAWSTranscribeClient, NdlaS3Client} +import no.ndla.common.brightcove.NdlaBrightcoveClient import no.ndla.common.configuration.BaseComponentRegistry import no.ndla.database.{DBMigrator, DataSource} import no.ndla.network.NdlaClient @@ -38,6 +39,7 @@ class ComponentRegistry(properties: AudioApiProperties) with HealthController with AudioController with SeriesController + with TranscriptionController with SearchService with AudioSearchService with SeriesSearchService @@ -54,7 +56,10 @@ class ComponentRegistry(properties: AudioApiProperties) with DBMigrator with ErrorHandling with SwaggerDocControllerConfig - with NdlaS3Client { + with NdlaS3Client + with TranscriptionService + with NdlaAWSTranscribeClient + with NdlaBrightcoveClient { override val props: AudioApiProperties = properties override val migrator: DBMigrator = DBMigrator( new V5__AddAgreementToAudio, @@ -63,7 +68,10 @@ class ComponentRegistry(properties: AudioApiProperties) override val dataSource: HikariDataSource = DataSource.getHikariDataSource DataSource.connectToDatabase() - lazy val s3Client = new NdlaS3Client(props.StorageName, props.StorageRegion) + lazy val s3Client = new NdlaS3Client(props.StorageName, props.StorageRegion) + lazy val s3TranscribeClient = new NdlaS3Client(props.TranscribeStorageName, props.TranscribeStorageRegion) + lazy val brightcoveClient = new NdlaBrightcoveClient() + lazy val transcribeClient = new NdlaAWSTranscribeClient(props.TranscribeStorageRegion) lazy val audioRepository = new AudioRepository lazy val seriesRepository = new SeriesRepository @@ -71,15 +79,17 @@ class ComponentRegistry(properties: AudioApiProperties) lazy val ndlaClient = new NdlaClient lazy val myndlaApiClient: MyNDLAApiClient = new MyNDLAApiClient - lazy val readService = new ReadService - lazy val writeService = new WriteService - lazy val validationService = new ValidationService - lazy val converterService = new ConverterService + lazy val readService = new ReadService + lazy val writeService = new WriteService + lazy val validationService = new ValidationService + lazy val converterService = new ConverterService + lazy val transcriptionService = new TranscriptionService - lazy val internController = new InternController - lazy val audioApiController = new AudioController - lazy val seriesController = new SeriesController - lazy val healthController = new HealthController + lazy val internController = new InternController + lazy val audioApiController = new AudioController + lazy val seriesController = new SeriesController + lazy val healthController = new HealthController + lazy val transcriptionController = new TranscriptionController var e4sClient: NdlaE4sClient = Elastic4sClientFactory.getClient(props.SearchServer) lazy val searchConverterService = new SearchConverterService @@ -97,7 +107,8 @@ class ComponentRegistry(properties: AudioApiProperties) audioApiController, seriesController, internController, - healthController + healthController, + transcriptionController ), SwaggerDocControllerConfig.swaggerInfo ) diff --git a/audio-api/src/main/scala/no/ndla/audioapi/controller/InternController.scala b/audio-api/src/main/scala/no/ndla/audioapi/controller/InternController.scala index 627e1cf9b..bd443d818 100644 --- a/audio-api/src/main/scala/no/ndla/audioapi/controller/InternController.scala +++ b/audio-api/src/main/scala/no/ndla/audioapi/controller/InternController.scala @@ -12,11 +12,12 @@ import cats.implicits.* import io.circe.generic.auto.* import no.ndla.audioapi.Props import no.ndla.audioapi.model.api -import no.ndla.audioapi.model.api.{AudioMetaDomainDumpDTO, ErrorHandling, NotFoundException} +import no.ndla.audioapi.model.api.{AudioMetaDomainDumpDTO, ErrorHandling} import no.ndla.audioapi.model.domain.AudioMetaInformation import no.ndla.audioapi.repository.AudioRepository import no.ndla.audioapi.service.search.{AudioIndexService, SeriesIndexService, TagIndexService} import no.ndla.audioapi.service.{ConverterService, ReadService} +import no.ndla.common.errors.NotFoundException import no.ndla.network.tapir.NoNullJsonPrinter.jsonBody import no.ndla.network.tapir.TapirController import no.ndla.network.tapir.TapirUtil.errorOutputsFor @@ -116,7 +117,7 @@ trait InternController { .serverLogicPure { id => audioRepository.withId(id) match { case Some(image) => image.asRight - case None => returnLeftError(new NotFoundException(s"Could not find audio with id: '$id'")) + case None => returnLeftError(NotFoundException(s"Could not find audio with id: '$id'")) } }, endpoint.post diff --git a/audio-api/src/main/scala/no/ndla/audioapi/controller/TranscriptionController.scala b/audio-api/src/main/scala/no/ndla/audioapi/controller/TranscriptionController.scala new file mode 100644 index 000000000..933ed1cf9 --- /dev/null +++ b/audio-api/src/main/scala/no/ndla/audioapi/controller/TranscriptionController.scala @@ -0,0 +1,165 @@ +/* + * Part of NDLA audio-api + * Copyright (C) 2024 NDLA + * + * See LICENSE + * + */ + +package no.ndla.audioapi.controller + +import no.ndla.audioapi.Props +import no.ndla.audioapi.model.api.TranscriptionResultDTO +import no.ndla.audioapi.service.{ReadService, TranscriptionService} +import no.ndla.network.tapir.NoNullJsonPrinter.jsonBody +import no.ndla.network.tapir.TapirController +import no.ndla.network.tapir.TapirUtil.errorOutputsFor +import no.ndla.network.tapir.auth.Permission.AUDIO_API_WRITE +import sttp.tapir.server.ServerEndpoint +import sttp.tapir.{EndpointInput, endpoint, path} +import sttp.tapir.* +import sttp.tapir.generic.auto.schemaForCaseClass + +import scala.util.{Failure, Success} +trait TranscriptionController { + this: Props & TapirController & ReadService & TranscriptionService => + val transcriptionController: TranscriptionController + class TranscriptionController() extends TapirController { + + override val serviceName: String = "transcription" + override val prefix: EndpointInput[Unit] = "audio-api" / "v1" / serviceName + + private val videoId = path[String]("videoId").description("The video id to transcribe") + private val audioName = path[String]("audioName").description("The audio name to transcribe") + private val audioId = path[Long]("audioId").description("The audio id to transcribe") + private val language = path[String]("language").description("The language to run the transcription in") + private val maxSpeaker = + query[Int]("maxSpeaker").description("The maximum number of speakers in the video").default(2) + private val format = query[String]("format").description("The format of the audio file").default("mp3") + + def postExtractAudio: ServerEndpoint[Any, Eff] = endpoint.post + .summary("Extract audio from video") + .description("Extracts audio from a Brightcove video and uploads it to S3.") + .in(videoId) + .in(language) + .in("extract-audio") + .errorOut(errorOutputsFor(400, 500)) + .requirePermission(AUDIO_API_WRITE) + .serverLogicPure { _ => + { case (videoId, language) => + transcriptionService.extractAudioFromVideo(videoId, language) match { + case Success(_) => Right(()) + case Failure(ex) => returnLeftError(ex) + } + } + } + + def getAudioExtraction: ServerEndpoint[Any, Eff] = endpoint.get + .summary("Get audio extraction status") + .description("Get the status of the audio extraction from a Brightcove video.") + .in(videoId) + .in(language) + .in("extract-audio") + .errorOut(errorOutputsFor(400, 500)) + .requirePermission(AUDIO_API_WRITE) + .serverLogicPure { _ => + { case (videoId, language) => + transcriptionService.getAudioExtractionStatus(videoId, language) match { + case Success(_) => Right(()) + case Failure(ex) => returnLeftError(ex) + } + } + } + + def postTranscription: ServerEndpoint[Any, Eff] = endpoint.post + .summary("Transcribe video") + .description("Transcribes a video and uploads the transcription to S3.") + .in("video") + .in(videoId) + .in(language) + .in(maxSpeaker) + .errorOut(errorOutputsFor(400, 500)) + .requirePermission(AUDIO_API_WRITE) + .serverLogicPure { _ => + { case (videoId, language, maxSpeakerOpt) => + transcriptionService.transcribeVideo(videoId, language, maxSpeakerOpt) match { + case Success(_) => Right(()) + case Failure(ex) => returnLeftError(ex) + } + } + } + + def getTranscription: ServerEndpoint[Any, Eff] = endpoint.get + .summary("Get the transcription status of a video") + .description("Get the transcription of a video.") + .in("video") + .in(videoId) + .in(language) + .errorOut(errorOutputsFor(400, 404, 405, 500)) + .out(jsonBody[TranscriptionResultDTO]) + .requirePermission(AUDIO_API_WRITE) + .serverLogicPure { _ => + { case (videoId, language) => + transcriptionService.getVideoTranscription(videoId, language) match { + case Success(Right(transcriptionContent)) => + Right(TranscriptionResultDTO("COMPLETED", Some(transcriptionContent))) + case Success(Left(jobStatus)) => + Right(TranscriptionResultDTO(jobStatus.toString, None)) + case Failure(ex) => returnLeftError(ex) + } + } + } + + def postAudioTranscription: ServerEndpoint[Any, Eff] = endpoint.post + .summary("Transcribe audio") + .description("Transcribes an audiofile and uploads the transcription to S3.") + .in("audio") + .in(audioName) + .in(audioId) + .in(language) + .in(maxSpeaker) + .in(format) + .errorOut(errorOutputsFor(400, 500)) + .requirePermission(AUDIO_API_WRITE) + .serverLogicPure { _ => + { case (audioName, audioId, language, maxSpeakerOpt, format) => + transcriptionService.transcribeAudio(audioName, audioId, language, maxSpeakerOpt, format) match { + case Success(_) => Right(()) + case Failure(ex) => returnLeftError(ex) + } + } + } + + def getAudioTranscription: ServerEndpoint[Any, Eff] = endpoint.get + .summary("Get the transcription status of an audiofile") + .description("Get the transcription of an audiofile .") + .in("audio") + .in(audioId) + .in(language) + .errorOut(errorOutputsFor(400, 404, 405, 500)) + .out(jsonBody[TranscriptionResultDTO]) + .requirePermission(AUDIO_API_WRITE) + .serverLogicPure { _ => + { case (audioId, language) => + transcriptionService.getAudioTranscription(audioId, language) match { + case Success(Right(transcriptionContent)) => + Right(TranscriptionResultDTO("COMPLETED", Some(transcriptionContent))) + case Success(Left(jobStatus)) => + Right(TranscriptionResultDTO(jobStatus.toString, None)) + case Failure(ex) => returnLeftError(ex) + } + } + } + + override val endpoints: List[ServerEndpoint[Any, Eff]] = + List( + postExtractAudio, + getAudioExtraction, + postTranscription, + getTranscription, + postAudioTranscription, + getAudioTranscription + ) + } + +} diff --git a/audio-api/src/main/scala/no/ndla/audioapi/model/api/Error.scala b/audio-api/src/main/scala/no/ndla/audioapi/model/api/Error.scala index 58c1de5a1..2548d19af 100644 --- a/audio-api/src/main/scala/no/ndla/audioapi/model/api/Error.scala +++ b/audio-api/src/main/scala/no/ndla/audioapi/model/api/Error.scala @@ -10,7 +10,7 @@ package no.ndla.audioapi.model.api import no.ndla.audioapi.Props import no.ndla.common.Clock -import no.ndla.common.errors.{AccessDeniedException, FileTooBigException, ValidationException} +import no.ndla.common.errors.{AccessDeniedException, FileTooBigException, NotFoundException, ValidationException} import no.ndla.database.DataSource import no.ndla.network.model.HttpRequestException import no.ndla.network.tapir.{AllErrors, ErrorBody, TapirErrorHandling, ValidationErrorBody} @@ -55,13 +55,14 @@ trait ErrorHandling extends TapirErrorHandling { if rf.error.rootCause .exists(x => x.`type` == "search_context_missing_exception" || x.reason == "Cannot parse scroll id") => invalidSearchContext + case jafe: JobAlreadyFoundException => ErrorBody(JOB_ALREADY_FOUND, jafe.getMessage, clock.now(), 400) } } -class NotFoundException(message: String = "The audio was not found") extends RuntimeException(message) -case class MissingIdException(message: String) extends RuntimeException(message) -case class CouldNotFindLanguageException(message: String) extends RuntimeException(message) -class AudioStorageException(message: String) extends RuntimeException(message) -class LanguageMappingException(message: String) extends RuntimeException(message) -class ImportException(message: String) extends RuntimeException(message) +case class MissingIdException(message: String) extends RuntimeException(message) +case class CouldNotFindLanguageException(message: String) extends RuntimeException(message) +class AudioStorageException(message: String) extends RuntimeException(message) +class LanguageMappingException(message: String) extends RuntimeException(message) +class ImportException(message: String) extends RuntimeException(message) +case class JobAlreadyFoundException(message: String) extends RuntimeException(message) diff --git a/audio-api/src/main/scala/no/ndla/audioapi/model/api/TranscriptionResultDTO.scala b/audio-api/src/main/scala/no/ndla/audioapi/model/api/TranscriptionResultDTO.scala new file mode 100644 index 000000000..faf5ef5aa --- /dev/null +++ b/audio-api/src/main/scala/no/ndla/audioapi/model/api/TranscriptionResultDTO.scala @@ -0,0 +1,23 @@ +/* + * Part of NDLA audio-api + * Copyright (C) 2024 NDLA + * + * See LICENSE + * + */ + +package no.ndla.audioapi.model.api + +import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} +import io.circe.{Decoder, Encoder} +import sttp.tapir.Schema.annotations.description + +@description("The result of a transcription job") +case class TranscriptionResultDTO( + @description("The status of the transcription job") status: String, + @description("The transcription of the audio") transcription: Option[String] +) +object TranscriptionResultDTO { + implicit val encoder: Encoder[TranscriptionResultDTO] = deriveEncoder + implicit val decoder: Decoder[TranscriptionResultDTO] = deriveDecoder +} diff --git a/audio-api/src/main/scala/no/ndla/audioapi/service/ReadService.scala b/audio-api/src/main/scala/no/ndla/audioapi/service/ReadService.scala index 06e8299c9..5addf3921 100644 --- a/audio-api/src/main/scala/no/ndla/audioapi/service/ReadService.scala +++ b/audio-api/src/main/scala/no/ndla/audioapi/service/ReadService.scala @@ -8,12 +8,11 @@ package no.ndla.audioapi.service -import cats.implicits._ +import cats.implicits.* import no.ndla.audioapi.model.api -import no.ndla.audioapi.model.api.NotFoundException import no.ndla.audioapi.repository.{AudioRepository, SeriesRepository} import no.ndla.audioapi.service.search.{SearchConverterService, TagSearchService} -import no.ndla.common.errors.ValidationException +import no.ndla.common.errors.{NotFoundException, ValidationException} import scala.util.{Failure, Success, Try} diff --git a/audio-api/src/main/scala/no/ndla/audioapi/service/TranscriptionService.scala b/audio-api/src/main/scala/no/ndla/audioapi/service/TranscriptionService.scala new file mode 100644 index 000000000..9169e8875 --- /dev/null +++ b/audio-api/src/main/scala/no/ndla/audioapi/service/TranscriptionService.scala @@ -0,0 +1,258 @@ +/* + * Part of NDLA audio-api + * Copyright (C) 2024 NDLA + * + * See LICENSE + * + */ + +package no.ndla.audioapi.service + +import com.typesafe.scalalogging.StrictLogging +import no.ndla.audioapi.Props +import no.ndla.audioapi.model.api.JobAlreadyFoundException +import no.ndla.common.aws.{NdlaAWSTranscribeClient, NdlaS3Client} +import no.ndla.common.brightcove.NdlaBrightcoveClient +import no.ndla.common.model.domain.UploadedFile +import sttp.client3.{HttpURLConnectionBackend, UriContext, asFile, basicRequest} +import ws.schild.jave.{Encoder, MultimediaObject} +import ws.schild.jave.encode.{AudioAttributes, EncodingAttributes} + +import java.io.File +import scala.util.{Failure, Success, Try} + +trait TranscriptionService { + this: NdlaS3Client & Props & NdlaBrightcoveClient & NdlaAWSTranscribeClient => + val transcriptionService: TranscriptionService + val s3TranscribeClient: NdlaS3Client + + class TranscriptionService extends StrictLogging { + + def transcribeVideo(videoId: String, language: String, maxSpeakers: Int): Try[Unit] = { + getVideoTranscription(videoId, language) match { + case Success(Right(_)) => + logger.info(s"Transcription already completed for videoId: $videoId") + return Failure(new JobAlreadyFoundException(s"Transcription already completed for videoId: $videoId")) + case Success(Left("IN_PROGRESS")) => + logger.info(s"Transcription already in progress for videoId: $videoId") + return Failure(new JobAlreadyFoundException(s"Transcription already in progress for videoId: $videoId")) + case _ => + logger.info(s"No existing transcription job for videoId: $videoId") + } + + getAudioExtractionStatus(videoId, language) match { + case Success(_) => + logger.info(s"Audio already extracted for videoId: $videoId") + case Failure(_) => + logger.info(s"Audio extraction required for videoId: $videoId") + extractAudioFromVideo(videoId, language) match { + case Success(_) => + logger.info(s"Audio extracted for videoId: $videoId") + case Failure(exception) => + return Failure(new RuntimeException(s"Failed to extract audio for videoId: $videoId", exception)) + + } + } + + val audioUri = s"s3://${props.TranscribeStorageName}/audio-extraction/$language/$videoId.mp3" + logger.info(s"Transcribing audio from: $audioUri") + val jobName = s"transcribe-video-$videoId-$language" + val mediaFormat = "mp3" + val outputKey = s"transcription/$language/$videoId" + val languageCode = language + + transcribeClient.startTranscriptionJob( + jobName, + audioUri, + mediaFormat, + languageCode, + props.TranscribeStorageName, + outputKey, + maxSpeakers + ) match { + case Success(_) => + logger.info(s"Transcription job started for videoId: $videoId") + Success(()) + case Failure(exception) => + Failure(new RuntimeException(s"Failed to start transcription for videoId: $videoId", exception)) + } + } + + def getVideoTranscription( + videoId: String, + language: String + ): Try[Either[String, String]] = { + val jobName = s"transcribe-video-$videoId-$language" + + transcribeClient.getTranscriptionJob(jobName).flatMap { transcriptionJobResponse => + val transcriptionJob = transcriptionJobResponse.transcriptionJob() + val transcriptionJobStatus = transcriptionJob.transcriptionJobStatus().toString + + if (transcriptionJobStatus == "COMPLETED") { + val transcribeUri = s"transcription/$language/${videoId}.vtt" + + getObjectFromS3(transcribeUri).map(Right(_)) + } else { + Success(Left(transcriptionJobStatus)) + } + } + } + + def transcribeAudio( + audioName: String, + audioId: Long, + language: String, + maxSpeakers: Int, + format: String + ): Try[Unit] = { + getAudioTranscription(audioId, language) match { + case Success(Right(_)) => + logger.info(s"Transcription already completed for audio: $audioName") + return Failure(new JobAlreadyFoundException(s"Transcription already completed for audio: $audioName")) + case Success(Left("IN_PROGRESS")) => + logger.info(s"Transcription already in progress for videoId: $audioName") + return Failure(new JobAlreadyFoundException(s"Transcription already in progress for audio: $audioName")) + case _ => + logger.info(s"No existing transcription job for audio name: $audioName") + } + val audioUri = s"s3://${props.StorageName}/$audioName" + logger.info(s"Transcribing audio from: $audioUri") + val jobName = s"transcribe-audio-$audioId-$language" + val mediaFormat = format + val outputKey = s"audio-transcription/$language/$audioId" + val languageCode = language + transcribeClient.startTranscriptionJob( + jobName, + audioUri, + mediaFormat, + languageCode, + props.TranscribeStorageName, + outputKey, + maxSpeakers, + includeSubtitles = false + ) match { + case Success(_) => + logger.info(s"Transcription job started for audio: $audioName") + Success(()) + case Failure(exception) => + Failure(new RuntimeException(s"Failed to start transcription for audio file: $audioName", exception)) + } + } + + def getAudioTranscription(audioId: Long, language: String): Try[Either[String, String]] = { + val jobName = s"transcribe-audio-$audioId-$language" + + transcribeClient.getTranscriptionJob(jobName).flatMap { transcriptionJobResponse => + val transcriptionJob = transcriptionJobResponse.transcriptionJob() + val transcriptionJobStatus = transcriptionJob.transcriptionJobStatus().toString + + if (transcriptionJobStatus == "COMPLETED") { + val transcribeUri = s"audio-transcription/$language/${audioId}" + + getObjectFromS3(transcribeUri).map(Right(_)) + } else { + Success(Left(transcriptionJobStatus)) + } + } + } + + private def getObjectFromS3(Uri: String): Try[String] = { + s3TranscribeClient.getObject(Uri).map { s3Object => + val content = scala.io.Source.fromInputStream(s3Object.stream).mkString + s3Object.stream.close() + content + } + } + + def extractAudioFromVideo(videoId: String, language: String): Try[Unit] = { + val accountId = props.BrightcoveAccountId + val videoUrl = getVideo(accountId, videoId) match { + case Success(sources) if sources.nonEmpty => sources.head + case Success(_) => return Failure(new RuntimeException(s"No video sources found for videoId: $videoId")) + case Failure(ex) => return Failure(new RuntimeException(s"Failed to get video sources: $ex")) + } + val videoFile = downloadVideo(videoId, videoUrl) match { + case Success(file) => file + case Failure(ex) => throw new RuntimeException(s"Failed to download video: $ex") + } + + val audioFile = new File(s"/tmp/audio_${videoId}.mp3") + + val audioAttributes = new AudioAttributes() + audioAttributes.setCodec("libmp3lame") + audioAttributes.setBitRate(128000) + audioAttributes.setChannels(2) + audioAttributes.setSamplingRate(44100) + + val encodingAttributes = new EncodingAttributes() + encodingAttributes.setOutputFormat("mp3") + encodingAttributes.setAudioAttributes(audioAttributes) + + val encoder = new Encoder() + Try { + encoder.encode(new MultimediaObject(videoFile), audioFile, encodingAttributes) + } match { + case Success(_) => + val s3Key = s"audio-extraction/$language/$videoId.mp3" + logger.info(s"Uploading audio file to S3: $s3Key") + val uploadedFile = UploadedFile( // convert to uploadedFile object + partName = "", + fileName = Some(s"audio_$videoId.mp3"), + fileSize = audioFile.length(), + contentType = Some("audio/mpeg"), + file = audioFile + ) + s3TranscribeClient.putObject(s3Key, uploadedFile) match { + case Success(_) => + logger.info(s"Audio file uploaded to S3: $s3Key") + for { + _ <- Try(audioFile.delete()) + _ <- Try(videoFile.delete()) + } yield () + case Failure(ex) => + Failure(new RuntimeException(s"Failed to upload audio file to S3.", ex)) + } + case Failure(exception) => Failure(exception) + } + } + + def getAudioExtractionStatus(videoId: String, language: String): Try[Unit] = { + s3TranscribeClient.getObject(s"audio-extraction/$language/${videoId}.mp3") match { + case Success(_) => Success(()) + case Failure(exception) => Failure(exception) + } + } + + private def getVideo(accountId: String, videoId: String): Try[Vector[String]] = { + val clientId = props.BrightcoveClientId + val clientSecret = props.BrightcoveClientSecret + + for { + token <- brightcoveClient.getToken(clientId, clientSecret) + sources <- brightcoveClient.getVideoSource(accountId, videoId, token) + mp4Sources = sources + .filter(source => source.hcursor.get[String]("container").toOption.contains("MP4")) + .map(source => source.hcursor.get[String]("src").toOption.getOrElse("")) + result <- + if (mp4Sources.nonEmpty) Success(mp4Sources) else Failure(new RuntimeException("No MP4 sources found")) + } yield result + } + + private def downloadVideo(videoId: String, videoUrl: String): Try[File] = { + val videoFile = new File(s"/tmp/video_$videoId.mp4") + val connection = HttpURLConnectionBackend() + + val response = basicRequest.get(uri"$videoUrl").response(asFile(videoFile)).send(connection) + Try { + response.body match { + case Right(file) => file + case Left(error) => throw new RuntimeException(s"Failed to download video: $error") + } + } match { + case Success(file) => Success(file) + case Failure(exception) => Failure(exception) + } + + } + } +} diff --git a/audio-api/src/main/scala/no/ndla/audioapi/service/WriteService.scala b/audio-api/src/main/scala/no/ndla/audioapi/service/WriteService.scala index 6dc06af43..b4ebceeba 100644 --- a/audio-api/src/main/scala/no/ndla/audioapi/service/WriteService.scala +++ b/audio-api/src/main/scala/no/ndla/audioapi/service/WriteService.scala @@ -9,14 +9,14 @@ package no.ndla.audioapi.service import cats.implicits.* import com.typesafe.scalalogging.StrictLogging -import no.ndla.audioapi.model.api.{AudioStorageException, MissingIdException, NotFoundException} +import no.ndla.audioapi.model.api.{AudioStorageException, MissingIdException} import no.ndla.audioapi.model.domain.Audio import no.ndla.audioapi.model.{api, domain} import no.ndla.audioapi.repository.{AudioRepository, SeriesRepository} import no.ndla.audioapi.service.search.{AudioIndexService, SeriesIndexService, TagIndexService} import no.ndla.common.Clock import no.ndla.common.aws.NdlaS3Client -import no.ndla.common.errors.ValidationException +import no.ndla.common.errors.{NotFoundException, ValidationException} import no.ndla.common.model.domain.UploadedFile import no.ndla.common.model.domain as common import no.ndla.language.Language.findByLanguageOrBestEffort @@ -278,7 +278,7 @@ trait WriteService { user: TokenUser ): Try[api.AudioMetaInformationDTO] = { audioRepository.withId(id) match { - case None => Failure(new NotFoundException) + case None => Failure(NotFoundException("Audio not found")) case Some(existingMetadata) => val metadataAndFile = fileOpt match { case None => mergeAudioMeta(existingMetadata, metadataToUpdate, None, user) diff --git a/audio-api/src/test/scala/no/ndla/audioapi/TestEnvironment.scala b/audio-api/src/test/scala/no/ndla/audioapi/TestEnvironment.scala index 5f006c814..a20e8992a 100644 --- a/audio-api/src/test/scala/no/ndla/audioapi/TestEnvironment.scala +++ b/audio-api/src/test/scala/no/ndla/audioapi/TestEnvironment.scala @@ -15,7 +15,8 @@ import no.ndla.audioapi.repository.{AudioRepository, SeriesRepository} import no.ndla.audioapi.service.* import no.ndla.audioapi.service.search.* import no.ndla.common.Clock -import no.ndla.common.aws.NdlaS3Client +import no.ndla.common.aws.{NdlaAWSTranscribeClient, NdlaS3Client} +import no.ndla.common.brightcove.NdlaBrightcoveClient import no.ndla.database.DataSource import no.ndla.network.NdlaClient import no.ndla.network.tapir.TapirApplication @@ -51,23 +52,29 @@ trait TestEnvironment with MockitoSugar with Clock with Props + with TranscriptionService + with NdlaAWSTranscribeClient + with NdlaBrightcoveClient with ErrorHandling { override val props: AudioApiProperties = new AudioApiProperties val dataSource: HikariDataSource = mock[HikariDataSource] - val storageName: String = props.StorageName val audioRepository: AudioRepository = mock[AudioRepository] val seriesRepository: SeriesRepository = mock[SeriesRepository] - val s3Client: NdlaS3Client = mock[NdlaS3Client] + val s3Client: NdlaS3Client = mock[NdlaS3Client] + val brightcoveClient: NdlaBrightcoveClient = mock[NdlaBrightcoveClient] + val transcribeClient: NdlaAWSTranscribeClient = mock[NdlaAWSTranscribeClient] val ndlaClient: NdlaClient = mock[NdlaClient] val myndlaApiClient: MyNDLAApiClient = mock[MyNDLAApiClient] - val readService: ReadService = mock[ReadService] - val writeService: WriteService = mock[WriteService] - val validationService: ValidationService = mock[ValidationService] - val converterService: ConverterService = mock[ConverterService] + val readService: ReadService = mock[ReadService] + val writeService: WriteService = mock[WriteService] + val validationService: ValidationService = mock[ValidationService] + val converterService: ConverterService = mock[ConverterService] + val transcriptionService: TranscriptionService = mock[TranscriptionService] + val s3TranscribeClient: NdlaS3Client = mock[NdlaS3Client] val internController: InternController = mock[InternController] val audioApiController: AudioController = mock[AudioController] diff --git a/audio-api/src/test/scala/no/ndla/audioapi/UnitSuite.scala b/audio-api/src/test/scala/no/ndla/audioapi/UnitSuite.scala index e06de1ad8..7a8ddc91e 100644 --- a/audio-api/src/test/scala/no/ndla/audioapi/UnitSuite.scala +++ b/audio-api/src/test/scala/no/ndla/audioapi/UnitSuite.scala @@ -24,4 +24,7 @@ trait UnitSuite extends UnitTestSuite with PrivateMethodTester { setPropEnv("SEARCH_REGION", "some-region") setPropEnv("RUN_WITH_SIGNED_SEARCH_REQUESTS", "false") setPropEnv("SEARCH_INDEX_NAME", "audio-integration-test-index") + setPropEnv("BRIGHTCOVE_API_CLIENT_ID", "client-id") + setPropEnv("BRIGHTCOVE_API_CLIENT_SECRET", "client") + setPropEnv("BRIGHTCOVE_ACCOUNT_ID", "312532") } diff --git a/audio-api/src/test/scala/no/ndla/audioapi/service/TranscriptionServiceTest.scala b/audio-api/src/test/scala/no/ndla/audioapi/service/TranscriptionServiceTest.scala new file mode 100644 index 000000000..8d64d3678 --- /dev/null +++ b/audio-api/src/test/scala/no/ndla/audioapi/service/TranscriptionServiceTest.scala @@ -0,0 +1,77 @@ +/* + * Part of NDLA audio-api + * Copyright (C) 2024 NDLA + * + * See LICENSE + * + */ + +package no.ndla.audioapi.service + +import no.ndla.audioapi.{AudioApiProperties, TestEnvironment, UnitSuite} +import no.ndla.common.aws.NdlaS3Object +import org.mockito.ArgumentMatchers.any +import org.mockito.Mockito.when +import software.amazon.awssdk.services.transcribe.model.{ + GetTranscriptionJobResponse, + StartTranscriptionJobResponse, + TranscriptionJob, + TranscriptionJobStatus +} + +import scala.util.Success + +class TranscriptionServiceTest extends UnitSuite with TestEnvironment { + override val transcriptionService: TranscriptionService = new TranscriptionService + override val brightcoveClient: NdlaBrightcoveClient = new NdlaBrightcoveClient + override val props: AudioApiProperties = new AudioApiProperties { + override val BrightcoveAccountId: String = "123" + override val BrightcoveClientId: String = "123" + override val BrightcoveClientSecret: String = "123" + } + + test("getAudioExtractionStatus returns Success when audio file exists") { + val videoId = "1" + val language = "en" + val fakeS3Object = mock[NdlaS3Object] + when(s3TranscribeClient.getObject(any)).thenReturn(Success(fakeS3Object)) + val result = transcriptionService.getAudioExtractionStatus(videoId, language) + + result should be(Success(())) + } + + test("getTranscription returns status of a transcription") { + val videoId = "1" + val language = "en" + val fakeS3Object = mock[NdlaS3Object] + val fakeTranscribeResponse = mock[GetTranscriptionJobResponse] + val fakeJob = mock[TranscriptionJob] + val fakeJobStatus = mock[TranscriptionJobStatus] + when(s3TranscribeClient.getObject(any)).thenReturn(Success(fakeS3Object)) + + when(fakeJob.transcriptionJobStatus()).thenReturn(fakeJobStatus) + when(fakeTranscribeResponse.transcriptionJob()).thenReturn(fakeJob) + when(transcribeClient.getTranscriptionJob(any)).thenReturn(Success(fakeTranscribeResponse)) + + val result = transcriptionService.getVideoTranscription(videoId, language) + + result should be(Success(Left(fakeJobStatus.toString))) + } + + test("transcribeVideo returns Success when transcription is started") { + val videoId = "1" + val language = "no-NO" + val maxSpeakers = 2 + val fakeS3Object = mock[NdlaS3Object] + val fakeTranscribeMock = mock[StartTranscriptionJobResponse] + when(transcribeClient.getTranscriptionJob(any)).thenReturn(Success(mock[GetTranscriptionJobResponse])) + when(s3TranscribeClient.getObject(any)).thenReturn(Success(fakeS3Object)) + when(transcriptionService.getAudioExtractionStatus(videoId, language)).thenReturn(Success(())) + when(transcribeClient.startTranscriptionJob(any, any, any, any, any, any, any, any, any)) + .thenReturn(Success(fakeTranscribeMock)) + val result = transcriptionService.transcribeVideo(videoId, language, maxSpeakers) + + result should be(Success(())) + } + +} diff --git a/audio-api/src/test/scala/no/ndla/audioapi/service/WriteServiceTest.scala b/audio-api/src/test/scala/no/ndla/audioapi/service/WriteServiceTest.scala index 8190e83b8..e053e30dc 100644 --- a/audio-api/src/test/scala/no/ndla/audioapi/service/WriteServiceTest.scala +++ b/audio-api/src/test/scala/no/ndla/audioapi/service/WriteServiceTest.scala @@ -12,7 +12,7 @@ import no.ndla.audioapi.model.api.* import no.ndla.audioapi.model.domain.{Audio, AudioType} import no.ndla.audioapi.model.{api, domain} import no.ndla.audioapi.{TestData, TestEnvironment, UnitSuite} -import no.ndla.common.errors.{ValidationException, ValidationMessage} +import no.ndla.common.errors.{NotFoundException, ValidationException, ValidationMessage} import no.ndla.common.model import no.ndla.common.model.api.{CopyrightDTO, LicenseDTO} import no.ndla.common.model.domain.UploadedFile @@ -393,7 +393,7 @@ class WriteServiceTest extends UnitSuite with TestEnvironment { val result = writeService.updateAudio(1, updatedAudioMeta, None, testUser) result.isFailure should be(true) - result.failed.get.getMessage should equal(new NotFoundException().getMessage) + result.failed.get.getMessage should equal(NotFoundException("Audio not found").getMessage) } test("that updateAudio returns Failure when audio file validation fails") { diff --git a/common/src/main/scala/no/ndla/common/aws/NdlaAWSTranscribeClient.scala b/common/src/main/scala/no/ndla/common/aws/NdlaAWSTranscribeClient.scala new file mode 100644 index 000000000..3704c86dc --- /dev/null +++ b/common/src/main/scala/no/ndla/common/aws/NdlaAWSTranscribeClient.scala @@ -0,0 +1,98 @@ +/* + * Part of NDLA common + * Copyright (C) 2024 NDLA + * + * See LICENSE + */ + +package no.ndla.common.aws + +import software.amazon.awssdk.services.transcribe.model.* +import software.amazon.awssdk.services.transcribe.{TranscribeClient, TranscribeClientBuilder} + +import scala.util.{Failure, Try} + +trait NdlaAWSTranscribeClient { + val transcribeClient: NdlaAWSTranscribeClient + + class NdlaAWSTranscribeClient(region: Option[String]) { + + private val builder: TranscribeClientBuilder = TranscribeClient.builder() + + val client: TranscribeClient = region match { + case Some(value) => builder.region(software.amazon.awssdk.regions.Region.of(value)).build() + case None => builder.build() + } + + def startTranscriptionJob( + jobName: String, + mediaUri: String, + mediaFormat: String, + languageCode: String, + outputBucket: String, + outputKey: String, + maxSpeakers: Int, + includeSubtitles: Boolean = true, + outputSubtitleFormat: String = "VTT" + ): Try[StartTranscriptionJobResponse] = Try { + val requestBuilder = StartTranscriptionJobRequest + .builder() + .transcriptionJobName(jobName) + .media(Media.builder().mediaFileUri(mediaUri).build()) + .mediaFormat(mediaFormat) + .languageCode(languageCode) + .outputBucketName(outputBucket) + .outputKey(outputKey) + .settings( + Settings + .builder() + .showSpeakerLabels(true) + .maxSpeakerLabels(maxSpeakers) + .build() + ) + + if (includeSubtitles) { + requestBuilder.subtitles( + Subtitles + .builder() + .formats(SubtitleFormat.valueOf(outputSubtitleFormat)) + .build() + ) + } + + client.startTranscriptionJob(requestBuilder.build()) + } + + def getTranscriptionJob(jobName: String): Try[GetTranscriptionJobResponse] = { + Try { + val request = GetTranscriptionJobRequest + .builder() + .transcriptionJobName(jobName) + .build() + client.getTranscriptionJob(request) + }.recoverWith { case e: BadRequestException => + val nfe = no.ndla.common.errors.NotFoundException("Transcription job not found") + Failure(nfe.initCause(e)) + } + } + + def listTranscriptionJobs(status: Option[String] = None): Try[ListTranscriptionJobsResponse] = Try { + val requestBuilder = ListTranscriptionJobsRequest.builder() + val request = status match { + case Some(jobStatus) => requestBuilder.status(jobStatus).build() + case None => requestBuilder.build() + } + + client.listTranscriptionJobs(request) + } + + def deleteTranscriptionJob(jobName: String): Try[DeleteTranscriptionJobResponse] = Try { + val request = DeleteTranscriptionJobRequest + .builder() + .transcriptionJobName(jobName) + .build() + + client.deleteTranscriptionJob(request) + } + } +} diff --git a/common/src/main/scala/no/ndla/common/brightcove/NdlaBrightcoveClient.scala b/common/src/main/scala/no/ndla/common/brightcove/NdlaBrightcoveClient.scala new file mode 100644 index 000000000..7bfc26302 --- /dev/null +++ b/common/src/main/scala/no/ndla/common/brightcove/NdlaBrightcoveClient.scala @@ -0,0 +1,72 @@ +/* + * Part of NDLA common + * Copyright (C) 2024 NDLA + * + * See LICENSE + * + */ + +package no.ndla.common.brightcove + +import io.circe.Json +import io.circe.generic.codec.DerivedAsObjectCodec.deriveCodec +import io.circe.parser.* +import sttp.client3.{HttpClientSyncBackend, UriContext, basicRequest} +import no.ndla.common.configuration.HasBaseProps + +import scala.util.Try + +case class TokenResponse(access_token: String, token_type: String, expires_in: Int) + +trait NdlaBrightcoveClient { + this: HasBaseProps => + val brightcoveClient: NdlaBrightcoveClient + + class NdlaBrightcoveClient { + private val backend = HttpClientSyncBackend() + + def getToken(clientID: String, clientSecret: String): Try[String] = { + val request = + basicRequest.auth + .basic(clientID, clientSecret) + .post(uri"${props.BrightCoveAuthUri}?grant_type=client_credentials") + val authResponse = request.send(backend) + Try { + authResponse.body match { + case Right(jsonString) => + decode[TokenResponse](jsonString) match { + case Right(tokenResponse) => tokenResponse.access_token + case Left(error) => throw new Exception(s"Failed to decode token response: ${error.getMessage}") + } + case Left(error) => throw new Exception(s"Failed to get token: ${error}") + } + } + } + + def getVideoSource(accountId: String, videoId: String, bearerToken: String): Try[Vector[Json]] = { + + val videoSourceUrl = props.BrightCoveVideoUri(accountId, videoId) + val request = basicRequest + .header("Authorization", s"Bearer $bearerToken") + .get(videoSourceUrl) + + implicit val backend = HttpClientSyncBackend() + + val response = request.send(backend) + Try { + response.body match { + case Right(jsonString) => + parse(jsonString) match { + case Right(json) => + json.asArray match { + case Some(videoSources) => videoSources + case None => throw new Exception("Failed to parse video source") + } + case Left(error) => throw new Exception(s"Failed to parse video source: ${error.getMessage}") + } + case Left(error) => throw new Exception(s"Failed to get video source: ${error}") + } + } + } + } +} diff --git a/common/src/main/scala/no/ndla/common/configuration/BaseProps.scala b/common/src/main/scala/no/ndla/common/configuration/BaseProps.scala index 556cc52f6..7ecd5f951 100644 --- a/common/src/main/scala/no/ndla/common/configuration/BaseProps.scala +++ b/common/src/main/scala/no/ndla/common/configuration/BaseProps.scala @@ -1,5 +1,8 @@ package no.ndla.common.configuration +import sttp.client3.UriContext +import sttp.model.Uri + import scala.util.Properties.{propOrElse, propOrNone} trait BaseProps { @@ -53,4 +56,9 @@ trait BaseProps { def SEARCH_INDEX_REPLICAS: Int = intPropOrDefault("SEARCH_INDEX_REPLICAS", 1) def TAPIR_THREADS: Int = intPropOrDefault("TAPIR_THREADS", 100) + + def BrightCoveAuthUri: String = s"https://oauth.brightcove.com/v4/access_token" + def BrightCoveVideoUri(accountId: String, videoId: String): Uri = + uri"https://cms.api.brightcove.com/v1/accounts/$accountId/videos/$videoId/sources" + } diff --git a/network/src/main/scala/no/ndla/network/tapir/TapirErrorHandling.scala b/network/src/main/scala/no/ndla/network/tapir/TapirErrorHandling.scala index 9c7fb791a..066b249b5 100644 --- a/network/src/main/scala/no/ndla/network/tapir/TapirErrorHandling.scala +++ b/network/src/main/scala/no/ndla/network/tapir/TapirErrorHandling.scala @@ -78,6 +78,7 @@ trait TapirErrorHandling extends StrictLogging { val VALIDATION = "VALIDATION_ERROR" val METHOD_NOT_ALLOWED = "METHOD_NOT_ALLOWED" val CONFLICT = "CONFLICT" + val JOB_ALREADY_FOUND = "JOB_ALREADY_FOUND" val PARAMETER_MISSING = "PARAMETER MISSING" val PROVIDER_NOT_SUPPORTED = "PROVIDER NOT SUPPORTED" diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 99c72f0c3..b15962d1d 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -65,6 +65,10 @@ object Dependencies { "software.amazon.awssdk" % "s3" % AwsSdkV ) + lazy val awsTranscribe: Seq[ModuleID] = Seq( + "software.amazon.awssdk" % "transcribe" % AwsSdkV + ) + lazy val awsCloudwatch: Seq[ModuleID] = Seq( "software.amazon.awssdk" % "cloudwatch" % AwsSdkV ) @@ -134,5 +138,9 @@ object Dependencies { "org.apache.httpcomponents" % "httpclient" % "4.5.14", "org.yaml" % "snakeyaml" % "2.0" ) + lazy val jave: Seq[ModuleID] = Seq( + "ws.schild" % "jave-core" % "3.5.0", + "ws.schild" % "jave-all-deps" % "3.5.0" + ) } } diff --git a/project/Module.scala b/project/Module.scala index ed4fd485d..2ec80029a 100644 --- a/project/Module.scala +++ b/project/Module.scala @@ -168,7 +168,6 @@ trait Module { ) ) } - val checkfmt = taskKey[Unit]("Check for code style errors") val fmt = taskKey[Unit]("Automatically apply code style fixes") diff --git a/project/audioapi.scala b/project/audioapi.scala index 3298bf850..5b5442375 100644 --- a/project/audioapi.scala +++ b/project/audioapi.scala @@ -23,7 +23,8 @@ object audioapi extends Module { elastic4s, database, tapirHttp4sCirce, - vulnerabilityOverrides + vulnerabilityOverrides, + jave ) lazy val tsSettings: Seq[Def.Setting[?]] = typescriptSettings( @@ -41,7 +42,8 @@ object audioapi extends Module { "AudioMetaInformationDTO", "UpdatedAudioMetaInformationDTO", "SeriesSummarySearchResultDTO", - "SeriesSearchParamsDTO" + "SeriesSearchParamsDTO", + "TranscriptionResultDTO" ) ) diff --git a/project/commonlib.scala b/project/commonlib.scala index 0e8dfabff..40582a16c 100644 --- a/project/commonlib.scala +++ b/project/commonlib.scala @@ -17,6 +17,7 @@ object commonlib extends Module { scalaTsi ), awsS3, + awsTranscribe, melody, tapirHttp4sCirce ) diff --git a/typescript/types-backend/audio-api.ts b/typescript/types-backend/audio-api.ts index c53e0c2ad..a31ed95b7 100644 --- a/typescript/types-backend/audio-api.ts +++ b/typescript/types-backend/audio-api.ts @@ -196,6 +196,11 @@ export interface ITitleDTO { language: string } +export interface ITranscriptionResultDTO { + status: string + transcription?: string +} + export interface IUpdatedAudioMetaInformationDTO { revision: number title: string