diff --git a/app/controllers/BanditDataController.scala b/app/controllers/BanditDataController.scala new file mode 100644 index 00000000..e2a14777 --- /dev/null +++ b/app/controllers/BanditDataController.scala @@ -0,0 +1,35 @@ +package controllers + +import com.gu.googleauth.AuthAction +import com.typesafe.scalalogging.LazyLogging +import io.circe.syntax.EncoderOps +import play.api.mvc.{AbstractController, Action, AnyContent, ControllerComponents, Result} +import services.DynamoBanditData +import utils.Circe.noNulls +import zio.{IO, ZEnv, ZIO} + +import scala.concurrent.{ExecutionContext, Future} + +class BanditDataController(authAction: AuthAction[AnyContent], + components: ControllerComponents, + stage: String, + runtime: zio.Runtime[ZEnv], + dynamo: DynamoBanditData, +)(implicit ec: ExecutionContext) extends AbstractController(components) with LazyLogging { + + private def run(f: => ZIO[ZEnv, Throwable, Result]): Future[Result] = + runtime.unsafeRunToFuture { + f.catchAll(error => { + logger.error(s"Returning InternalServerError to client: ${error.getMessage}", error) + IO.succeed(InternalServerError(error.getMessage)) + }) + } + + def getDataForTest(channel: String, testName: String): Action[AnyContent] = authAction.async { request => + run { + dynamo + .getDataForTest(testName, channel) + .map(data => Ok(noNulls(data.asJson))) + } + } +} diff --git a/app/services/DynamoBanditData.scala b/app/services/DynamoBanditData.scala new file mode 100644 index 00000000..b60c36ee --- /dev/null +++ b/app/services/DynamoBanditData.scala @@ -0,0 +1,119 @@ +package services + +import com.typesafe.scalalogging.StrictLogging +import io.circe.{Decoder, Encoder} +import models.DynamoErrors.{DynamoError, DynamoGetError} +import software.amazon.awssdk.services.dynamodb.DynamoDbClient +import software.amazon.awssdk.services.dynamodb.model.{AttributeValue, QueryRequest} +import utils.Circe.dynamoMapToJson +import io.circe.generic.auto._ +import zio.blocking.effectBlocking +import zio.{ZEnv, ZIO} + +import scala.jdk.CollectionConverters._ + +/** + * Models for data received from DynamoDb + */ + +case class VariantSample(variantName: String, annualisedValueInGBP: Double, annualisedValueInGBPPerView: Double, views: Double) +case class TestSample(testName: String, variants: List[VariantSample], timestamp: String) +object TestSample { + implicit val decoder = Decoder[TestSample] + implicit val encoder = Encoder[TestSample] +} + +/** + * Models for data returned to the client + */ + +// models the mean and views for each variant up to a certain timestamp +case class EnrichedVariantSampleData(variantName: String, views: Double, mean: Double) +case class EnrichedTestSampleData(timestamp: String, variants: List[EnrichedVariantSampleData]) + +// Final mean and views for a variant +case class VariantSummary(variantName: String, mean: Double, views: Double) + +case class BanditData(variantSummaries: List[VariantSummary], samples: List[EnrichedTestSampleData]) +object BanditData { + implicit val decoder = Decoder[BanditData] + implicit val encoder = Encoder[BanditData] +} + +class DynamoBanditData(stage: String, client: DynamoDbClient) extends StrictLogging { + private val tableName = s"support-bandit-$stage" + + private def query(testName: String, channel: String): ZIO[ZEnv, DynamoGetError, java.util.List[java.util.Map[String, AttributeValue]]] = { + effectBlocking { + client.query( + QueryRequest + .builder() + .tableName(tableName) + .keyConditionExpression("testName = :testName") + .expressionAttributeValues(Map( + ":testName" -> AttributeValue.builder.s(s"${channel}_$testName").build + ).asJava) + .scanIndexForward(true) + .build() + ).items() + }.mapError(DynamoGetError) + } + + private def sampleMean(samples: List[VariantSample], population: Double): Double = + samples.foldLeft(0D)((acc, sample) => + acc + (sample.views / population) * sample.annualisedValueInGBPPerView + ) + + private def buildVariantSummaries(samples: List[TestSample]): List[VariantSummary] = + samples + .flatMap(_.variants) + .groupBy(variantSample => variantSample.variantName) + .map { case (variantName, samples) => + val population = samples.map(_.views).sum + val mean = sampleMean(samples, population) + VariantSummary(variantName = variantName, mean = mean, views = population) + } + .toList + + // For each hourly sample, calculate the means up to that point, so that we can visualise how the Bandit's view of the variants changed over time + private def buildEnrichedSamples(samples: List[TestSample]): List[EnrichedTestSampleData] = { + val samplesByVariant: Map[String, List[VariantSample]] = samples + .flatMap(_.variants) + .groupBy(variantSample => variantSample.variantName) + + samples + .zipWithIndex + .foldLeft(Array.empty[EnrichedTestSampleData]) { case (enrichedSamples, (sample, idx)) => + val variants = sample.variants.map(currentVariantSample => { + val previousSamples = samplesByVariant(currentVariantSample.variantName).take(idx+1) + + val population = previousSamples.map(_.views).sum + val mean = sampleMean(previousSamples, population) + EnrichedVariantSampleData(currentVariantSample.variantName, currentVariantSample.views, mean) + }) + + enrichedSamples :+ EnrichedTestSampleData(sample.timestamp, variants) + } + .toList + } + + def getDataForTest(testName: String, channel: String): ZIO[ZEnv, DynamoError, BanditData] = + query(testName, channel) + .map { results => + results.asScala + .map(item => dynamoMapToJson(item).as[TestSample]) + .flatMap { + case Right(row) => Some(row) + case Left(error) => + logger.error(s"Failed to decode item from Dynamo: ${error.getMessage}") + None + } + .toList + } + .map { samples: List[TestSample] => + val variantSummaries = buildVariantSummaries(samples) + val enrichedSamples = buildEnrichedSamples(samples) + + BanditData(variantSummaries, enrichedSamples) + } +} diff --git a/app/wiring/AppComponents.scala b/app/wiring/AppComponents.scala index 70762b52..5be6796f 100644 --- a/app/wiring/AppComponents.scala +++ b/app/wiring/AppComponents.scala @@ -11,7 +11,7 @@ import play.api.libs.ws.ahc.AhcWSComponents import play.api.mvc.AnyContent import play.api.{BuiltInComponentsFromContext, NoHttpFiltersComponents} import router.Routes -import services.{Athena, Aws, CapiService, DynamoArchivedBannerDesigns, DynamoArchivedChannelTests, DynamoBannerDesigns, DynamoCampaigns, DynamoChannelTests, DynamoSuperMode, S3} +import services.{Athena, Aws, CapiService, DynamoArchivedBannerDesigns, DynamoArchivedChannelTests, DynamoBanditData, DynamoBannerDesigns, DynamoCampaigns, DynamoChannelTests, DynamoSuperMode, S3} import software.amazon.awssdk.services.dynamodb.DynamoDbClient import software.amazon.awssdk.services.s3.model.GetObjectRequest @@ -73,6 +73,8 @@ class AppComponents(context: Context, stage: String) extends BuiltInComponentsFr val dynamoBannerDesigns = new DynamoBannerDesigns(stage, dynamoClient) val dynamoArchivedBannerDesigns = new DynamoArchivedBannerDesigns(stage, dynamoClient) + val dynamoBanditData = new DynamoBanditData(stage, dynamoClient) + val sdcUrlOverride: Option[String] = sys.env.get("SDC_URL") override lazy val router: Router = new Routes( @@ -98,6 +100,7 @@ class AppComponents(context: Context, stage: String) extends BuiltInComponentsFr new DefaultPromosController(authAction,controllerComponents, stage, runtime), new SuperModeController(authAction, controllerComponents, stage, runtime, dynamoSuperModeService, athena), new AnalyticsController(authAction, controllerComponents, stage, runtime, athena), + new BanditDataController(authAction, controllerComponents, stage, runtime, dynamoBanditData), assets ) } diff --git a/conf/routes b/conf/routes index c9a77450..e5dee122 100644 --- a/conf/routes +++ b/conf/routes @@ -228,6 +228,7 @@ GET /frontend/epic-article-data controllers.SuperModeCont # ----- analytics ----- # GET /frontend/analytics/:channel/:testName controllers.AnalyticsController.getDataForVariants(channel: String, testName: String) +GET /frontend/bandit/:channel/:testName controllers.BanditDataController.getDataForTest(channel: String, testName: String) # Map static resources from the /public folder to the /assets URL path GET /assets/*file controllers.Assets.at(path="/public", file) diff --git a/public/src/components/channelManagement/BanditAnalyticsButton.tsx b/public/src/components/channelManagement/BanditAnalyticsButton.tsx new file mode 100644 index 00000000..36f7a62d --- /dev/null +++ b/public/src/components/channelManagement/BanditAnalyticsButton.tsx @@ -0,0 +1,145 @@ +import { makeStyles } from '@mui/styles'; +import { Button, Dialog, Theme, Typography } from '@mui/material'; +import React, { useEffect } from 'react'; +import useOpenable from '../../hooks/useOpenable'; +import { format } from 'date-fns'; +import { LineChart, CartesianGrid, Line, XAxis, YAxis, Legend, Tooltip } from 'recharts'; + +const useStyles = makeStyles(({}: Theme) => ({ + dialog: { + padding: '10px', + }, + heading: { + margin: '6px 12px 0 12px', + fontSize: 18, + fontWeight: 500, + }, + chartContainer: { + margin: '12px', + }, +})); + +interface VariantSummary { + variantName: string; + mean: number; + views: number; +} +interface VariantSample { + variantName: string; + views: number; + mean: number; +} +interface TestSample { + timestamp: string; + variants: VariantSample[]; +} +interface BanditData { + variantSummaries: VariantSummary[]; + samples: TestSample[]; +} + +interface SamplesChartProps { + data: BanditData; + variantNames: string[]; + fieldName: keyof VariantSample; +} + +const Colours = ['red', 'blue', 'green', 'orange', 'yellow']; + +type ChartDataPoint = Record; + +const SamplesChart = ({ data, variantNames, fieldName }: SamplesChartProps) => { + const chartData = data.samples + .map(({ timestamp, variants }) => { + if (variants.length > 0) { + const sample: ChartDataPoint = { + dateHour: format(Date.parse(timestamp), 'yyyy-MM-dd hh:mm'), + }; + variants.forEach(variant => { + sample[variant.variantName] = variant[fieldName]; + }); + return sample; + } + return undefined; + }) + .filter(sample => !!sample); + + return ( + + + + + + + {variantNames.map((name, idx) => ( + + ))} + + ); +}; + +interface BanditAnalyticsButton { + testName: string; + channel: string; +} + +export const BanditAnalyticsButton: React.FC = ({ + testName, + channel, +}: BanditAnalyticsButton) => { + const classes = useStyles(); + const [isOpen, open, close] = useOpenable(); + const [data, setData] = React.useState(); + const [loading, setLoading] = React.useState(false); + + useEffect(() => { + setLoading(true); + + fetch(`/frontend/bandit/${channel}/${testName}`) + .then(resp => resp.json()) + .then(data => { + setData(data); + setLoading(false); + }); + }, [testName, channel]); + + const variantNames = data?.variantSummaries.map(variant => variant.variantName) ?? []; + + return ( + <> + + + +
+

Bandit data for: {testName}

+
+ +
+ {loading && Loading...} + {data && ( +
+

Impressions

+ + +

Scores

+ + +

Total impressions:

+ {data.variantSummaries + .sort((va, vb) => vb.views - va.views) + .map(({ variantName, views }) => ( +
+ + {variantName}: {views} + +
+ ))} +
+ )} +
+
+ + ); +}; diff --git a/public/src/components/channelManagement/TestMethodologyEditor.tsx b/public/src/components/channelManagement/TestMethodologyEditor.tsx index d9fd9977..15874303 100644 --- a/public/src/components/channelManagement/TestMethodologyEditor.tsx +++ b/public/src/components/channelManagement/TestMethodologyEditor.tsx @@ -1,6 +1,7 @@ import React from 'react'; import { Methodology } from './helpers/shared'; import { makeStyles } from '@mui/styles'; +import { BanditAnalyticsButton } from './BanditAnalyticsButton'; import { Button, MenuItem, @@ -63,8 +64,9 @@ const defaultEpsilonGreedyBandit: Methodology = { interface TestMethodologyProps { methodology: Methodology; - audiencePercentage: number; testName: string; + channel: string; + audiencePercentage: number; isDisabled: boolean; onChange: (methodology: Methodology) => void; onDelete: () => void; @@ -72,6 +74,8 @@ interface TestMethodologyProps { const TestMethodology: React.FC = ({ methodology, + testName, + channel, audiencePercentage, isDisabled, onChange, @@ -103,8 +107,8 @@ const TestMethodology: React.FC = ({ -
- {methodology.name === 'EpsilonGreedyBandit' && ( + {methodology.name === 'EpsilonGreedyBandit' && ( + <>
= ({ }} />
- )} -
+ + + )}
{methodology.testName &&
{methodology.testName}
}
@@ -135,6 +140,7 @@ const TestMethodology: React.FC = ({ interface TestMethodologyEditorProps { methodologies: Methodology[]; testName: string; + channel: string; onChange: (methodologies: Methodology[]) => void; isDisabled: boolean; } @@ -142,6 +148,7 @@ interface TestMethodologyEditorProps { export const TestMethodologyEditor: React.FC = ({ methodologies, testName, + channel, onChange, isDisabled, }: TestMethodologyEditorProps) => { @@ -174,6 +181,7 @@ export const TestMethodologyEditor: React.FC = ({ key={`methodology-${idx}`} methodology={method} testName={testName} + channel={channel} audiencePercentage={Math.round(100 / methodologies.length)} isDisabled={isDisabled} onChange={updatedMethodology => { diff --git a/public/src/components/channelManagement/bannerTests/bannerTestEditor.tsx b/public/src/components/channelManagement/bannerTests/bannerTestEditor.tsx index 72589778..82b49d2c 100644 --- a/public/src/components/channelManagement/bannerTests/bannerTestEditor.tsx +++ b/public/src/components/channelManagement/bannerTests/bannerTestEditor.tsx @@ -241,6 +241,7 @@ const BannerTestEditor: React.FC> = ({ diff --git a/public/src/components/channelManagement/epicTests/testEditor.tsx b/public/src/components/channelManagement/epicTests/testEditor.tsx index 9a0644fa..6fe25c92 100644 --- a/public/src/components/channelManagement/epicTests/testEditor.tsx +++ b/public/src/components/channelManagement/epicTests/testEditor.tsx @@ -267,6 +267,7 @@ export const getEpicTestEditor = (