From 9d309f11f2a8c9677f2abb8bf2d564cfdfedb8eb Mon Sep 17 00:00:00 2001 From: Manuel Kiessling Date: Wed, 1 Mar 2017 14:22:52 +0100 Subject: [PATCH] Added logging for #48, "HAR seems to contain HTTP status codes which cannot be converted to Int". --- .../analyze/spark/SparkApp.scala | 14 ++- .../analyze/spark/SparkAppSpec.scala | 93 ++++++++++++++++++- 2 files changed, 103 insertions(+), 4 deletions(-) diff --git a/spark/src/main/scala/com/journeymonitor/analyze/spark/SparkApp.scala b/spark/src/main/scala/com/journeymonitor/analyze/spark/SparkApp.scala index 3502daa..cb7bcfb 100644 --- a/spark/src/main/scala/com/journeymonitor/analyze/spark/SparkApp.scala +++ b/spark/src/main/scala/com/journeymonitor/analyze/spark/SparkApp.scala @@ -4,6 +4,7 @@ import java.text.SimpleDateFormat import java.util.Calendar import com.datastax.spark.connector._ +import org.apache.log4j.LogManager import org.apache.spark._ import org.apache.spark.rdd.RDD import org.json4s._ @@ -48,7 +49,18 @@ object HarAnalyzer { implicit val formats = org.json4s.DefaultFormats val requestCounter = for { entry <- entries - if ((entry \ "response" \ "status").extract[Int] >= status && (entry \ "response" \ "status").extract[Int] < status + 100) + if { + try { + val entryStatus = (entry \ "response" \ "status").extract[Int] + entryStatus >= status && entryStatus < status + 100 + } catch { + case e: Exception => { + val log = LogManager.getRootLogger + log.warn(s"Problem: '${e.getMessage}' while trying to get the status code at 'response -> status' within ${entry.toString()}") + false + } + } + } } yield 1 if (requestCounter.isEmpty) 0 else requestCounter.reduce(_ + _) // This is a "normal" Scala reduce, not an RDD reduce. diff --git a/spark/src/test/scala/com/journeymonitor/analyze/spark/SparkAppSpec.scala b/spark/src/test/scala/com/journeymonitor/analyze/spark/SparkAppSpec.scala index 8e0976e..bcf0ab1 100644 --- a/spark/src/test/scala/com/journeymonitor/analyze/spark/SparkAppSpec.scala +++ b/spark/src/test/scala/com/journeymonitor/analyze/spark/SparkAppSpec.scala @@ -8,7 +8,7 @@ import org.json4s.jackson.JsonMethods._ import org.scalatest.{BeforeAndAfter, FunSpec, Matchers} object FixtureGenerator { - def getTestresultsRDD(sc: SparkContext) = { + def getValidTestresultsRDD(sc: SparkContext) = { val format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss") val datetimeRun1 = format.parse("2015-01-02 23:59:59") val datetimeRun2 = format.parse("2015-11-18 00:00:00") @@ -68,6 +68,67 @@ object FixtureGenerator { ) )) } + + def getInvalidTestresultsRDD(sc: SparkContext) = { + val format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss") + val datetimeRun1 = format.parse("2015-01-02 23:59:59") + val datetimeRun2 = format.parse("2015-11-18 00:00:00") + sc.parallelize(Seq( + Testresult( + testcaseId = "testcaseId1", + testresultId = "testresultId1", + datetimeRun = datetimeRun1, + har = parse( + """ + |{ + | "log": { + | "entries": [ + | { + | "response": { + | "status": "foo" + | }, + | "time": 10 + | }, + | { + | "response": { + | "status": 400 + | }, + | "time": 15 + | } + | ] + | } + |} + """.stripMargin, false) + ), + Testresult( + testcaseId = "testcaseId1", + testresultId = "testresultId2", + datetimeRun = datetimeRun2, + har = parse( + """ +{ + | "log": { + | "entries": [ + | { + | "response": { + | "status": 400 + | }, + | "time": 16 + | }, + | { + | "response": { + | "status": 400 + | }, + | "time": 4 + | } + | ] + | } + |} + """.stripMargin, false) + + ) + )) + } } class SparkExampleSpec extends FunSpec with BeforeAndAfter with Matchers { @@ -92,8 +153,9 @@ class SparkExampleSpec extends FunSpec with BeforeAndAfter with Matchers { } describe("The HarAnalyzer") { - it("should extract statistics from HARs") { - val testresultsRDD = FixtureGenerator.getTestresultsRDD(sc) + + it("should extract statistics from valid HARs") { + val testresultsRDD = FixtureGenerator.getValidTestresultsRDD(sc) val statisticsRDD = HarAnalyzer.calculateRequestStatistics(testresultsRDD) val statistics = statisticsRDD.collect() @@ -115,6 +177,31 @@ class SparkExampleSpec extends FunSpec with BeforeAndAfter with Matchers { statistics(1).numberOfRequestsWithStatus400 should be(2) statistics(1).totalRequestTime should be(20) } + + it("should gracefully handle invalid HARs") { + val testresultsRDD = FixtureGenerator.getInvalidTestresultsRDD(sc) + val statisticsRDD = HarAnalyzer.calculateRequestStatistics(testresultsRDD) + val statistics = statisticsRDD.collect() + + statistics(0).testcaseId should be("testcaseId1") + statistics(0).dayBucket should be("2015-01-02") + statistics(0).testresultId should be("testresultId1") + statistics(0).testresultDatetimeRun.toString.substring(0, 20) should be("Fri Jan 02 23:59:59 ") // @TODO: Stupid hack because we do not yet store the timezone + statistics(0).testresultDatetimeRun.toString.substring(24) should be("2015") + statistics(0).numberOfRequestsWithStatus200 should be(0) + statistics(0).numberOfRequestsWithStatus400 should be(1) + statistics(0).totalRequestTime should be(25) + + statistics(1).testcaseId should be("testcaseId1") + statistics(1).dayBucket should be("2015-11-18") + statistics(1).testresultId should be("testresultId2") + statistics(1).testresultDatetimeRun.toString.substring(0, 20) should be("Wed Nov 18 00:00:00 ") + statistics(1).testresultDatetimeRun.toString.substring(24) should be("2015") + statistics(1).numberOfRequestsWithStatus200 should be(0) + statistics(1).numberOfRequestsWithStatus400 should be(2) + statistics(1).totalRequestTime should be(20) + } + } }