diff --git a/app-conf/elephant.conf b/app-conf/elephant.conf index e3ee9fb6f..bbe56a037 100644 --- a/app-conf/elephant.conf +++ b/app-conf/elephant.conf @@ -29,7 +29,7 @@ enable_analytics=false # Check https://www.playframework.com/documentation/2.2.x/ProductionConfiguration # Adding the below line for Heap Tuning and Java OPTS # Use mem for tuning Heap Memory -jvm_args="-Devolutionplugin=enabled -DapplyEvolutions.default=true -mem 1024 -J-Xloggc:$project_root../logs/elephant/dr-gc.`date +'%Y%m%d%H%M'` -J-XX:+PrintGCDetails" +jvm_args="-Devolutionplugin=disabled -DapplyEvolutions.default=true -mem 1024 -J-Xloggc:$project_root../logs/elephant/dr-gc.`date +'%Y%m%d%H%M'` -J-XX:+PrintGCDetails" # Property enables dropwizard metrics for the application. @@ -57,7 +57,7 @@ metrics=true # metrics_agent_jar="-javaagent:lib/your_agent.jar=app-name=dr-elephant,app-host=foo" # -#Filter for applications to be processed. This filter will be applied in API call to RM for getting -#applications finished in last window. Useful to restrict applications to be processed. +#Filter for applications to be processed. This filter will be applied in API call to RM for getting +#applications finished in last window. Useful to restrict applications to be processed. # #rm_application_filter="applicationTags=tag1&user=user1&applicationType=SPARK diff --git a/app/Global.java b/app/Global.java index 9b346e0b5..ec5d199ee 100644 --- a/app/Global.java +++ b/app/Global.java @@ -1,19 +1,3 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - import com.linkedin.drelephant.DrElephant; import com.sun.security.sasl.util.AbstractSaslImpl; @@ -25,6 +9,11 @@ import java.lang.reflect.Field; import java.lang.reflect.Modifier; import java.util.logging.Level; +import play.libs.F; +import play.mvc.Action; +import play.mvc.Http; +import play.mvc.Result; +import play.mvc.SimpleResult; /** @@ -87,4 +76,29 @@ static void setFinalStatic(Field field, Object newValue) throws Exception { modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL); field.set(null, newValue); } -} + + private class ActionWrapper extends Action.Simple { + public ActionWrapper(Action action) { + this.delegate = action; + } + + @Override + public F.Promise call(Http.Context ctx) throws java.lang.Throwable { + F.Promise result = this.delegate.call(ctx); + Http.Response response = ctx.response(); + response.setHeader("Access-Control-Allow-Origin", "*"); + response.setHeader("Allow", "*"); + response.setHeader("Access-Control-Allow-Methods", "POST, GET, PUT, DELETE, OPTIONS"); + response.setHeader("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept, Referer, User-Agent"); + return result; + } + } + + @Override + public Action onRequest(Http.Request request, java.lang.reflect.Method actionMethod) { + return new ActionWrapper(super.onRequest(request, actionMethod)); + } + + + +} \ No newline at end of file diff --git a/app/controllers/Application.java b/app/controllers/Application.java index efe877319..33675db12 100644 --- a/app/controllers/Application.java +++ b/app/controllers/Application.java @@ -54,8 +54,10 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.HashSet; import java.util.TreeSet; +import javafx.util.Pair; import models.AppHeuristicResult; import models.AppResult; import models.JobDefinition; @@ -798,6 +800,8 @@ private static Result getJobHistory(Version version) { if (maxStages > STAGE_LIMIT) { maxStages = STAGE_LIMIT; } + + if (version.equals(Version.NEW)) { if (graphType.equals("heuristics")) { return ok(jobHistoryPage.render(jobDefPair.getId(), graphType, @@ -806,6 +810,10 @@ private static Result getJobHistory(Version version) { return ok(jobHistoryPage.render(jobDefPair.getId(), graphType, jobMetricsHistoryResults.render(jobDefPair, graphType, executionMap, maxStages, flowExecTimeList))); } + else if (graphType.equals("auto-tuning")) { + return ok(jobHistoryPage.render(jobDefPair.getId(), graphType, + jobAnalysisPage.render(jobDefPair, graphType))); + } } else { if (graphType.equals("heuristics")) { return ok(oldJobHistoryPage.render(jobDefPair.getId(), graphType, @@ -1320,6 +1328,7 @@ public static Result restFlowGraphData(String flowDefId) { } // Fetch available flow executions with latest JOB_HISTORY_LIMIT mr jobs. + //direct database query for appllications with given flowadefId List results = getRestFlowAppResults(flowDefId); if (results.size() == 0) { @@ -1801,6 +1810,7 @@ public static Result getTuningParameter(String jobId) { .order() .desc(TuningJobDefinition.TABLE.createdTs) .findUnique(); + logger.info("Respective JobSuggestedParamSet: " + jobSuggestedParamSet.id); if (userSuggestedParamSet == null) { logger.info("No user suggested param set for jobDefinitionId: " + jobDefinitionId); @@ -1814,6 +1824,7 @@ public static Result getTuningParameter(String jobId) { JsonArray tuningParameters = getTuningParameterDetails(parametersList, jobSuggestedParamSet, userSuggestedParamSet); String currentTuningAlgorithm = getCurrentTuningAlgorithmName(tuningAlgorithm); JsonArray tuningAlgorithmList = new JsonArray(); + tuneIn.addProperty(ID, jobId); tuneIn.addProperty(JOB_DEFINTITION_ID, jobDefinitionId); //Two tuning types {HBT, OBT} @@ -2493,4 +2504,307 @@ private static TuningParameter getTuningParameter(int paramId) { return tuningParameter; } -} + /* + * Rest api implementation for autotuning dashboard + * @results: + * [ + * { + * resourceusage: ... + * inputSize: ... + * createdTs: ... + * executionTime: ... + * autoTuningEnabled: ... + * jobExecutionId: ... + * resourceused: ... + * suggestedParameters: [ + * {parameterId: ..., parameterValue: ..., parameterName: ...} + * {...} + * ... + * ] + * } + * .. + * .. + * { + * Autotuning: ... + * } + * { + * BestParamId: ..., + * createdTs: ... + * } + * { + * createdTs: ... , + * autoTuningEnabled:... + * } + * ] + */ + public static Result restAutotuningData(String jobId , String startDate ,String endDate) { + JsonArray datasets = new JsonArray(); + if (jobId == null || jobId.isEmpty()) { + return ok(new Gson().toJson(datasets)); + } + JobDefinition jobDef = getJobDefIdFromJobId(jobId); + Integer jobDefId = jobDef.id; + + List results = new ArrayList(); + if(startDate.equals("Begin") || endDate.equals("End")) { + results = getAutotuningGraphData(jobDefId); + } + else{ + results = getAutotuningGraphDataWithDates(jobDefId , startDate , endDate); + } + if (results.size() == 0) { + logger.info("No results for Job url"); + } + + JobSuggestedParamSet result2 = getTimeStamp( jobDefId); + JobSuggestedParamSet bestParameterId = getBestParameterId( jobDefId); + + //get details of all tuning paramerters + Set paramId = new HashSet(); + Map> searchId = new HashMap>(); + + JsonObject tuningEnabled = new JsonObject(); + int flag=0; + for (JobExecution result : results) { + JsonObject dataset = new JsonObject(); + dataset.addProperty("resourceused", Math.round(result.resourceUsage*100.000)/100.000); + dataset.addProperty("inputSize", Math.round((result.inputSizeInBytes/(1024*1024*1024))*100.00)/100.00); + dataset.addProperty("executionTime",Math.round(result.executionTime*100.000)/100.000 ); + dataset.addProperty("jobExecutionId", result.id); + + String timeStamp = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(result.createdTs); + dataset.addProperty("createdTs", timeStamp); + + List SuggestedParamValue = getJobParameters(result.id); + + JsonArray parameters = new JsonArray(); + + for(JobSuggestedParamValue suggestedValue : SuggestedParamValue) { + paramId.add(suggestedValue.tuningParameter.id); + TuningParameter paramDetails = getTuningParameter(suggestedValue.tuningParameter.id); + searchId.put(paramDetails.id , new Pair(paramDetails.paramName, paramDetails.defaultValue)); + JsonObject suggestedParam = new JsonObject(); + suggestedParam.addProperty("parameterId",suggestedValue.tuningParameter.id); + suggestedParam.addProperty("parameterValue",Math.round(suggestedValue.paramValue*100.00)/100.00 ); + suggestedParam.addProperty("parameterName",paramDetails.paramName); + parameters.add(suggestedParam); + } + + dataset.add("suggestedParameters",parameters ); + datasets.add(dataset); + if(result.createdTs.after(result2.createdTs)) + { + if(flag==0){ + tuningEnabled.addProperty("createdTs", timeStamp); + tuningEnabled.addProperty("autoTuningEnabled", 1); + } + flag = 1; + } + dataset.addProperty("autoTuningEnabled", flag); + } + + for(int i =0 ; i < datasets.size() ; i++){ + JsonArray suggestedValues = datasets.get(i).getAsJsonObject().get("suggestedParameters").getAsJsonArray(); + for(Integer key : searchId.keySet()) { + int flag2 =0 ; + for(int j=0 ; j < suggestedValues.size() ; j++){ + if ( suggestedValues.get(j).getAsJsonObject().get("parameterId").getAsInt() == key){ + flag2=1; + break; + } + } + if(flag2 == 0){ + JsonObject defaultParam = new JsonObject(); + defaultParam.addProperty("parameterId",key); + defaultParam.addProperty("parameterValue",searchId.get(key).getValue()); + defaultParam.addProperty("parameterName",searchId.get(key).getKey()); + datasets.get(i).getAsJsonObject().get("suggestedParameters").getAsJsonArray().add(defaultParam); + } + } + } + + //add if autotuning was enabled or not + TuningJobDefinition tuningJobDefinition = TuningJobDefinition.find + .select("*") + .where() + .eq(TuningJobDefinition.TABLE.job + '.' + JobDefinition.TABLE.id, jobDefId) + .order() + .desc(TuningJobDefinition.TABLE.createdTs) + .findUnique(); + + JsonObject autoTune = new JsonObject(); + autoTune.addProperty("Autotuning",tuningJobDefinition.tuningEnabled ); + datasets.add(autoTune); + + if(bestParameterId != null) { + TuningJobExecutionParamSet bestParamTime = getBestParamTime(bestParameterId.id); //merge these two functions + //add timestamp when best parameter was set + if(bestParamTime.createdTs.before(results.get(0).createdTs) || bestParamTime.createdTs.after(results.get(results.size()-1).createdTs)){ + JsonObject bestParam = new JsonObject(); + bestParam.addProperty("NoBestParamId", "0"); + bestParam.addProperty("createdTs", ""); + datasets.add(bestParam); + } + else { + String timeStamp2 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(bestParamTime.createdTs); + JsonObject bestParam = new JsonObject(); + bestParam.addProperty("BestParamId", bestParameterId.id); + bestParam.addProperty("createdTs", timeStamp2); + datasets.add(bestParam); + } + } + else{ + JsonObject bestParam = new JsonObject(); + bestParam.addProperty("NoBestParamId", "0"); + bestParam.addProperty("createdTs", ""); + datasets.add(bestParam); + } + //add timestamp when Autotuning was enabled + if(flag==1){ + datasets.add(tuningEnabled); + } + else{ + JsonObject autotuning = new JsonObject(); + autotuning.addProperty("TuningDisabled", "0"); + datasets.add(autotuning); + } + return ok(new Gson().toJson(datasets)); + } + + private static TuningJobExecutionParamSet getBestParamTime(Long paramId) { + TuningJobExecutionParamSet jobSuggestedParamSetId = TuningJobExecutionParamSet.find.select("*") + .where() + .eq(TuningJobExecutionParamSet.TABLE.jobSuggestedParamSet + '.' + JobSuggestedParamSet.TABLE.id, paramId) + .setMaxRows(1) + .findUnique(); + return jobSuggestedParamSetId; + } + + /* + * Find tuned parameters + * */ + private static List getJobParameters(Long jobExecId){ + TuningJobExecutionParamSet jobSuggestedParamSetId = TuningJobExecutionParamSet.find + .select("*") + .where() + .eq(TuningJobExecutionParamSet.TABLE.jobExecution + '.' + JobExecution.TABLE.id, jobExecId) + .setMaxRows(1) + .findUnique(); + + List jobSuggestedParameters = JobSuggestedParamValue.find + .select("*") + .where() + .eq(JobSuggestedParamValue.TABLE.jobSuggestedParamSet + '.' + JobSuggestedParamSet.TABLE.id , jobSuggestedParamSetId.jobSuggestedParamSet.id) + .findList() + ; + + return jobSuggestedParameters; + } + +/* +* get job definition id from url +*/ + private static JobDefinition getJobDefIdFromJobId(String jobId) { + + JobDefinition jobDefinition = JobDefinition.find.select("*") + .where() + .eq(JobDefinition.TABLE.jobDefId, jobId) + .findUnique(); + + return jobDefinition; + } + + /* + * Get data for given job definition id + */ + private static List getAutotuningGraphData(Integer jobDefId) { + List results = JobExecution.find.select("*") + .where() + .eq(JobExecution.TABLE.job + "." + JobDefinition.TABLE.id , jobDefId ) + .eq(JobExecution.TABLE.executionState , "SUCCEEDED" ) + .order() + .desc(JobExecution.TABLE.createdTs) + .setMaxRows(30) + .findList(); + + return Lists.reverse(results); + } + +/* + get executions for given date range + */ + private static List getAutotuningGraphDataWithDates(Integer jobDefId , String startDate , String endDate) { + // Fetch available flow executions with latest JOB_HISTORY_LIMIT mr jobs. + List results = JobExecution.find.select("*") + .where() + . between("date(" + JobExecution.TABLE.createdTs + ")", startDate,endDate ) + .eq(JobExecution.TABLE.job + "." + JobDefinition.TABLE.id , jobDefId ) + .eq(JobExecution.TABLE.executionState , "SUCCEEDED" ) + .order() + .asc(JobExecution.TABLE.createdTs) + .findList(); + + return results; + } +/* +* Find timestamp when autotuning was enabled +*/ + private static JobSuggestedParamSet getTimeStamp(Integer jobDefId) { + JobSuggestedParamSet jobSuggestedParamSet = JobSuggestedParamSet.find.select("*") + .where() + .eq(JobSuggestedParamSet.TABLE.jobDefinition + "." + JobDefinition.TABLE.id, jobDefId) + .eq(JobSuggestedParamSet.TABLE.isParamSetSuggested, true) + .eq(JobSuggestedParamSet.TABLE.paramSetState, "FITNESS_COMPUTED") + .order() + .asc(JobSuggestedParamSet.TABLE.createdTs) + .setMaxRows(1) + .findUnique(); + + return jobSuggestedParamSet; + } + + /* + * Find Timestamp for best parameter values + */ + private static JobSuggestedParamSet getBestParameterId(Integer jobDefId) { + JobSuggestedParamSet jobSuggestedParamSet = JobSuggestedParamSet.find.select("*") + .where() + .eq(JobSuggestedParamSet.TABLE.jobDefinition + "." + JobDefinition.TABLE.id, jobDefId) + .eq(JobSuggestedParamSet.TABLE.isParamSetBest, true) + .eq(JobSuggestedParamSet.TABLE.isParamSetSuggested, true) + .order() + .asc(JobSuggestedParamSet.TABLE.createdTs) + .setMaxRows(1) + .findUnique(); + + return jobSuggestedParamSet; + } + + /* + Get date of first execution and last execution of a given job + */ + public static Result restgetDateRange(String jobId){ + //get list of all dates when a particular job is executed + JobDefinition jobDefinitionId = getJobDefIdFromJobId(jobId); + + List datesOfExecution = JobExecution.find + .select( "date("+JobExecution.TABLE.createdTs+")" ) + .where() + .eq(JobExecution.TABLE.job + "." + JobDefinition.TABLE.id , jobDefinitionId.id ) + .eq(JobExecution.TABLE.executionState , "SUCCEEDED" ) + .findList(); + + HashSet distinctDates = new HashSet(); + JsonArray dates =new JsonArray(); + for(JobExecution date : datesOfExecution){ + String timeStamp = new SimpleDateFormat("yyyy-MM-dd").format(date.createdTs); + if(!distinctDates.contains(timeStamp)) { + distinctDates.add(timeStamp); + JsonObject result = new JsonObject(); + result.addProperty("date", timeStamp); + dates.add(result); + } + } + return ok(new Gson().toJson(dates)); + } +} \ No newline at end of file diff --git a/app/views/page/jobHistoryPage.scala.html b/app/views/page/jobHistoryPage.scala.html index 84829a59e..c7aa0074e 100644 --- a/app/views/page/jobHistoryPage.scala.html +++ b/app/views/page/jobHistoryPage.scala.html @@ -50,7 +50,6 @@

Job History

} else { } - @@ -59,5 +58,33 @@

Job History

+ + + @if(graphType == "auto-tuning") { +
+
+ + +
+
+ + + + + + + +
+ + + + + +
+
+ } @results -} \ No newline at end of file +} + diff --git a/app/views/results/jobAnalysisPage.scala.html b/app/views/results/jobAnalysisPage.scala.html new file mode 100644 index 000000000..1e1007f46 --- /dev/null +++ b/app/views/results/jobAnalysisPage.scala.html @@ -0,0 +1,57 @@ +@* +* Copyright 2016 LinkedIn Corp. +* +* Licensed under the Apache License, Version 2.0 (the "License"); you may not +* use this file except in compliance with the License. You may obtain a copy of +* the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +* License for the specific language governing permissions and limitations under +* the License. +*@ +@(jobDefPair: IdUrlPair, graphType : String ) +@import com.linkedin.drelephant.analysis.Severity +@import com.linkedin.drelephant.util.Utils; + +@getSeverityColor(severity: Severity) = @{ + var color: String = "#5cb85c"; // LOW or NONE + + if(severity.getText.equalsIgnoreCase("CRITICAL")) { + color = "#d9534f" + } else if(severity.getText.equalsIgnoreCase("SEVERE")) { + color = "#e4804e" + } else if(severity.getText.equalsIgnoreCase("MODERATE")) { + color = "#f0ad4e" + } + + color +} + +
+

Auto-tuning Dashboard

+
+ + + +
+ +
+ Loading... + +
+

+
+
+
+

Plots for parameter values

+
+
+
+
+
+ + diff --git a/app/views/results/jobMetricsHistoryResults.scala.html b/app/views/results/jobMetricsHistoryResults.scala.html index 7ca3cff78..823d3511a 100644 --- a/app/views/results/jobMetricsHistoryResults.scala.html +++ b/app/views/results/jobMetricsHistoryResults.scala.html @@ -14,61 +14,61 @@ * the License. *@ @(jobDefPair: IdUrlPair, graphType: String, results: java.util.Map[IdUrlPair, java.util.List[models.AppResult]], maxStages: Int, - flowExecTimeList: java.util.List[Long]) +flowExecTimeList: java.util.List[Long]) - @import com.linkedin.drelephant.analysis.Severity - @import com.linkedin.drelephant.util.Utils; +@import com.linkedin.drelephant.analysis.Severity +@import com.linkedin.drelephant.util.Utils; - @getSeverityColor(severity: Severity) = @{ - var color: String = "#5cb85c"; // LOW or NONE +@getSeverityColor(severity: Severity) = @{ + var color: String = "#5cb85c"; // LOW or NONE - if(severity.getText.equalsIgnoreCase("CRITICAL")) { - color = "#d9534f" - } else if(severity.getText.equalsIgnoreCase("SEVERE")) { - color = "#e4804e" - } else if(severity.getText.equalsIgnoreCase("MODERATE")) { - color = "#f0ad4e" - } - - color + if(severity.getText.equalsIgnoreCase("CRITICAL")) { + color = "#d9534f" + } else if(severity.getText.equalsIgnoreCase("SEVERE")) { + color = "#e4804e" + } else if(severity.getText.equalsIgnoreCase("MODERATE")) { + color = "#f0ad4e" } - @if(results != null && results.nonEmpty) { -
-

Job History

-
- @if(graphType.equals("resources")) { - - - - } else { - - - - } + color +} -
+@if(results != null && results.nonEmpty) { +
+

Job History

+
+ @if(graphType.equals("resources")) { + + + + } else { + + + + } + +
-
- Loading... - -
+
+ Loading... + +
-
+
-
- +
+
- - - + + + - @for(i <- 1 to maxStages) { - - } - - + } + + - - @for((flowExecPair, jobs) <- results) { - + + @for((flowExecPair, jobs) <- results) { + - - + + - - @for(i <- 1 to maxStages) { - + + @for(i <- 1 to maxStages) { + - } - -
Job Executions
Job ExecutionsStage @i - - - +
+ @for(i <- 1 to maxStages) { + Stage @i + + + - -
@@ -98,50 +98,50 @@

Job History

+
- Loading... - + Loading... + - @if(i <= jobs.length) { -
- - - - - - - - - -
@(Utils.getResourceInGBHours(jobs(i - 1).resourceUsed).split("GB")(0)) - @(Utils.getResourceInGBHours(jobs(i - 1).resourceWasted).split("GB")(0)) - @(Utils.getDurationBreakdown(jobs(i - 1).finishTime - jobs(i - 1).startTime).split("Hours")(0)) - @(Utils.getDurationBreakdown(jobs(i - 1).totalDelay).split("Hours")(0)) -
-
- } - 		+ @if(i <= jobs.length) { +
+ + + + + + + + + +
@(Utils.getResourceInGBHours(jobs(i - 1).resourceUsed).split("GB")(0)) + @(Utils.getResourceInGBHours(jobs(i - 1).resourceWasted).split("GB")(0)) + @(Utils.getDurationBreakdown(jobs(i - 1).finishTime - jobs(i - 1).startTime).split("Hours")(0)) + @(Utils.getDurationBreakdown(jobs(i - 1).totalDelay).split("Hours")(0)) +
+
} -
-
+ + } + + } + +
- } \ No newline at end of file +
+} \ No newline at end of file diff --git a/compile.sh b/compile.sh index 039b9bf68..1c868b3da 100755 --- a/compile.sh +++ b/compile.sh @@ -31,12 +31,12 @@ function play_command() { function require_programs() { echo "Checking for required programs..." missing_programs="" - + for program in $@; do if ! command -v "$program" > /dev/null; then missing_programs=$(printf "%s\n\t- %s" "$missing_programs" "$program") fi - done + done if [ ! -z "$missing_programs" ]; then echo "[ERROR] The following programs are required and are missing: $missing_programs" @@ -149,7 +149,7 @@ tuning_performance_report_script=${project_root}/scripts/tuning_performance_repo rm -rf ${project_root}/dist mkdir dist -play_command $OPTS clean test compile dist +play_command $OPTS clean dist cd target/universal diff --git a/conf/routes b/conf/routes index 0c517f737..3e037ade2 100644 --- a/conf/routes +++ b/conf/routes @@ -33,6 +33,8 @@ GET /newjobhistory controllers.Application.jobHisto GET /newhelp controllers.Application.help() # Rest calls +GET /rest/newgraphdata controllers.Application.restAutotuningData(id: String , startDate: String, endDate : String) +GET /rest/getDateRange controllers.Application.restgetDateRange(id: String) GET /rest/job controllers.Application.restAppResult(id: String) GET /rest/jobexec controllers.Application.restJobExecResult(id: String) GET /rest/flowexec controllers.Application.restFlowExecResult(id: String) diff --git a/public/css/main.css b/public/css/main.css index e2a8618bf..ca832f203 100644 --- a/public/css/main.css +++ b/public/css/main.css @@ -115,6 +115,48 @@ a.list-group-item-danger.active:focus { display:none; /* Hide the y axis */ } +div.tooltip { + position: absolute; + text-align: center; + width: 40px; + height: 30px; + padding: 5px; + font: 12px; + border: 1.5px solid black; + background: white; + border-radius: 5px; + pointer-events: none; +} + +.new { + overflow-x: auto; +} + +.new thead tr th { + text-align:center; + word-wrap:break-word; + height: 35px; + font-size: 18px; + background: #f9f9f9; +} +.new tbody tr td { + text-align:center; + color: #0077B5; + height: 35px; +} + +.new thead tr th, +.new tbody tr td { + overflow: hidden; + width: 200px; + padding-right:1px; + border: 1px solid #ddd; +} +.new thead tr th a{ + text-decoration:none; + +} + /* Override bootstrap. Horizontal scrollable history table */ body { font-family: "HelveticaNeue-Light", "Helvetica Neue Light", "Helvetica Neue", Arial Narrow, Helvetica, sans-serif; @@ -122,21 +164,21 @@ body { .table-responsive { overflow-x: auto; } -.table-responsive > .table > thead > tr > th { +.table-responsive .table thead tr th { text-align:center; vertical-align:top; word-wrap:break-word; } -.table-responsive > .table > tbody > tr > td { +.table-responsive .table tbody tr td { text-align:left; } -.table-responsive > .table > thead > tr > th, -.table-responsive > .table > tbody > tr > td { +.table-responsive .table thead tr th, +.table-responsive .table tbody tr td { overflow: hidden; width: 120px; padding-right:0px; } -.table-responsive > .table > thead > tr > th > a{ +.table-responsive .table thead tr th a{ text-decoration:none; } diff --git a/public/js/autotuningGraph.js b/public/js/autotuningGraph.js new file mode 100644 index 000000000..95468d4df --- /dev/null +++ b/public/js/autotuningGraph.js @@ -0,0 +1,27 @@ +/* + * Copyright 2016 LinkedIn Corp. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +$(document).ready(function(){ + var startDate = queryString()['startDate']; + var endDate = queryString()['endDate']; + + startDate= startDate || 'Begin' ; + endDate= endDate || 'End' ; + $.getJSON('/rest/newgraphdata?id=' + queryString()['job-def-id'] +'&startDate='+ startDate+'&endDate='+endDate , function(data) { + plotter(data); + }); +}); + diff --git a/public/js/autotuningGraphutility.js b/public/js/autotuningGraphutility.js new file mode 100644 index 000000000..f8212cd61 --- /dev/null +++ b/public/js/autotuningGraphutility.js @@ -0,0 +1,503 @@ +/* + * Copyright 2016 LinkedIn Corp. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +/* Show loading sign during ajax call */ +$(document).ajaxStart(function() { + $("#loading-indicator").show(); +}); + +$(document).ajaxStop(function() { + $("#loading-indicator").hide(); +}); + +/* Plot the performance graph for the data */ +function plotter(data ) { + + + var lastEle = data[data.length-1]; + data.pop(); + var lastEle2 = data[data.length-1]; + data.pop(); + var lastEle3 = data[data.length-1]; + data.pop(); + + var graphContainer = d3.select("#visualisation"); + var MARGINS = {top: 50, right: 50, bottom: 100, left: 50}, + WIDTH = graphContainer.style("width").replace("px", ""), + HEIGHT = graphContainer.style("height").replace("px", ""), + GRAPH_WIDTH = WIDTH - MARGINS.left - MARGINS.right, + GRAPH_HEIGHT = HEIGHT - MARGINS.top - MARGINS.bottom; + var parseDate = d3.time.format("%Y-%m-%d %H:%M:%S").parse; + +////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + + + var table1 = d3.select('#table').append('table'); + + var thead = table1.append('thead'); + var tbody = table1.append('tbody'); + + var columns = ['createdTs','resourceused', 'inputSize', 'executionTime'], column_id = 'code', column_class = 'norm'; + + thead.append('tr') + .selectAll('th') + .data(['Job Executions','Resources Used (GB Hours)', 'Input Size (GB)', 'Execution Time (sec)']) + .enter() + .append('th') + .text(function(column) { + return column; + }); + + var rows = tbody.selectAll('tr') + .data(data) + .enter() + .append('tr'); + + + + var cells = rows.selectAll('td') + .data(function(row) { + return columns.map(function(column) { + return { + column: column, + value: row[column], + id: row[column_id], + class: row[column_class] + }; + }); + }) + .enter() + .append('td') + .html(function(d) { + return d.value; + }); + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + + for( i=0 ;i < data.length ; i++){ + data[i].suggestedParameters.sort(function(a,b){ + return (a.parameterId > b.parameterId) ? 1 : -1; + } + ); + }; + + data.forEach(function(d) { + d.createdTs = parseDate(d.createdTs);}); + + var xScale = d3.time.scale() + .range([2*MARGINS.left, GRAPH_WIDTH-MARGINS.right]) + .domain(d3.extent(data, function(d) { return d.createdTs; })), + + yScale = d3.scale.linear() + .range([MARGINS.bottom + GRAPH_HEIGHT-10, 2*MARGINS.top+20]) + .domain([0,Math.max( + d3.max(data, function (d) { return Math.max(d.inputSize) }), + d3.max(data, function (d) { return Math.max(d.resourceused) }) + )]); + var yScaleRight = d3.scale.linear() + .range([MARGINS.bottom + GRAPH_HEIGHT-10, 2*MARGINS.top+20]) + .domain([ + d3.min(data, function (d) { return Math.min(d.executionTime) }), + d3.max(data, function (d) { return Math.max(d.executionTime) })]); + + var customTimeFormat = d3.time.format("%b-%d %I:%M:%S"); + + + var xAxis = d3.svg.axis() + .scale(xScale) + .orient("bottom") + .ticks(5) + .tickFormat(customTimeFormat) + .tickSize(1), + + yAxis = d3.svg.axis() + .scale(yScale) + .orient("left") + .ticks(6) + ; + + var yAxisRight = d3.svg.axis() + .scale(yScaleRight) + .orient("right") + .ticks(6); + + + graphContainer.append("svg:g") + .attr("class", "x axis") + .attr("transform", "translate(0 ," + (MARGINS.bottom + GRAPH_HEIGHT-10)+ ")") + .call(xAxis) + .selectAll("text") + .style("text-anchor","end") + .attr("dx", "-.8em") + .attr("dy", ".15em") + .attr("transform","rotate(-20)"); + + graphContainer.append("svg:g") + .attr("class", "y axis") + .attr("transform", "translate(" + (MARGINS.left) + ", 0)") + .call(yAxis) + .selectAll("text") + .attr("fill", "rgb(0, 119, 181)"); + + graphContainer.append("svg:g") + .attr("class", "y axis") + .attr("transform", "translate(" + (GRAPH_WIDTH) + ", 0)") + .call(yAxisRight) + .attr("id", "ExecTimeAxis") + .selectAll("text") + .attr("fill", "rgb(0, 119, 181)"); + + + //specify meaning of graphs + + graphContainer.append("svg:text") + .style("font-size", "16px") + .style("fill", "#006060") + .attr("transform", "translate(" + (MARGINS.left/10) + ", " + (MARGINS.top + 22) + ")") + .text("ResourcesUsed(GB Hours)"); + + + graphContainer.append("svg:text") + .attr("id","ExecText") + .style("font-size", "16px") + .style("fill", "#006060") + .attr("transform", "translate(" + (GRAPH_WIDTH - MARGINS.left-35) + ", " +(MARGINS.top + 22) + ")") + .text("Execution Time(sec)"); + + + // Add the small rectangles to specify the graph meaning + graphContainer.append("rect") + .attr("x", GRAPH_WIDTH - 18) + .attr("width", 14) + .attr("height", 14) + .style("fill", 'blue' ); + + graphContainer.append("text") + .attr("x", GRAPH_WIDTH - 26) + .attr("y", 9) + .attr("dy", ".30em") + .style("text-anchor", "end") + .text(function(d) { return "Resource Usage" }); + + graphContainer.append("rect") + .attr("x", GRAPH_WIDTH - 18) + .attr("y", 20) + .attr("width", 14) + .attr("height", 14) + .style("fill", '#FF0000' ) + .on("click", function(){ + var active = InputSizePath.active ? false : true, + newOpacity = active ? 0 : 1; + d3.select("#InputSizePath").style("opacity", newOpacity); + d3.select("#InputSizeDots").style("opacity", newOpacity); + InputSizePath.active = active; + }); + + graphContainer.append("text") + .attr("x", GRAPH_WIDTH - 26) + .attr("y", 29) + .attr("dy", ".30em") + .style("text-anchor", "end") + .text(function(d) { return "Input Size(GB)" }) + .on("click", function(){ + var active = InputSizePath.active ? false : true, + newOpacity = active ? 0 : 1; + d3.select("#InputSizePath").style("opacity", newOpacity); + d3.select("#InputSizeDots").style("opacity", newOpacity); + InputSizePath.active = active; + }); + + graphContainer.append("rect") + .attr("x", GRAPH_WIDTH - 18) + .attr("y", 40) + .attr("width", 14) + .attr("height", 14) + .style("fill", 'green' ) + .on("click", function(){ + var active = ExecTime.active ? false : true, + newOpacity = active ? 0 : 1; + d3.select("#ExecTime").style("opacity", newOpacity); + d3.select("#ExecDots").style("opacity", newOpacity); + d3.select("#ExecTimeAxis").style("opacity", newOpacity); + d3.select("#ExecText").style("opacity", newOpacity); + ExecTime.active = active; + }); + + graphContainer.append("text") + .attr("x", GRAPH_WIDTH - 30) + .attr("y", 50) + .attr("dy", ".30em") + .style("text-anchor", "end") + .on("click", function(){ + // Determine if current line is visible + var active = ExecTime.active ? false : true, + newOpacity = active ? 0 : 1; + // Hide or show the elements + d3.select("#ExecTime").style("opacity", newOpacity); + d3.select("#ExecDots").style("opacity", newOpacity); + d3.select("#ExecTimeAxis").style("opacity", newOpacity); + d3.select("#ExecText").style("opacity", newOpacity); + // Update whether or not the elements are active + ExecTime.active = active; + }) + .text(function(d) { return "Execution Time" }); + + + var lineGen = d3.svg.line() + .x(function(d) { return xScale(d.createdTs); }) + .y(function(d) { + return yScale(d.resourceused); + }) + .interpolate('linear'); + + var lineGenInputSize = d3.svg.line() + .x(function(d){ return xScale(d.createdTs); }) + .y(function(d) { + return yScale(d.inputSize); + }) + .interpolate('linear'); + + var lineGenExecTime = d3.svg.line() + .x(function(d){ return xScale(d.createdTs); }) + .y(function(d) { + return yScaleRight(d.executionTime); + }) + .interpolate('linear'); + + + + var div = d3.select("body").append("div") + .attr("class", "tooltip") + .style("opacity", 0); + + + graphContainer.append("svg:g") + .selectAll("scatter-dots") + .data(data) + .enter().append("svg:circle") + .style({stroke: 'white', fill: 'blue'}) + .attr("cx", function (d) { return xScale(d.createdTs); } ) + .attr("cy", function (d) { return yScale(d.resourceused); } ) + .attr("r", 6) + + .on("mouseover", function(d) { + div.transition() + .duration(50) + .style("opacity", .9); + div.html(d.jobExecutionId) + .style('top', (d3.event.pageY + 10)+'px') + .style('left', (d3.event.pageX + 10)+'px'); + ; + }) + .on("mouseout", function(d) { + div.transition() + .duration(500) + .style("opacity", 0); + }); + + graphContainer.append("svg:g") + .attr("id","InputSizeDots") + .selectAll("scatter-dots") + .data(data) + .enter().append("svg:circle") + .style({stroke: 'white', fill: '#FF0000'}) + .attr("cx", function (d) { return xScale(d.createdTs); } ) + .attr("cy", function (d) { return yScale(d.inputSize); } ) + .attr("r", 5); + + + graphContainer.append("svg:g") + .attr("id","ExecDots") + .selectAll("scatter-dots") + .data(data) + .enter().append("svg:circle") + .style({stroke: 'white', fill: 'green'}) + .attr("cx", function (d) { return xScale(d.createdTs); } ) + .attr("cy", function (d) { return yScaleRight(d.executionTime); } ) + .attr("r", 5); + + +//plot linear graphs + + graphContainer.append('svg:path') + .attr('d', lineGen(data)) + .attr('stroke', 'blue') + .attr('stroke-width', 3) + .attr('fill', 'none'); + + + graphContainer.append('svg:path') + .attr('d', lineGenInputSize(data)) + .attr("id", "InputSizePath") + .attr('stroke', '#FF0000') + .attr('stroke-width', 2) + .attr('fill', 'none'); + + graphContainer.append('svg:path') + .attr('d', lineGenExecTime(data)) + .attr("id", "ExecTime") + .attr('stroke', 'green') + .attr('stroke-width', 1) + .attr('fill', 'none'); + + + //Mark the date when autotuning is enabled for the first time + + graphContainer.append('svg:line') + .style('stroke', 'black') + .attr('stroke-width', 1) + .attr('x1', xScale(parseDate(lastEle.createdTs))) + .attr('y1', 94) + .attr('x2', xScale(parseDate(lastEle.createdTs))) + .attr('y2', MARGINS.bottom + GRAPH_HEIGHT-10); + + graphContainer.append("text") + .attr('x', xScale(parseDate(lastEle.createdTs))) + .attr('y', 89) + .attr("text-anchor", "middle") + .style("font-size", "15px") + .text("Autotuning enabled"); + + graphContainer.append("text") + .attr('x', xScale(parseDate(lastEle2.createdTs))) + .attr('y', 109) + .attr("text-anchor", "middle") + .style("font-size", "15px") + .text("Best Parameter Set"); + + + + graphContainer.append('svg:line') + .style('stroke', 'black') + .attr('stroke-width', 1) + .attr('x1', xScale(parseDate(lastEle2.createdTs))) + .attr('y1', 111) + .attr('x2', xScale(parseDate(lastEle2.createdTs))) + .attr('y2', MARGINS.bottom + GRAPH_HEIGHT-10); + + graphContainer.append("text") + .attr('x',2*MARGINS.left ) + .attr('y', 12) + .attr("text-anchor", "middle") + .style("font-size", "16px") + .text(function(d){ + if(lastEle3.Autotuning== true) + return ""; + else + return "Autotuning Disabled"; + }); + +///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +//Plot for parameters // +///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + + for (var i =0 ;i Job execution + Autotuning Dashboard {{/if}} diff --git a/web/app/templates/components/tunein-info.hbs b/web/app/templates/components/tunein-info.hbs index fcb324355..f2df70186 100644 --- a/web/app/templates/components/tunein-info.hbs +++ b/web/app/templates/components/tunein-info.hbs @@ -75,6 +75,14 @@ {{#if (or (eq job.severity "None") (eq job.severity "Low"))}}
All Heuristics Passed with the above mentioned parameters
{{/if}} + + + +
+

working

+ {{#if (eq job.jobtype "HadoopJava" ) }} +

We are not considering these heuristics:

+ {{/if}} {{#unless (eq tunein.reasonForTuningDisable "NONE")}}
Disabling TuneIn for this application as {{tunein.reasonForTuningDisable}}
{{/unless}} @@ -83,6 +91,7 @@
{{/if}} -
For any queries, contact ask_tunein
+ +
For any queriess ask_tunein
{{/unless}} diff --git a/web/app/templates/job.hbs b/web/app/templates/job.hbs index f9aa6b683..d0de12779 100644 --- a/web/app/templates/job.hbs +++ b/web/app/templates/job.hbs @@ -75,6 +75,9 @@ Job execution + Autotuning Dashboard diff --git a/web/dr-elephant b/web/dr-elephant new file mode 160000 index 000000000..a045b64d6 --- /dev/null +++ b/web/dr-elephant @@ -0,0 +1 @@ +Subproject commit a045b64d60facd28e58a4a856f24fb78d4817e7d diff --git a/web/package.json b/web/package.json index 1e2eeabf3..83db2a6f9 100644 --- a/web/package.json +++ b/web/package.json @@ -34,7 +34,7 @@ "ember-cli-jshint": "1.0.5", "ember-cli-moment-shim": "2.2.0", "ember-cli-notifications": "4.0.9", - "ember-cli-qunit": "1.4.2", + "ember-cli-qunit": "1.4.3", "ember-cli-release": "0.2.9", "ember-cli-sri": "2.1.1", "ember-cli-uglify": "1.2.0",