aro-max
diff --git a/‎.idea/workspace.xml
+9-101 b/‎.idea/workspace.xml
+9-101
diff --git a/‎project/target/streams/$global/dependencyPositions/$global/streams/update_cache_2.10/output_dsp
0 Bytes b/‎project/target/streams/$global/dependencyPositions/$global/streams/update_cache_2.10/output_dsp
0 Bytes
diff --git a/‎src/main/scala/com/.DS_Store
0 Bytes b/‎src/main/scala/com/.DS_Store
0 Bytes
diff --git a/‎src/main/scala/com/sparkProject/Trainer.scala
+12-11 b/‎src/main/scala/com/sparkProject/Trainer.scala
+12-11
diff --git a/‎target/scala-2.11/TP_ParisTech_2017_2018-assembly-1.0.jar
12 Bytes b/‎target/scala-2.11/TP_ParisTech_2017_2018-assembly-1.0.jar
12 Bytes
diff --git a/‎target/scala-2.11/classes/com/sparkProject/Trainer$$anonfun$1.class
0 Bytes b/‎target/scala-2.11/classes/com/sparkProject/Trainer$$anonfun$1.class
0 Bytes
diff --git a/‎target/scala-2.11/classes/com/sparkProject/Trainer$.class
19 Bytes b/‎target/scala-2.11/classes/com/sparkProject/Trainer$.class
19 Bytes
diff --git a/‎target/streams/$global/assembly/$global/streams/assembly-inputs
0 Bytes b/‎target/streams/$global/assembly/$global/streams/assembly-inputs
0 Bytes
diff --git a/‎target/streams/$global/assembly/$global/streams/out
+1-1 b/‎target/streams/$global/assembly/$global/streams/out
+1-1
diff --git a/‎target/streams/$global/assemblyOption/$global/streams/assembly/e57bd0e1612aa259630e2845ad9f5e63342fdd33_dir/com/sparkProject/Trainer$$anonfun$1.class
0 Bytes b/‎target/streams/$global/assemblyOption/$global/streams/assembly/e57bd0e1612aa259630e2845ad9f5e63342fdd33_dir/com/sparkProject/Trainer$$anonfun$1.class
0 Bytes
diff --git a/‎target/streams/$global/assemblyOption/$global/streams/assembly/e57bd0e1612aa259630e2845ad9f5e63342fdd33_dir/com/sparkProject/Trainer$.class
19 Bytes b/‎target/streams/$global/assemblyOption/$global/streams/assembly/e57bd0e1612aa259630e2845ad9f5e63342fdd33_dir/com/sparkProject/Trainer$.class
19 Bytes
diff --git a/‎target/streams/$global/dependencyPositions/$global/streams/update_cache_2.11/output_dsp
0 Bytes b/‎target/streams/$global/dependencyPositions/$global/streams/update_cache_2.11/output_dsp
0 Bytes
diff --git a/‎target/streams/compile/compileIncremental/$global/streams/out
+2-2 b/‎target/streams/compile/compileIncremental/$global/streams/out
+2-2
diff --git a/‎target/streams/compile/incCompileSetup/$global/streams/inc_compile_2.11
+12-11 b/‎target/streams/compile/incCompileSetup/$global/streams/inc_compile_2.11
+12-11
@@ -31,7 +31,7 @@ object Trainer {
       .appName("TP_spark")
       .getOrCreate()
 
-    // only for self made function
+    // only for self made functions
     // import spark.implicits._
 
     /*******************************************************************************
@@ -45,6 +45,8 @@ object Trainer {
       *
       *       if problems with unimported modules => sbt plugins update
       *
+      *       ALEXANDRE ROUXEL
+      *
       ********************************************************************************/
 
    /** CHARGER LE DATASET **/
@@ -62,7 +64,6 @@ object Trainer {
       .setInputCol("text")
       .setOutputCol("tokens")
 
-    //StopWordsRemover
     // stage 2
     val remover = new StopWordsRemover()
       .setInputCol("tokens")
@@ -86,12 +87,12 @@ object Trainer {
     // stage 5
 
     // stream indexer
-    val indexer = new StringIndexer()
+    val indexerCountry = new StringIndexer()
       .setInputCol("country2")
       .setOutputCol("country_indexed")
 
     // stage 6
-    val indexer2 = new StringIndexer()
+    val indexerCurrency = new StringIndexer()
       .setInputCol("currency2")
       .setOutputCol("currency_indexed")
 
@@ -119,11 +120,11 @@ object Trainer {
     /** PIPELINE **/
 
     val pipeline = new Pipeline()
-      .setStages(Array ( tokenizer , remover ,  countvectorizer , idf , indexer , indexer2 , vecAssembler ,lr))
+      .setStages(Array ( tokenizer , remover ,  countvectorizer , idf , indexerCountry , indexerCurrency , vecAssembler ,lr))
 
     /** TRAINING AND GRID-SEARCH **/
 
-    /** build a training set  **/
+    /** build a training set  (90 % ,  10 %) for (training , testing) **/
     val Array(training, test) = df.randomSplit(Array(0.9, 0.1), seed = 12345)
 
 
@@ -143,17 +144,17 @@ object Trainer {
       .setPredictionCol("predictions")
       .setMetricName("f1")
 
-
+    /** define the cross validation with F1 measure and 70% of the data for training */
     val cv = new TrainValidationSplit()
       .setEstimator(pipeline)
       .setEvaluator(evaluatorF1)
       .setEstimatorParamMaps(paramGrid)
       .setTrainRatio(0.7)
 
-    /** run the cross validator on the training set **/
+    /** fit the best output of the cross validator on the training set **/
     val cvModel = cv.fit(training)
 
-    /** run the cross validator on test and training set **/
+    /** transform  the test set and the training set with the best transformer found by cross validation **/
     val trainPredictions = cvModel.transform(training)
     val testPredictions = cvModel.transform(test)
 
@@ -168,9 +169,9 @@ object Trainer {
 
     df_WithPredictions.groupBy( "final_status" , "predictions" ).count.show()
 
-    println("F1 measurement on training set  ")
+    println("F1 measurement on training set  : ")
     println(f1Train)
-    println("F1 measurement on test set  ")
+    println("F1 measurement on test set   : ")
     println(f1Test)
 
     /** save the trained model **/
 
@@ -1,7 +1,7 @@
 [0m[[0mdebug[0m] [0mCalculate mappings...[0m
 [0m[[0minfo[0m] [0mChecking every *.class/*.jar file's SHA-1.[0m
 [0m[[0minfo[0m] [0mMerging files...[0m
-[0m[[0minfo[0m] [0mSHA-1: ce0b476c860eaf6c622567df87af3ea87eefec4e[0m
+[0m[[0minfo[0m] [0mSHA-1: 4959e7de8eb5805b66f88853e69632f7c461c412[0m
 [0m[[0minfo[0m] [0mPackaging /Users/alexandre/MSBGD/spark/tp/TP_ParisTech_2017_2018_starter/target/scala-2.11/TP_ParisTech_2017_2018-assembly-1.0.jar ...[0m
 [0m[[0mdebug[0m] [0mInput file mappings:[0m
 [0m[[0mdebug[0m] [0m	com/sparkProject/Exercice$.class[0m