tp spark

Main file :
Trainer.scala

Author: aro-max

.idea/vcs.xml

@@ -1 +1 @@
-{"class":"org.apache.spark.ml.PipelineModel","timestamp":1511110080430,"sparkVersion":"2.2.0","uid":"pipeline_81f8322b79a0","paramMap":{"stageUids":["regexTok_d0eba840be33","stopWords_c0fd49b80eba","cntVec_09e92d8ef0ef","idf_c290bcecfc79","strIdx_b9394f3efcdb","strIdx_678029772bab","vecAssembler_f0e4f9110f47","logreg_d39bcdb14fc4"]}}
+{"class":"org.apache.spark.ml.PipelineModel","timestamp":1511217003981,"sparkVersion":"2.2.0","uid":"pipeline_c52b8a07d9e9","paramMap":{"stageUids":["regexTok_dd472a3f490d","stopWords_fcc7bae6b0cb","cntVec_797fec8a4884","idf_aa9807b955e2","strIdx_f9b06d1a9509","strIdx_0ac1cec40b18","vecAssembler_d2eba952c090","logreg_33e8252da4e9"]}}

.idea/workspace.xml

@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.feature.RegexTokenizer","timestamp":1511217004052,"sparkVersion":"2.2.0","uid":"regexTok_dd472a3f490d","paramMap":{"gaps":true,"minTokenLength":1,"pattern":"\\W+","inputCol":"text","toLowercase":true,"outputCol":"tokens"}}

fittedBinaryClassifier/.DS_Store

@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.feature.StopWordsRemover","timestamp":1511217004115,"sparkVersion":"2.2.0","uid":"stopWords_fcc7bae6b0cb","paramMap":{"stopWords":["i","me","my","myself","we","our","ours","ourselves","you","your","yours","yourself","yourselves","he","him","his","himself","she","her","hers","herself","it","its","itself","they","them","their","theirs","themselves","what","which","who","whom","this","that","these","those","am","is","are","was","were","be","been","being","have","has","had","having","do","does","did","doing","a","an","the","and","but","if","or","because","as","until","while","of","at","by","for","with","about","against","between","into","through","during","before","after","above","below","to","from","up","down","in","out","on","off","over","under","again","further","then","once","here","there","when","where","why","how","all","any","both","each","few","more","most","other","some","such","no","nor","not","only","own","same","so","than","too","very","s","t","can","will","just","don","should","now","i'll","you'll","he'll","she'll","we'll","they'll","i'd","you'd","he'd","she'd","we'd","they'd","i'm","you're","he's","she's","it's","we're","they're","i've","we've","you've","they've","isn't","aren't","wasn't","weren't","haven't","hasn't","hadn't","don't","doesn't","didn't","won't","wouldn't","shan't","shouldn't","mustn't","can't","couldn't","cannot","could","here's","how's","let's","ought","that's","there's","what's","when's","where's","who's","why's","would"],"inputCol":"tokens","outputCol":"filtered","caseSensitive":false}}

fittedBinaryClassifier/bestModel/metadata/.part-00000.crc

@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.feature.CountVectorizerModel","timestamp":1511217004180,"sparkVersion":"2.2.0","uid":"cntVec_797fec8a4884","paramMap":{"binary":false,"outputCol":"vectorized","vocabSize":262144,"minDF":20.0,"inputCol":"filtered","minTF":1.0}}

fittedBinaryClassifier/bestModel/metadata/part-00000

@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.feature.IDFModel","timestamp":1511217004511,"sparkVersion":"2.2.0","uid":"idf_aa9807b955e2","paramMap":{"outputCol":"tfidf","minDocFreq":0,"inputCol":"vectorized"}}

fittedBinaryClassifier/bestModel/stages/0_regexTok_d0eba840be33/metadata/.part-00000.crc

@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.feature.StringIndexerModel","timestamp":1511217004736,"sparkVersion":"2.2.0","uid":"strIdx_f9b06d1a9509","paramMap":{"outputCol":"country_indexed","inputCol":"country2","handleInvalid":"error"}}

fittedBinaryClassifier/bestModel/stages/0_regexTok_d0eba840be33/metadata/part-00000

@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.feature.StringIndexerModel","timestamp":1511217004925,"sparkVersion":"2.2.0","uid":"strIdx_0ac1cec40b18","paramMap":{"inputCol":"currency2","outputCol":"currency_indexed","handleInvalid":"error"}}

fittedBinaryClassifier/bestModel/stages/0_regexTok_d0eba840be33/metadata/._SUCCESS.crc fittedBinaryClassifier/bestModel/stages/0_regexTok_dd472a3f490d/metadata/._SUCCESS.crc

@@ -1 +1 @@
-{"class":"org.apache.spark.ml.feature.VectorAssembler","timestamp":1511110082499,"sparkVersion":"2.2.0","uid":"vecAssembler_f0e4f9110f47","paramMap":{"inputCols":["tfidf","days_campaign","hours_prepa","goal","country_indexed","currency_indexed"],"outputCol":"features"}}
+{"class":"org.apache.spark.ml.feature.VectorAssembler","timestamp":1511217005097,"sparkVersion":"2.2.0","uid":"vecAssembler_d2eba952c090","paramMap":{"inputCols":["tfidf","days_campaign","hours_prepa","goal","country_indexed","currency_indexed"],"outputCol":"features"}}

fittedBinaryClassifier/bestModel/stages/0_regexTok_dd472a3f490d/metadata/.part-00000.crc

@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.classification.LogisticRegressionModel","timestamp":1511217005164,"sparkVersion":"2.2.0","uid":"logreg_33e8252da4e9","paramMap":{"probabilityCol":"probability","elasticNetParam":0.0,"predictionCol":"predictions","rawPredictionCol":"raw_predictions","maxIter":300,"aggregationDepth":2,"featuresCol":"features","labelCol":"final_status","threshold":0.5,"tol":1.0E-6,"regParam":3.3546262790251185E-4,"thresholds":[0.7,0.3],"fitIntercept":true,"family":"auto","standardization":true}}

fittedBinaryClassifier/bestModel/stages/0_regexTok_d0eba840be33/metadata/_SUCCESS fittedBinaryClassifier/bestModel/stages/0_regexTok_dd472a3f490d/metadata/_SUCCESS

@@ -1 +1 @@
-{"class":"org.apache.spark.ml.Pipeline","timestamp":1511110078681,"sparkVersion":"2.2.0","uid":"pipeline_81f8322b79a0","paramMap":{"stageUids":["regexTok_d0eba840be33","stopWords_c0fd49b80eba","cntVec_09e92d8ef0ef","idf_c290bcecfc79","strIdx_b9394f3efcdb","strIdx_678029772bab","vecAssembler_f0e4f9110f47","logreg_d39bcdb14fc4"]}}
+{"class":"org.apache.spark.ml.Pipeline","timestamp":1511217003251,"sparkVersion":"2.2.0","uid":"pipeline_c52b8a07d9e9","paramMap":{"stageUids":["regexTok_dd472a3f490d","stopWords_fcc7bae6b0cb","cntVec_797fec8a4884","idf_aa9807b955e2","strIdx_f9b06d1a9509","strIdx_0ac1cec40b18","vecAssembler_d2eba952c090","logreg_33e8252da4e9"]}}

fittedBinaryClassifier/bestModel/stages/0_regexTok_dd472a3f490d/metadata/part-00000

@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.feature.RegexTokenizer","timestamp":1511217003332,"sparkVersion":"2.2.0","uid":"regexTok_dd472a3f490d","paramMap":{"gaps":true,"minTokenLength":1,"pattern":"\\W+","inputCol":"text","toLowercase":true,"outputCol":"tokens"}}

fittedBinaryClassifier/bestModel/stages/1_stopWords_c0fd49b80eba/metadata/.part-00000.crc

@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.feature.StopWordsRemover","timestamp":1511217003417,"sparkVersion":"2.2.0","uid":"stopWords_fcc7bae6b0cb","paramMap":{"stopWords":["i","me","my","myself","we","our","ours","ourselves","you","your","yours","yourself","yourselves","he","him","his","himself","she","her","hers","herself","it","its","itself","they","them","their","theirs","themselves","what","which","who","whom","this","that","these","those","am","is","are","was","were","be","been","being","have","has","had","having","do","does","did","doing","a","an","the","and","but","if","or","because","as","until","while","of","at","by","for","with","about","against","between","into","through","during","before","after","above","below","to","from","up","down","in","out","on","off","over","under","again","further","then","once","here","there","when","where","why","how","all","any","both","each","few","more","most","other","some","such","no","nor","not","only","own","same","so","than","too","very","s","t","can","will","just","don","should","now","i'll","you'll","he'll","she'll","we'll","they'll","i'd","you'd","he'd","she'd","we'd","they'd","i'm","you're","he's","she's","it's","we're","they're","i've","we've","you've","they've","isn't","aren't","wasn't","weren't","haven't","hasn't","hadn't","don't","doesn't","didn't","won't","wouldn't","shan't","shouldn't","mustn't","can't","couldn't","cannot","could","here's","how's","let's","ought","that's","there's","what's","when's","where's","who's","why's","would"],"inputCol":"tokens","outputCol":"filtered","caseSensitive":false}}

fittedBinaryClassifier/bestModel/stages/1_stopWords_c0fd49b80eba/metadata/part-00000

@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.feature.CountVectorizer","timestamp":1511217003507,"sparkVersion":"2.2.0","uid":"cntVec_797fec8a4884","paramMap":{"binary":false,"outputCol":"vectorized","vocabSize":262144,"minDF":2.0,"inputCol":"filtered","minTF":1.0}}

fittedBinaryClassifier/bestModel/stages/1_stopWords_c0fd49b80eba/metadata/._SUCCESS.crc fittedBinaryClassifier/bestModel/stages/1_stopWords_fcc7bae6b0cb/metadata/._SUCCESS.crc

@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.feature.IDF","timestamp":1511217003585,"sparkVersion":"2.2.0","uid":"idf_aa9807b955e2","paramMap":{"outputCol":"tfidf","minDocFreq":0,"inputCol":"vectorized"}}

fittedBinaryClassifier/bestModel/stages/1_stopWords_fcc7bae6b0cb/metadata/.part-00000.crc

@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.feature.StringIndexer","timestamp":1511217003663,"sparkVersion":"2.2.0","uid":"strIdx_f9b06d1a9509","paramMap":{"outputCol":"country_indexed","inputCol":"country2","handleInvalid":"error"}}

fittedBinaryClassifier/bestModel/stages/1_stopWords_c0fd49b80eba/metadata/_SUCCESS fittedBinaryClassifier/bestModel/stages/1_stopWords_fcc7bae6b0cb/metadata/_SUCCESS

@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.feature.StringIndexer","timestamp":1511217003744,"sparkVersion":"2.2.0","uid":"strIdx_0ac1cec40b18","paramMap":{"inputCol":"currency2","outputCol":"currency_indexed","handleInvalid":"error"}}

fittedBinaryClassifier/bestModel/stages/1_stopWords_fcc7bae6b0cb/metadata/part-00000

@@ -1 +1 @@
-{"class":"org.apache.spark.ml.feature.VectorAssembler","timestamp":1511110080197,"sparkVersion":"2.2.0","uid":"vecAssembler_f0e4f9110f47","paramMap":{"inputCols":["tfidf","days_campaign","hours_prepa","goal","country_indexed","currency_indexed"],"outputCol":"features"}}
+{"class":"org.apache.spark.ml.feature.VectorAssembler","timestamp":1511217003823,"sparkVersion":"2.2.0","uid":"vecAssembler_d2eba952c090","paramMap":{"inputCols":["tfidf","days_campaign","hours_prepa","goal","country_indexed","currency_indexed"],"outputCol":"features"}}

fittedBinaryClassifier/bestModel/stages/2_cntVec_09e92d8ef0ef/data/.part-00000-451a0ed3-71ea-4f30-9dd9-db8f675e7ba3-c000.snappy.parquet.crc

@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.classification.LogisticRegression","timestamp":1511217003899,"sparkVersion":"2.2.0","uid":"logreg_33e8252da4e9","paramMap":{"probabilityCol":"probability","elasticNetParam":0.0,"predictionCol":"predictions","rawPredictionCol":"raw_predictions","maxIter":300,"aggregationDepth":2,"featuresCol":"features","labelCol":"final_status","threshold":0.5,"regParam":0.0,"tol":1.0E-6,"thresholds":[0.7,0.3],"fitIntercept":true,"family":"auto","standardization":true}}

fittedBinaryClassifier/bestModel/stages/2_cntVec_09e92d8ef0ef/data/part-00000-451a0ed3-71ea-4f30-9dd9-db8f675e7ba3-c000.snappy.parquet

@@ -1 +1 @@
-{"class":"org.apache.spark.ml.evaluation.MulticlassClassificationEvaluator","timestamp":1511110078581,"sparkVersion":"2.2.0","uid":"mcEval_c1ba9fa849b3","paramMap":{"labelCol":"final_status","predictionCol":"predictions","metricName":"f1"}}
+{"class":"org.apache.spark.ml.evaluation.MulticlassClassificationEvaluator","timestamp":1511217003169,"sparkVersion":"2.2.0","uid":"mcEval_c07e58615b92","paramMap":{"metricName":"f1","labelCol":"final_status","predictionCol":"predictions"}}

fittedBinaryClassifier/bestModel/stages/2_cntVec_09e92d8ef0ef/metadata/.part-00000.crc

@@ -1 +1 @@
-{"class":"org.apache.spark.ml.tuning.TrainValidationSplitModel","timestamp":1511110078368,"sparkVersion":"2.2.0","uid":"tvs_398cd9acb8ff","paramMap":{"trainRatio":0.7,"estimatorParamMaps":[[{"parent":"cntVec_09e92d8ef0ef","name":"minDF","value":"55.0"},{"parent":"logreg_d39bcdb14fc4","name":"regParam","value":"3.3546262790251185E-4"}],[{"parent":"cntVec_09e92d8ef0ef","name":"minDF","value":"95.0"},{"parent":"logreg_d39bcdb14fc4","name":"regParam","value":"3.3546262790251185E-4"}],[{"parent":"cntVec_09e92d8ef0ef","name":"minDF","value":"20.0"},{"parent":"logreg_d39bcdb14fc4","name":"regParam","value":"3.3546262790251185E-4"}],[{"parent":"cntVec_09e92d8ef0ef","name":"minDF","value":"55.0"},{"parent":"logreg_d39bcdb14fc4","name":"regParam","value":"0.1353352832366127"}],[{"parent":"cntVec_09e92d8ef0ef","name":"minDF","value":"95.0"},{"parent":"logreg_d39bcdb14fc4","name":"regParam","value":"0.1353352832366127"}],[{"parent":"cntVec_09e92d8ef0ef","name":"minDF","value":"20.0"},{"parent":"logreg_d39bcdb14fc4","name":"regParam","value":"0.1353352832366127"}],[{"parent":"cntVec_09e92d8ef0ef","name":"minDF","value":"55.0"},{"parent":"logreg_d39bcdb14fc4","name":"regParam","value":"7.38905609893065"}],[{"parent":"cntVec_09e92d8ef0ef","name":"minDF","value":"95.0"},{"parent":"logreg_d39bcdb14fc4","name":"regParam","value":"7.38905609893065"}],[{"parent":"cntVec_09e92d8ef0ef","name":"minDF","value":"20.0"},{"parent":"logreg_d39bcdb14fc4","name":"regParam","value":"7.38905609893065"}]],"seed":-1772833110},"validationMetrics":[0.6427559464974216,0.6387647426733698,0.6472731825778881,0.6292117450403923,0.6233967537374357,0.6369990618096109,0.19164917943322474,0.18150700803152733,0.21242018809143703]}
+{"class":"org.apache.spark.ml.tuning.TrainValidationSplitModel","timestamp":1511217003031,"sparkVersion":"2.2.0","uid":"tvs_1595d27dbd41","paramMap":{"trainRatio":0.7,"estimatorParamMaps":[[{"parent":"logreg_33e8252da4e9","name":"regParam","value":"3.3546262790251185E-4"},{"parent":"cntVec_797fec8a4884","name":"minDF","value":"55.0"}],[{"parent":"logreg_33e8252da4e9","name":"regParam","value":"0.1353352832366127"},{"parent":"cntVec_797fec8a4884","name":"minDF","value":"55.0"}],[{"parent":"logreg_33e8252da4e9","name":"regParam","value":"7.38905609893065"},{"parent":"cntVec_797fec8a4884","name":"minDF","value":"55.0"}],[{"parent":"logreg_33e8252da4e9","name":"regParam","value":"3.3546262790251185E-4"},{"parent":"cntVec_797fec8a4884","name":"minDF","value":"95.0"}],[{"parent":"logreg_33e8252da4e9","name":"regParam","value":"0.1353352832366127"},{"parent":"cntVec_797fec8a4884","name":"minDF","value":"95.0"}],[{"parent":"logreg_33e8252da4e9","name":"regParam","value":"7.38905609893065"},{"parent":"cntVec_797fec8a4884","name":"minDF","value":"95.0"}],[{"parent":"logreg_33e8252da4e9","name":"regParam","value":"3.3546262790251185E-4"},{"parent":"cntVec_797fec8a4884","name":"minDF","value":"20.0"}],[{"parent":"logreg_33e8252da4e9","name":"regParam","value":"0.1353352832366127"},{"parent":"cntVec_797fec8a4884","name":"minDF","value":"20.0"}],[{"parent":"logreg_33e8252da4e9","name":"regParam","value":"7.38905609893065"},{"parent":"cntVec_797fec8a4884","name":"minDF","value":"20.0"}]],"seed":-1772833110},"validationMetrics":[0.6427559464974216,0.6292117450403923,0.19164917943322474,0.6387647426733698,0.6233967537374357,0.18150700803152733,0.6472731825778881,0.6369990618096109,0.21242018809143703]}