SDAP-71 full ingestion workflow and process broken in several places (#16)

Author: lewismc

.gitignore

@@ -15,3 +15,5 @@ core/.externalToolBuilders/Maven_Ant_Builder.launch
 core/maven-eclipse.xml
 service/.classpath
 web/.classpath
+web/.externalToolBuilders/
+web/maven-eclipse.xml

core/src/main/java/org/apache/sdap/mudrod/driver/ESDriver.java

@@ -177,15 +177,31 @@ public List<String> customAnalyzing(String indexName, List<String> list) throws
   }
 
   public void deleteAllByQuery(String index, String type, QueryBuilder query) {
-    ImmutableOpenMap<String, MappingMetaData> mappings = getClient().admin().cluster().prepareState().execute().actionGet()
-        .getState().metaData().index(index).getMappings();
+    ImmutableOpenMap<String, MappingMetaData> mappings = getClient()
+            .admin()
+            .cluster()
+            .prepareState()
+            .execute()
+            .actionGet()
+            .getState()
+            .metaData()
+            .index(index)
+            .getMappings();
 
     //check if the type exists
-    if (!mappings.containsKey(type)) return;
+    if (!mappings.containsKey(type))
+      return;
 
     createBulkProcessor();
-    SearchResponse scrollResp = getClient().prepareSearch(index).setSearchType(SearchType.QUERY_AND_FETCH).setTypes(type).setScroll(new TimeValue(60000)).setQuery(query).setSize(10000).execute()
-        .actionGet();
+    SearchResponse scrollResp = getClient()
+            .prepareSearch(index)
+            .setSearchType(SearchType.QUERY_AND_FETCH)
+            .setTypes(type)
+            .setScroll(new TimeValue(60000))
+            .setQuery(query)
+            .setSize(10000)
+            .execute()
+            .actionGet();
 
     while (true) {
       for (SearchHit hit : scrollResp.getHits().getHits()) {

core/src/main/java/org/apache/sdap/mudrod/integration/LinkageIntegration.java

@@ -171,7 +171,7 @@ public JsonObject getIngeratedListInJson(String input) {
    * the similarities from different sources
    */
   public Map<String, List<LinkedTerm>> aggregateRelatedTermsFromAllmodel(String input) {
-    aggregateRelatedTerms(input, MudrodConstants.USE_HISTORY_LINKAGE_TYPE);
+    aggregateRelatedTerms(input, MudrodConstants.USER_HISTORY_LINKAGE_TYPE);
     aggregateRelatedTerms(input, MudrodConstants.CLICK_STREAM_LINKAGE_TYPE);
     aggregateRelatedTerms(input, MudrodConstants.METADATA_LINKAGE_TYPE);
     aggregateRelatedTermsSWEET(input, MudrodConstants.ONTOLOGY_LINKAGE_TYPE);
@@ -180,7 +180,7 @@ public Map<String, List<LinkedTerm>> aggregateRelatedTermsFromAllmodel(String in
   }
 
   public int getModelweight(String model) {
-    if (model.equals(MudrodConstants.USE_HISTORY_LINKAGE_TYPE)) {
+    if (model.equals(MudrodConstants.USER_HISTORY_LINKAGE_TYPE)) {
       return Integer.parseInt(props.getProperty(MudrodConstants.USER_HISTORY_W));
     }
 

core/src/main/java/org/apache/sdap/mudrod/main/MudrodConstants.java

@@ -15,23 +15,22 @@
 
 /**
  * Class contains static constant keys and values relating to Mudrod
- * configuration properties. Property values are read from <a href=
- * "https://github.com/mudrod/mudrod/blob/master/core/src/main/resources/config.xml">config.xml</a>
+ * configuration properties. Property values are read from <code>config.properties</code>.
  */
 public interface MudrodConstants {
 
   public static final String CLEANUP_TYPE = "cleanup.log";
 
-  public static final String CLICK_STREAM_LINKAGE_TYPE = "click.stream.linkage";
+  public static final String CLICK_STREAM_LINKAGE_TYPE = "mudrod.clickstream.linkage";
 
-  public static final String CLICK_STREAM_MATRIX_TYPE = "click.stream.matrix";
+  public static final String CLICK_STREAM_MATRIX_TYPE = "mudrod.clickstream.matrix";
 
   public static final String CLICKSTREAM_SVD_DIM = "mudrod.clickstream.svd.d";
 
   public static final String CLICKSTREAM_W = "mudrod.clickstream.weight";
-  
+
   public static final String CLICKSTREAM_PATH = "mudrod.clickstream.path";
-  
+
   public static final String CLICKSTREAM_SVD_PATH = "mudrod.clickstream.svd.path";
 
   /** Defined on CLI */
@@ -52,25 +51,25 @@ public interface MudrodConstants {
   public static final String FTP_PREFIX = "mudrod.ftp.prefix";
 
   public static final String FTP_TYPE = "raw.ftp";
-  
+
   public static final String FTP_LOG = "ftp";
 
   public static final String HTTP_PREFIX = "mudrod.http.prefix";
 
   public static final String HTTP_TYPE = "raw.http";
-  
+
   public static final String HTTP_LOG = "http";
-  
+
   public static final String BASE_URL = "mudrod.base.url";
-  
+
   public static final String BLACK_LIST_REQUEST = "mudrod.black.request.list";
-  
+
   public static final String BLACK_LIST_AGENT = "mudrod.black.agent.list";
 
   public static final String LOG_INDEX = "mudrod.log.index";
 
   public static final String METADATA_LINKAGE_TYPE = "metadata.linkage";
-  
+
   public static final String METADATA_DOWNLOAD_URL = "mudrod.metadata.download.url";
 
   public static final String METADATA_SVD_DIM = "mudrod.metadata.svd.d";
@@ -93,38 +92,38 @@ public interface MudrodConstants {
   public static final String ONTOLOGY_LINKAGE_TYPE = "ontology.linkage";
 
   public static final String ONTOLOGY_W = "mudrod.ontology.weight";
-  
+
   public static final String ONTOLOGY_PATH = "mudrod.ontology.path";
-  
+
   public static final String ONTOLOGY_INPUT_PATH = "mudrod.ontology.input.path";
 
   /** Defined on CLI */
   public static final String METADATA_DOWNLOAD = "mudrod.metadata.download";
-  
+
   public static final String RAW_METADATA_PATH = "mudrod.metadata.path";
 
   public static final String RAW_METADATA_TYPE = "mudrod.metadata.type";
-  
+
   public static final String METADATA_MATRIX_PATH = "mudrod.metadata.matrix.path";
-  
+
   public static final String METADATA_SVD_PATH = "mudrod.metadata.svd.path";
-  
+
   public static final String RECOM_METADATA_TYPE = "recommedation.metadata";
-  
+
   public static final String METADATA_ID = "mudrod.metadata.id";
-  
+
   public static final String SEMANTIC_FIELDS = "mudrod.metadata.semantic.fields";
-  
+
   public static final String METADATA_WORD_SIM_TYPE = "metadata.word.sim";
-  
+
   public static final String METADATA_FEATURE_SIM_TYPE = "metadata.feature.sim";
-  
+
   public static final String METADATA_SESSION_SIM_TYPE = "metadata.session.sim";
-  
+
   public static final String METADATA_TERM_MATRIX_PATH = "metadata.term.matrix.path";
-  
+
   public static final String METADATA_WORD_MATRIX_PATH = "metadata.word.matrix.path";
-  
+
   public static final String METADATA_SESSION_MATRIX_PATH = "metadata.session.matrix.path";
 
   public static final String REQUEST_RATE = "mudrod.request.rate";
@@ -138,32 +137,29 @@ public interface MudrodConstants {
   public static final String SPARK_APP_NAME = "mudrod.spark.app.name";
 
   public static final String SPARK_MASTER = "mudrod.spark.master";
-  /**
-   * Absolute local location of javaSVMWithSGDModel directory. This is typically
-   * <code>file:///usr/local/mudrod/core/src/main/resources/javaSVMWithSGDModel</code>
-   */
+
   public static final String RANKING_MODEL = "mudrod.ranking.model";
-  
+
   public static final String RANKING_ML = "mudrod.ranking.machine.learning";
 
   public static final String REQUEST_TIME_GAP = "mudrod.request.time.gap";
 
   public static final String TIME_SUFFIX = "time.suffix";
 
-  public static final String USE_HISTORY_LINKAGE_TYPE = "user.history.linkage";
+  public static final String USER_HISTORY_LINKAGE_TYPE = "mudrod.user.history.linkage";
 
   public static final String USER_HISTORY_W = "mudrod.user.history.weight";
-  
+
   public static final String USER_HISTORY_PATH = "mudrod.user.history.path";
 
   public static final String VIEW_F = "mudrod.view.freq";
-  
+
   public static final String VIEW_MARKER = "mudrod.view.url.marker";
-  
+
   public static final String SEARCH_MARKER = "mudrod.search.url.marker";
-  
+
   public static final String SEARCH_F = "mudrod.search.freq";
-  
+
   public static final String DOWNLOAD_F = "mudrod.download.freq";
 
 }

core/src/main/java/org/apache/sdap/mudrod/metadata/pre/ApiHarvester.java

@@ -32,7 +32,7 @@
 import java.util.Properties;
 
 /**
- * ClassName: ApiHarvester Function: Harvest metadata from PO.DAACweb service.
+ * Harvest metadata from PO.DAAC Webservices.
  */
 public class ApiHarvester extends DiscoveryStepAbstract {
 
@@ -70,7 +70,7 @@ public Object execute() {
   }
 
   /**
-   * addMetadataMapping: Add mapping to index metadata in Elasticsearch. Please
+   * Add mapping to index metadata in Elasticsearch. Please
    * invoke this method before import metadata to Elasticsearch.
    */
   public void addMetadataMapping() {
@@ -84,7 +84,7 @@ public void addMetadataMapping() {
   }
 
   /**
-   * importToES: Index metadata into elasticsearch from local file directory.
+   * Index metadata into elasticsearch from local file directory.
    * Please make sure metadata have been harvest from web service before
    * invoking this method.
    */
@@ -118,12 +118,12 @@ private void importSingleFileToES(InputStream is) {
   }
 
   /**
-   * harvestMetadatafromWeb: Harvest metadata from PO.DAAC web service.
+   * Harvest metadata from PO.DAAC web service.
    */
   private void harvestMetadatafromWeb() {
     LOG.info("Metadata download started.");
     int startIndex = 0;
-    int doc_length = 0;
+    int docLength = 0;
     JsonParser parser = new JsonParser();
     do {
       String searchAPI = props.getProperty(MudrodConstants.METADATA_DOWNLOAD_URL);
@@ -135,7 +135,7 @@ private void harvestMetadatafromWeb() {
       JsonObject responseObject = json.getAsJsonObject();
       JsonArray docs = responseObject.getAsJsonObject("response").getAsJsonArray("docs");
 
-      doc_length = docs.size();
+      docLength = docs.size();
 
       File file = new File(props.getProperty(MudrodConstants.RAW_METADATA_PATH));
       if (!file.exists()) {
@@ -145,7 +145,7 @@ private void harvestMetadatafromWeb() {
           LOG.error("Failed to create directory!");
         }
       }
-      for (int i = 0; i < doc_length; i++) {
+      for (int i = 0; i < docLength; i++) {
         JsonElement item = docs.get(i);
         int docId = startIndex + i;
         File itemfile = new File(props.getProperty(MudrodConstants.RAW_METADATA_PATH) + "/" + docId + ".json");
@@ -167,7 +167,7 @@ private void harvestMetadatafromWeb() {
         Thread.currentThread().interrupt();
       }
 
-    } while (doc_length != 0);
+    } while (docLength != 0);
 
     LOG.info("Metadata downloading finished");
   }

core/src/main/java/org/apache/sdap/mudrod/metadata/pre/MatrixGenerator.java

@@ -64,7 +64,12 @@ public Object execute() {
     String metadataMatrixFile = props.getProperty(MudrodConstants.METADATA_MATRIX_PATH);
     try {
       MetadataExtractor extractor = new MetadataExtractor();
-      JavaPairRDD<String, List<String>> metadataTermsRDD = extractor.loadMetadata(this.es, this.spark.sc, props.getProperty(MudrodConstants.ES_INDEX_NAME), props.getProperty(MudrodConstants.RAW_METADATA_TYPE));
+      JavaPairRDD<String, List<String>> metadataTermsRDD = 
+              extractor.loadMetadata(
+                      this.es,
+                      this.spark.sc,
+                      props.getProperty(MudrodConstants.ES_INDEX_NAME),
+                      props.getProperty(MudrodConstants.RAW_METADATA_TYPE));
       LabeledRowMatrix wordDocMatrix = MatrixUtil.createWordDocMatrix(metadataTermsRDD);
       MatrixUtil.exportToCSV(wordDocMatrix.rowMatrix, wordDocMatrix.rowkeys, wordDocMatrix.colkeys, metadataMatrixFile);
 

core/src/main/java/org/apache/sdap/mudrod/metadata/process/MetadataAnalyzer.java

@@ -27,8 +27,7 @@
 import java.util.Properties;
 
 /**
- * ClassName: MetadataAnalyzer
- * Function: Calculate semantic relationship of vocabularies extracted from
+ * Calculate semantic relationship of vocabularies extracted from
  * metadata.
  */
 public class MetadataAnalyzer extends DiscoveryStepAbstract implements Serializable {

core/src/main/java/org/apache/sdap/mudrod/recommendation/pre/MetadataTFIDFGenerator.java

@@ -81,7 +81,11 @@ public LabeledRowMatrix generateWordBasedTFIDF() throws Exception {
 
     LabeledRowMatrix wordtfidfMatrix = opt.tFIDFTokens(metadataWords, spark);
 
-    MatrixUtil.exportToCSV(wordtfidfMatrix.rowMatrix, wordtfidfMatrix.rowkeys, wordtfidfMatrix.colkeys, props.getProperty(MudrodConstants.METADATA_WORD_MATRIX_PATH));
+    MatrixUtil.exportToCSV(
+            wordtfidfMatrix.rowMatrix,
+            wordtfidfMatrix.rowkeys,
+            wordtfidfMatrix.colkeys,
+            props.getProperty(MudrodConstants.METADATA_WORD_MATRIX_PATH));
 
     return wordtfidfMatrix;
   }
@@ -100,7 +104,11 @@ public LabeledRowMatrix generateTermBasedTFIDF() throws Exception {
 
     LabeledRowMatrix tokentfidfMatrix = opt.tFIDFTokens(metadataTokens, spark);
 
-    MatrixUtil.exportToCSV(tokentfidfMatrix.rowMatrix, tokentfidfMatrix.rowkeys, tokentfidfMatrix.colkeys, props.getProperty(MudrodConstants.METADATA_TERM_MATRIX_PATH));
+    MatrixUtil.exportToCSV(
+            tokentfidfMatrix.rowMatrix,
+            tokentfidfMatrix.rowkeys,
+            tokentfidfMatrix.colkeys,
+            props.getProperty(MudrodConstants.METADATA_TERM_MATRIX_PATH));
 
     return tokentfidfMatrix;
   }