Refactoring OODT engine, and adding WingsTask & WMServices

.gitignore

@@ -1,5 +1,4 @@
 .*
 target
 !/.gitignore
-oodt/*
 portal/src/main/webapp/lib/*

engines/oodt/README

@@ -0,0 +1,52 @@
+Instrutions for integrating with OODT
+=====================================
+
+Modifications to Wings
+----------------------
+1. In the Wings POM, add the following module
+    <module>engines/oodt</module>
+2. In the Wings Portal POM, uncomment the part which says "Wings OODT Adapter"
+3. Compile Wings. "mvn install". 
+	This should also compile oodt engine, and put appropriate jars in the Wings Portal
+4. Put portal/target/wings-portal-<version>.war to the tomcat container
+5. Put engines/oodt/wmservices/target/wings-oodt-wmservices-<version>.war to the tomcat container
+
+Modifications to OODT
+---------------------
+1. Install OODT (Latest 0.8-SNAPSHOT from trunk)
+2. Copy engines/oodt/task/target/wings-oodt-task-<version>.jar to [OODT_HOME]/workflow/lib
+3. Copy [OODT_HOME]/filemgr/lib/cas-filemgr-<version>.jar to [OODT_HOME]/workflow/lib
+4. Make sure that the cas-curator webapp is installed in tomcat
+
+Additions to Wings portal.properties
+------------------------------------
+1. Add the following OODT engine config to the execution section. Modify the urls and paths according to your installation.
+        engine = 
+        {
+            name = OODT;
+            implementation = edu.isi.wings.execution.engine.api.impl.oodt.OODTExecutionEngine;
+            type = BOTH;
+			properties = 
+			{
+				extern_data_catalog = edu.isi.wings.catalog.data.api.impl.oodt.DataCreationFM;
+				oodt =
+				{
+					fmurl = http://localhost:9000;
+					fmpolicy = wings;
+					wmurl = http://localhost:9001;
+					wmsurl = http://localhost:8080/wings-oodt-wmservices;
+					curatorurl = http://localhost:8080/cas-curator;
+					archivedir = /Users/varun/Servers/oodt/filemgr/archive;
+				}
+			}
+        }
+
+That's it !
+
+
+Usage
+-----
+1. From the "Manage Domains" interface, change the domain execution engine to OODT
+2. Go to the "Manage Data" interface once. This will sync up the data with OODT file manager
+
+

oodt-adapter/pom.xml → engines/oodt/adapter/pom.xml

@@ -12,9 +12,9 @@
 
   <parent>
     <groupId>edu.isi.wings</groupId>
-    <artifactId>wings-core</artifactId>
+    <artifactId>wings-oodt</artifactId>
     <version>4.0-SNAPSHOT</version>
-    <relativePath>../core/pom.xml</relativePath>
+    <relativePath>../pom.xml</relativePath>
   </parent>
 
   <properties>
@@ -32,15 +32,16 @@
       <version>${project.version}</version>
     </dependency>
 
-    <!-- Apache OODT Dependencies -->
     <dependency>
-      <groupId>org.apache.oodt</groupId>
-      <artifactId>cas-filemgr</artifactId>
-      <version>${oodt.version}</version>
+      <groupId>edu.isi.wings</groupId>
+      <artifactId>wings-oodt-wmservices</artifactId>
+      <version>${project.version}</version>
     </dependency>
+
+    <!-- Apache OODT Dependencies -->
     <dependency>
       <groupId>org.apache.oodt</groupId>
-      <artifactId>cas-wm-services</artifactId>
+      <artifactId>cas-filemgr</artifactId>
       <version>${oodt.version}</version>
     </dependency>
   </dependencies>

oodt-adapter/src/main/java/edu/isi/wings/execution/engine/api/impl/oodt/OODTExecutionEngine.java → engines/oodt/adapter/src/main/java/edu/isi/wings/execution/engine/api/impl/oodt/OODTExecutionEngine.java

@@ -36,7 +36,6 @@ public class OODTExecutionEngine implements PlanExecutionEngine, StepExecutionEn
   String wlogfile;
 
   public OODTExecutionEngine(Properties props) {
-    System.out.println(props);
     this.props = props;
     this.stepEngine = this;
     this.planEngine = this;

engines/oodt/pom.xml

@@ -0,0 +1,26 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>edu.isi.wings</groupId>
+  <artifactId>wings-oodt</artifactId>
+  <name>Wings OODT Engine</name>
+  <description>Wings OODT Engine information</description>
+  <packaging>pom</packaging>
+
+  <parent>
+    <groupId>edu.isi.wings</groupId>
+    <artifactId>wings-core</artifactId>
+    <version>4.0-SNAPSHOT</version>
+    <relativePath>../../core/pom.xml</relativePath>
+  </parent>
+
+  <properties>
+    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+  </properties>
+
+  <modules>
+    <module>adapter</module>
+    <module>task</module>
+    <module>wmservices</module>
+  </modules>
+</project>

engines/oodt/task/pom.xml

@@ -0,0 +1,36 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>org.apache.oodt.cas.workflow.misc</groupId>
+  <artifactId>wings-oodt-task</artifactId>
+  <name>Wings OODT Task</name>
+  <packaging>jar</packaging>
+  <description>Wings Task for OODT Workflow system</description>
+
+  <parent>
+    <groupId>edu.isi.wings</groupId>
+    <artifactId>wings-oodt</artifactId>
+    <version>4.0-SNAPSHOT</version>
+    <relativePath>../pom.xml</relativePath>
+  </parent>
+
+  <properties>
+    <oodt.version>0.8-SNAPSHOT</oodt.version>
+    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+  </properties>
+
+ <dependencies>    
+    <dependency>
+      <groupId>org.apache.oodt</groupId>
+      <artifactId>cas-filemgr</artifactId>
+      <version>${oodt.version}</version>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.oodt</groupId>
+      <artifactId>cas-workflow</artifactId>
+      <version>${oodt.version}</version>
+      <scope>provided</scope>
+    </dependency>
+  </dependencies>
+</project>

engines/oodt/task/src/main/java/org/apache/oodt/cas/workflow/misc/WingsTask.java

@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oodt.cas.workflow.misc;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.PrintStream;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Properties;
+
+import org.apache.commons.io.FileUtils;
+//OODT imports
+import org.apache.oodt.cas.workflow.structs.WorkflowTaskInstance;
+import org.apache.oodt.cas.workflow.structs.WorkflowTaskConfiguration;
+import org.apache.oodt.cas.filemgr.datatransfer.DataTransfer;
+import org.apache.oodt.cas.filemgr.datatransfer.RemoteDataTransferFactory;
+import org.apache.oodt.cas.filemgr.structs.Product;
+import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient;
+import org.apache.oodt.cas.metadata.Metadata;
+
+/**
+ * @author Varun Ratnakar
+ * @version $Revsion$
+ * 
+ * <p>A Wings Task (http://www.wings-workflows.org)</p>
+ */
+public class WingsTask implements WorkflowTaskInstance {
+
+	/**
+	 * 
+	 */
+	public WingsTask() {
+	}
+
+	/* (non-Javadoc)
+	 * @see org.apache.oodt.cas.workflow.structs.WorkflowTaskInstance#run(java.util.Map, 
+	 * org.apache.oodt.cas.workflow.structs.WorkflowTaskConfiguration)
+	 */
+	public void run(Metadata metadata, WorkflowTaskConfiguration config) {
+		Properties props = config.getProperties();
+		// Component Info
+		String tname = props.getProperty("TASKNAME");
+		String jobid = props.getProperty("JOBID");
+		String argstring = props.getProperty("ARGUMENT");
+		ArrayList<String> inputs = fetchFromProps(props, "INPUT");
+		ArrayList<String> outputs = fetchFromProps(props, "OUTPUT");
+
+		// Following paths should be Shared across the cluster
+		String script = props.getProperty("SCRIPT_PATH");
+		String jobdir = props.getProperty("JOB_DIR");
+		String datadir = props.getProperty("DATA_DIR");
+
+		// File Manager Access
+		String fmurl = props.getProperty("FM_URL");
+		String fmprefix = props.getProperty("FM_PREFIX");
+
+		// Logging specific info
+		String logfile = props.getProperty("LOGFILE");
+		String wlogfile = props.getProperty("W_LOGFILE");
+
+		PrintStream wlogout = null;
+		PrintStream logout = null;
+
+		try {
+			XmlRpcFileManagerClient fmclient = new XmlRpcFileManagerClient(new URL(fmurl));
+			DataTransfer dt = new RemoteDataTransferFactory().createDataTransfer();
+			dt.setFileManagerUrl(new URL(fmurl));
+
+			wlogout = new PrintStream(new FileOutputStream(jobdir+wlogfile, true));
+			logout = new PrintStream(jobdir+logfile);
+
+			wlogout.println(jobid+" ("+tname+"): RUNNING");
+			wlogout.flush();
+
+			logout.println("[INFO]: Component Initializing");
+			logout.println(tname+" "+argstring);
+
+			// Fetch input files from file manager if not already present in directory
+			for (String ip : inputs) {
+				File f = new File(jobdir + ip);
+				if (!f.exists()) {
+					logout.println("[INFO] Fetching Input from File Manager: " + ip);
+					Product prod = fmclient.getProductById(fmprefix + ip);
+					prod.setProductReferences(fmclient.getProductReferences(prod));
+					dt.retrieveProduct(prod, new File(jobdir));
+				}
+			}
+			logout.flush();
+
+		    ArrayList<String> command = new ArrayList<String>();
+		    command.add(script);
+		    for(String s : argstring.split(" "))
+		    	command.add(s);
+
+		    ProcessBuilder builder = new ProcessBuilder(command);
+			builder.directory(new File(jobdir));
+			builder.redirectErrorStream(true);
+
+		    final Process process = builder.start();
+
+			InputStream is = process.getInputStream();
+			InputStreamReader isr = new InputStreamReader(is);
+			BufferedReader br = new BufferedReader(isr);
+			String line;
+			while ((line = br.readLine()) != null) {
+				logout.println(line);
+			}
+			process.waitFor();
+			int exitStatus = process.exitValue();
+			if (exitStatus != 0) 
+				throw new Exception("[ERROR] Component failed with a non-zero exit code");
+
+			for (String op : outputs) {
+				File f = new File(jobdir + op);
+				if(!f.exists())
+					throw new Exception("[ERROR] Missing Output "+op);
+
+				// Copy output to wings data location
+				FileUtils.copyFileToDirectory(f, new File(datadir));
+
+				// TODO: Ingest output files to file manager
+			}
+		    logout.println("SUCCESS: Component finished successfully !");
+		    logout.close();
+		    wlogout.println(jobid+" ("+tname+"): SUCCESS");
+		    wlogout.close();
+		}
+		catch (Exception e) {
+			if(logout != null) {
+				logout.println(e.getMessage());
+				logout.println("FAILURE: Component Failed");
+				logout.close();
+				wlogout.println(jobid+" ("+tname+"): FAILURE");
+				wlogout.close();
+			}
+		}
+	}
+
+	private ArrayList<String> fetchFromProps(Properties props, String argtype) {
+		ArrayList<String> args = new ArrayList<String>();
+		int i=1;
+		while(props.containsKey(argtype+i)) {
+			args.add(props.getProperty(argtype+i));
+			i++;
+		}
+		return args;
+	}
+}

engines/oodt/task/src/main/resources/config/PGEConfig.xml

@@ -0,0 +1,42 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<pgeConfig>
+        <!-- How to run the PGE -->
+        <exe dir="[JobDir]" shell="/bin/bash">
+          <!-- cd to PGE root -->
+          <cmd>cd [PGE_ROOT]/file_concatenator</cmd>
+          <cmd>cp [InputFile1] [OutputFile]</cmd>
+          <cmd>cat [InputFile2] >> [OutputFile]</cmd>
+        </exe>
+
+	    <!-- Files to ingest -->
+	    <output>
+        	<!-- one or more of these -->
+       	 	<dir path="[JobDir]" createBeforeExe="false">
+
+            	<!-- one or more of these ** regExp or name can be used-->
+            	<files regExp=".*\.txt" metFileWriterClass="org.apache.oodt.cas.workflow.misc.writers.ConcactenatingFilenameExtractorWriter"  args="[PGE_ROOT]/file_concatenator/extractors/concatenatingfilename.extractor.config.xml"/>
+                <files regExp=".*\.txt" metFileWriterClass="org.apache.oodt.cas.pge.writers.metlist.MetadataListPcsMetFileWriter" args="[PGE_ROOT]/file_concatenator/extractors/metlistwriter/metout.xml"/>
+            </dir>
+        </output>
+
+        <!-- Custom metadata to add to output files -->
+        <customMetadata>
+                <!-- helpful keys -->
+                <metadata key="LessThan" val="&#x3C;"/>
+                <metadata key="LessThanOrEqualTo" val="[LessThan]="/>
+                <metadata key="GreaterThan" val="&#x3E;"/>
+                <metadata key="GreaterThanOrEqualTo" val="[GreaterThan]="/>
+                <metadata key="Exclamation" val="&#33;"/>
+                <metadata key="Ampersand" val="&#38;"/>
+                <metadata key="NotEqualTo" val="[Ampersand]="/>
+                <metadata key="LogicalAnd" val="[Ampersand][Ampersand]"/>
+                <metadata key="CshPipeToStdOutAndError" val="[GreaterThan][Ampersand][Exclamation]"/>
+
+				<metadata key="ProductionDateTime" val="[DATE.UTC]"/>
+				<metadata key="JobDir" val="[PGE_ROOT]/file_concatenator/output/jobs/job-[ProductionDateTime]"/>
+				<metadata key="InputFile1" val="[PGE_ROOT]/file_concatenator/files/concatenatingInputFile1.txt"/>
+				<metadata key="InputFile2" val="[PGE_ROOT]/file_concatenator/files/concatenatingInputFile2.txt"/>
+				<metadata key="OutputFile" val="[JobDir]/concatenatedOutputFile-[ProductionDateTime].txt"/>        
+      </customMetadata>
+
+</pgeConfig>