ISSUE-9 Implement Workflows in COAL-SDS

crawler/src/main/resources/bin/crawlctl

@@ -16,7 +16,8 @@
 ##########################################################################
 
 #java -Djava.util.logging.config.file=../etc/logging.properties -classpath "../lib/*" org.apache.oodt.cas.crawl.daemon.CrawlDaemonController $@
-./crawler_launcher --filemgrUrl http://localhost:9000 \
+./crawler/bin/crawler_launcher \
+  --filemgrUrl http://localhost:9000 \
   --operation --launchMetCrawler \
   --clientTransferer org.apache.oodt.cas.filemgr.datatransfer.LocalDataTransferFactory \
   --productPath /usr/local/coal-sds-deploy/data/staging \
@@ -26,4 +27,5 @@
   --daemonPort 9003 \
   --daemonWait 2 \
   --successDir /usr/local/coal-sds-deploy/data/archive \
-  --actionIds DeleteDataFile
+  --actionIds DeleteDataFile TriggerPostIngestWorkflow \
+  --workflowMgrUrl http://localhost:9001

crawler/src/main/resources/policy/cmd-line-options.xml

@@ -253,7 +253,7 @@
             <bean class="org.apache.oodt.cas.crawl.cli.option.handler.CrawlerBeansPropHandler">
                 <property name="properties">
                     <list>
-                        <value>UpdateWorkflowStatusToIngest.workflowMgrUrl</value>
+                        <value>TriggerPostIngestWorkflow.workflowMgrUrl</value>
                     </list>
                 </property>
             </bean>

distribution/src/main/resources/bin/oodt

@@ -106,14 +106,14 @@ if [ "$1" = "start" ]; then
 # Print confirmation messages to the end user
 # FileManager
 # check if the PID file exists:
-  if [ ! -e "../filemgr/run/cas.filemgr.pid" ] 
+  if [ ! -e "$FILEMGR_HOME/run/cas.filemgr.pid" ] 
   then
     # no pid file was created by the process, it must have failed.
     echo -e 'Starting OODT File Manager [' $'\e[31m' 'Failed' $'\e[00m' ']'
 
   else
     # pid file exists, check if the associated process is running.
-    if kill -0 `cat ../filemgr/run/cas.filemgr.pid` > /dev/null 2>&1;
+    if kill -0 `cat "$FILEMGR_HOME"/run/cas.filemgr.pid` > /dev/null 2>&1;
     then
         echo -e 'Starting OODT File Manager [' $'\e[32m' 'Successful' $'\e[00m' ']'
     else
@@ -123,14 +123,14 @@ if [ "$1" = "start" ]; then
 
 # ResourceManager
 # check if the PID file exists:
-  if [ ! -e "../resmgr/run/cas.resmgr.pid" ]
+  if [ ! -e "$RESMGR_HOME/resmgr/run/cas.resmgr.pid" ]
   then
     # no pid file was created by the process, it must have failed.
     echo -e 'Starting OODT Resource Manager [' $'\e[31m' 'Failed' $'\e[00m' ']'
 
   else
     # pid file exists, check if the associated process is running.
-    if kill -0 `cat ../resmgr/run/cas.resmgr.pid` > /dev/null 2>&1;
+    if kill -0 `cat "$RESMGR_HOME"/run/cas.resmgr.pid` > /dev/null 2>&1;
     then
         echo -e 'Starting OODT Resource Manager [' $'\e[32m' 'Successful' $'\e[00m' ']'
     else
@@ -141,14 +141,14 @@ if [ "$1" = "start" ]; then
 
 # WorkflowManager
 # check if the PID file exists:
-  if [ ! -e "../workflow/run/cas.workflow.pid" ]
+  if [ ! -e "$WORKFLOW_HOME/run/cas.workflow.pid" ]
   then
     # no pid file was created by the process, it must have failed.
     echo -e 'Starting OODT Workflow Manager [' $'\e[31m' 'Failed' $'\e[00m' ']'
 
   else
     # pid file exists, check if the associated process is running.
-    if kill -0 `cat ../workflow/run/cas.workflow.pid` > /dev/null 2>&1;
+    if kill -0 `cat "$WORKFLOW_HOME"/run/cas.workflow.pid` > /dev/null 2>&1;
     then
         echo -e 'Starting OODT Workflow Manager [' $'\e[32m' 'Successful' $'\e[00m' ']'
     else

filemgr/src/main/resources/bin/filemgr

@@ -85,7 +85,7 @@ if [ "$1" = "start" ]; then
     -Djava.util.logging.config.file="$FILEMGR_HOME"/etc/logging.properties \
     -Dorg.apache.oodt.cas.filemgr.properties="$FILEMGR_HOME"/etc/filemgr.properties \
     -Djava.io.tmpdir="$OODT_TMPDIR" \
-    org.apache.oodt.cas.filemgr.system.XmlRpcFileManager \
+    org.apache.oodt.cas.filemgr.system.FileManagerServerMain \
     --portNum $FILEMGR_PORT 2>&1 &
 
   if [ ! -z "$FILEMGR_PID" ]; then

filemgr/src/main/resources/bin/filemgr-client

@@ -74,4 +74,4 @@ cd "$FILEMGR_HOME"/bin
   -Djava.util.logging.config.file="$FILEMGR_HOME"/etc/logging.properties \
   -Dorg.apache.oodt.cas.cli.action.spring.config=file:"$FILEMGR_HOME"/policy/cmd-line-actions.xml \
   -Dorg.apache.oodt.cas.cli.option.spring.config=file:"$FILEMGR_HOME"/policy/cmd-line-options.xml \
-  org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient "$@"
+  org.apache.oodt.cas.filemgr.system.FileManagerClientMain "$@"

filemgr/src/main/resources/etc/filemgr.fm-solr-catalog.properties

@@ -15,6 +15,9 @@
 
 # Configuration properties for the File Manager
 
+filemgr.server=org.apache.oodt.cas.filemgr.system.rpc.AvroFileManagerServerFactory
+filemgr.client=org.apache.oodt.cas.filemgr.system.rpc.AvroFileManagerClientFactory
+
 # repository factory
 filemgr.repository.factory=org.apache.oodt.cas.filemgr.repository.XMLRepositoryManagerFactory
 
@@ -56,58 +59,13 @@ org.apache.oodt.cas.filemgr.datatransfer.remote.chunkSize=1024
 # location of Mime-Type repository
 org.apache.oodt.cas.filemgr.mime.type.repository=/usr/local/coal-sds-deploy/filemgr/etc/mime-types.xml
 
-
-############ data source configuration ##################################
-#
-# These 3 blocks of config properties can be used to setup a catalog,
-# repository manager, and validation layer based on a RDBMS such as
-# Oracle, MySQL, PostgreSQL, or any others that support a jdbc connection.
-# Just un-comment the following blocks of properties and configure as
-# needed.
-#
-#########################################################################
-
-# datasource catalog config
-#org.apache.oodt.cas.filemgr.catalog.datasource.jdbc.url=some_jdbc_url
-#org.apache.oodt.cas.filemgr.catalog.datasource.jdbc.user=user
-#org.apache.oodt.cas.filemgr.catalog.datasource.jdbc.pass=pass
-#org.apache.oodt.cas.filemgr.catalog.datasource.jdbc.driver=driver.class.name
-#org.apache.oodt.cas.filemgr.catalog.datasource.quoteFields=false
-#org.apache.oodt.cas.filemgr.catalog.datasource.pageSize=20
-#org.apache.oodt.cas.filemgr.catalog.datasource.cacheUpdateMinutes=5
-
-# data source repository manager configuration
-#org.apache.oodt.cas.filemgr.repositorymgr.datasource.jdbc.url=some_jdbc_url
-#org.apache.oodt.cas.filemgr.repositorymgr.datasource.jdbc.user=user
-#org.apache.oodt.cas.filemgr.repositorymgr.datasource.jdbc.pass=pass
-#org.apache.oodt.cas.filemgr.repositorymgr.datasource.jdbc.driver=driver.class.name
-
-# data source validation layer configuration
-#org.apache.oodt.cas.filemgr.validation.datasource.jdbc.url=some_jdbc_url
-#org.apache.oodt.cas.filemgr.validation.datasource.jdbc.user=user
-#org.apache.oodt.cas.filemgr.validation.datasource.jdbc.pass=pass
-#org.apache.oodt.cas.filemgr.validation.datasource.jdbc.driver=driver.class.name
-#org.apache.oodt.cas.filemgr.validation.datasource.quoteFields=false
-
-
-# tells the file manager system layer to include product instance metadata
-# NOTE: here are the expected field mappings
-#
-# product.getProductId() -> ProductId
-# product.getProductName() -> ProductName
-# product.getProductStructure() -> ProductStructure
-# product.getTransferStatus() -> ProductTransferStatus
-# product.getRootRef() -> ProductRootReference
-
-# for the references returned by product.getProductReferences() the following
-# metadata fields will be added (order will be maintained, such that data store
-# ref at index 0 will map to orig ref at index 0, etc.)
-#
-# ProductDataStoreReferences (list of all data store references:
-# note already translated into path, not URI)
-# ProductOrigReferences (list of all orig references:
-# note already translated into path, not URI)
-# ProductMimeType (list of all references' mime-types)
-# ProductFileSize (list of all references' file sizes)
-
 org.apache.oodt.cas.filemgr.metadata.expandProduct=false
+
+# Amazon S3 data transfer configuration.
+# Region can be any of the values found here:
+#   http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/regions/Regions.html
+org.apache.oodt.cas.filemgr.datatransfer.s3.bucket.name=some_bucket_name
+org.apache.oodt.cas.filemgr.datatransfer.s3.region=EU_WEST_1
+org.apache.oodt.cas.filemgr.datatransfer.s3.access.key=s3_access_key
+org.apache.oodt.cas.filemgr.datatransfer.s3.secret.key=s3_secret_key
+org.apache.oodt.cas.filemgr.datatransfer.s3.encrypt=false