diff --git a/crawler/src/main/resources/bin/crawlctl b/crawler/src/main/resources/bin/crawlctl index 12a63f9..2043fb7 100644 --- a/crawler/src/main/resources/bin/crawlctl +++ b/crawler/src/main/resources/bin/crawlctl @@ -16,7 +16,8 @@ ########################################################################## #java -Djava.util.logging.config.file=../etc/logging.properties -classpath "../lib/*" org.apache.oodt.cas.crawl.daemon.CrawlDaemonController $@ -./crawler_launcher --filemgrUrl http://localhost:9000 \ +./crawler/bin/crawler_launcher \ + --filemgrUrl http://localhost:9000 \ --operation --launchMetCrawler \ --clientTransferer org.apache.oodt.cas.filemgr.datatransfer.LocalDataTransferFactory \ --productPath /usr/local/coal-sds-deploy/data/staging \ @@ -26,4 +27,5 @@ --daemonPort 9003 \ --daemonWait 2 \ --successDir /usr/local/coal-sds-deploy/data/archive \ - --actionIds DeleteDataFile + --actionIds DeleteDataFile TriggerPostIngestWorkflow \ + --workflowMgrUrl http://localhost:9001 diff --git a/crawler/src/main/resources/policy/cmd-line-options.xml b/crawler/src/main/resources/policy/cmd-line-options.xml index e6b6811..ca58918 100644 --- a/crawler/src/main/resources/policy/cmd-line-options.xml +++ b/crawler/src/main/resources/policy/cmd-line-options.xml @@ -253,7 +253,7 @@ - UpdateWorkflowStatusToIngest.workflowMgrUrl + TriggerPostIngestWorkflow.workflowMgrUrl diff --git a/distribution/src/main/resources/bin/oodt b/distribution/src/main/resources/bin/oodt index 3cd4e58..5a55abc 100644 --- a/distribution/src/main/resources/bin/oodt +++ b/distribution/src/main/resources/bin/oodt @@ -106,14 +106,14 @@ if [ "$1" = "start" ]; then # Print confirmation messages to the end user # FileManager # check if the PID file exists: - if [ ! -e "../filemgr/run/cas.filemgr.pid" ] + if [ ! -e "$FILEMGR_HOME/run/cas.filemgr.pid" ] then # no pid file was created by the process, it must have failed. echo -e 'Starting OODT File Manager [' $'\e[31m' 'Failed' $'\e[00m' ']' else # pid file exists, check if the associated process is running. - if kill -0 `cat ../filemgr/run/cas.filemgr.pid` > /dev/null 2>&1; + if kill -0 `cat "$FILEMGR_HOME"/run/cas.filemgr.pid` > /dev/null 2>&1; then echo -e 'Starting OODT File Manager [' $'\e[32m' 'Successful' $'\e[00m' ']' else @@ -123,14 +123,14 @@ if [ "$1" = "start" ]; then # ResourceManager # check if the PID file exists: - if [ ! -e "../resmgr/run/cas.resmgr.pid" ] + if [ ! -e "$RESMGR_HOME/resmgr/run/cas.resmgr.pid" ] then # no pid file was created by the process, it must have failed. echo -e 'Starting OODT Resource Manager [' $'\e[31m' 'Failed' $'\e[00m' ']' else # pid file exists, check if the associated process is running. - if kill -0 `cat ../resmgr/run/cas.resmgr.pid` > /dev/null 2>&1; + if kill -0 `cat "$RESMGR_HOME"/run/cas.resmgr.pid` > /dev/null 2>&1; then echo -e 'Starting OODT Resource Manager [' $'\e[32m' 'Successful' $'\e[00m' ']' else @@ -141,14 +141,14 @@ if [ "$1" = "start" ]; then # WorkflowManager # check if the PID file exists: - if [ ! -e "../workflow/run/cas.workflow.pid" ] + if [ ! -e "$WORKFLOW_HOME/run/cas.workflow.pid" ] then # no pid file was created by the process, it must have failed. echo -e 'Starting OODT Workflow Manager [' $'\e[31m' 'Failed' $'\e[00m' ']' else # pid file exists, check if the associated process is running. - if kill -0 `cat ../workflow/run/cas.workflow.pid` > /dev/null 2>&1; + if kill -0 `cat "$WORKFLOW_HOME"/run/cas.workflow.pid` > /dev/null 2>&1; then echo -e 'Starting OODT Workflow Manager [' $'\e[32m' 'Successful' $'\e[00m' ']' else diff --git a/filemgr/src/main/resources/bin/filemgr b/filemgr/src/main/resources/bin/filemgr index cd18385..80eb075 100644 --- a/filemgr/src/main/resources/bin/filemgr +++ b/filemgr/src/main/resources/bin/filemgr @@ -85,7 +85,7 @@ if [ "$1" = "start" ]; then -Djava.util.logging.config.file="$FILEMGR_HOME"/etc/logging.properties \ -Dorg.apache.oodt.cas.filemgr.properties="$FILEMGR_HOME"/etc/filemgr.properties \ -Djava.io.tmpdir="$OODT_TMPDIR" \ - org.apache.oodt.cas.filemgr.system.XmlRpcFileManager \ + org.apache.oodt.cas.filemgr.system.FileManagerServerMain \ --portNum $FILEMGR_PORT 2>&1 & if [ ! -z "$FILEMGR_PID" ]; then diff --git a/filemgr/src/main/resources/bin/filemgr-client b/filemgr/src/main/resources/bin/filemgr-client index dd2106b..1c25b10 100644 --- a/filemgr/src/main/resources/bin/filemgr-client +++ b/filemgr/src/main/resources/bin/filemgr-client @@ -74,4 +74,4 @@ cd "$FILEMGR_HOME"/bin -Djava.util.logging.config.file="$FILEMGR_HOME"/etc/logging.properties \ -Dorg.apache.oodt.cas.cli.action.spring.config=file:"$FILEMGR_HOME"/policy/cmd-line-actions.xml \ -Dorg.apache.oodt.cas.cli.option.spring.config=file:"$FILEMGR_HOME"/policy/cmd-line-options.xml \ - org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient "$@" + org.apache.oodt.cas.filemgr.system.FileManagerClientMain "$@" diff --git a/filemgr/src/main/resources/etc/filemgr.fm-solr-catalog.properties b/filemgr/src/main/resources/etc/filemgr.fm-solr-catalog.properties index b7764a1..92acb6e 100644 --- a/filemgr/src/main/resources/etc/filemgr.fm-solr-catalog.properties +++ b/filemgr/src/main/resources/etc/filemgr.fm-solr-catalog.properties @@ -15,6 +15,9 @@ # Configuration properties for the File Manager +filemgr.server=org.apache.oodt.cas.filemgr.system.rpc.AvroFileManagerServerFactory +filemgr.client=org.apache.oodt.cas.filemgr.system.rpc.AvroFileManagerClientFactory + # repository factory filemgr.repository.factory=org.apache.oodt.cas.filemgr.repository.XMLRepositoryManagerFactory @@ -56,58 +59,13 @@ org.apache.oodt.cas.filemgr.datatransfer.remote.chunkSize=1024 # location of Mime-Type repository org.apache.oodt.cas.filemgr.mime.type.repository=/usr/local/coal-sds-deploy/filemgr/etc/mime-types.xml - -############ data source configuration ################################## -# -# These 3 blocks of config properties can be used to setup a catalog, -# repository manager, and validation layer based on a RDBMS such as -# Oracle, MySQL, PostgreSQL, or any others that support a jdbc connection. -# Just un-comment the following blocks of properties and configure as -# needed. -# -######################################################################### - -# datasource catalog config -#org.apache.oodt.cas.filemgr.catalog.datasource.jdbc.url=some_jdbc_url -#org.apache.oodt.cas.filemgr.catalog.datasource.jdbc.user=user -#org.apache.oodt.cas.filemgr.catalog.datasource.jdbc.pass=pass -#org.apache.oodt.cas.filemgr.catalog.datasource.jdbc.driver=driver.class.name -#org.apache.oodt.cas.filemgr.catalog.datasource.quoteFields=false -#org.apache.oodt.cas.filemgr.catalog.datasource.pageSize=20 -#org.apache.oodt.cas.filemgr.catalog.datasource.cacheUpdateMinutes=5 - -# data source repository manager configuration -#org.apache.oodt.cas.filemgr.repositorymgr.datasource.jdbc.url=some_jdbc_url -#org.apache.oodt.cas.filemgr.repositorymgr.datasource.jdbc.user=user -#org.apache.oodt.cas.filemgr.repositorymgr.datasource.jdbc.pass=pass -#org.apache.oodt.cas.filemgr.repositorymgr.datasource.jdbc.driver=driver.class.name - -# data source validation layer configuration -#org.apache.oodt.cas.filemgr.validation.datasource.jdbc.url=some_jdbc_url -#org.apache.oodt.cas.filemgr.validation.datasource.jdbc.user=user -#org.apache.oodt.cas.filemgr.validation.datasource.jdbc.pass=pass -#org.apache.oodt.cas.filemgr.validation.datasource.jdbc.driver=driver.class.name -#org.apache.oodt.cas.filemgr.validation.datasource.quoteFields=false - - -# tells the file manager system layer to include product instance metadata -# NOTE: here are the expected field mappings -# -# product.getProductId() -> ProductId -# product.getProductName() -> ProductName -# product.getProductStructure() -> ProductStructure -# product.getTransferStatus() -> ProductTransferStatus -# product.getRootRef() -> ProductRootReference - -# for the references returned by product.getProductReferences() the following -# metadata fields will be added (order will be maintained, such that data store -# ref at index 0 will map to orig ref at index 0, etc.) -# -# ProductDataStoreReferences (list of all data store references: -# note already translated into path, not URI) -# ProductOrigReferences (list of all orig references: -# note already translated into path, not URI) -# ProductMimeType (list of all references' mime-types) -# ProductFileSize (list of all references' file sizes) - org.apache.oodt.cas.filemgr.metadata.expandProduct=false + +# Amazon S3 data transfer configuration. +# Region can be any of the values found here: +# http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/regions/Regions.html +org.apache.oodt.cas.filemgr.datatransfer.s3.bucket.name=some_bucket_name +org.apache.oodt.cas.filemgr.datatransfer.s3.region=EU_WEST_1 +org.apache.oodt.cas.filemgr.datatransfer.s3.access.key=s3_access_key +org.apache.oodt.cas.filemgr.datatransfer.s3.secret.key=s3_secret_key +org.apache.oodt.cas.filemgr.datatransfer.s3.encrypt=false diff --git a/filemgr/src/main/resources/etc/filemgr.properties b/filemgr/src/main/resources/etc/filemgr.properties index c1ca47d..565d08c 100755 --- a/filemgr/src/main/resources/etc/filemgr.properties +++ b/filemgr/src/main/resources/etc/filemgr.properties @@ -1,36 +1,28 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # Configuration properties for the File Manager -# repository factory -filemgr.repository.factory=org.apache.oodt.cas.filemgr.repository.XMLRepositoryManagerFactory - -# RPC implementation, options include the deprecated XMLRPC or the preferred AvroRPC -# uncomment the Avro implementations to use AvroRPC -#filemgr.server=org.apache.oodt.cas.filemgr.system.rpc.XmlRpcFileManagerServerFactory -#filemgr.client=org.apache.oodt.cas.filemgr.system.rpc.XmlRpcFileManagerClientFactory filemgr.server=org.apache.oodt.cas.filemgr.system.rpc.AvroFileManagerServerFactory filemgr.client=org.apache.oodt.cas.filemgr.system.rpc.AvroFileManagerClientFactory -# Solr catalog factory -filemgr.catalog.factory=org.apache.oodt.cas.filemgr.catalog.solr.SolrCatalogFactory -org.apache.oodt.cas.filemgr.catalog.solr.url=http://localhost:8080/solr/oodt-fm +# repository factory +filemgr.repository.factory=org.apache.oodt.cas.filemgr.repository.XMLRepositoryManagerFactory -# Lucene catalog factory -#filemgr.catalog.factory=org.apache.oodt.cas.filemgr.catalog.LuceneCatalogFactory +# catalog factory +filemgr.catalog.factory=org.apache.oodt.cas.filemgr.catalog.LuceneCatalogFactory # data transfer factory filemgr.datatransfer.factory=org.apache.oodt.cas.filemgr.datatransfer.LocalDataTransferFactory @@ -44,80 +36,128 @@ org.apache.oodt.cas.filemgr.system.xmlrpc.requestTimeout.minutes=60 #org.apache.oodt.cas.filemgr.system.xmlrpc.connection.retries=0 #org.apache.oodt.cas.filemgr.system.xmlrpc.connection.retry.interval.seconds=3 +# data source catalog configuration +org.apache.oodt.cas.filemgr.catalog.datasource.jdbc.url=some_jdbc_url +org.apache.oodt.cas.filemgr.catalog.datasource.jdbc.user=user +org.apache.oodt.cas.filemgr.catalog.datasource.jdbc.pass=pass +org.apache.oodt.cas.filemgr.catalog.datasource.jdbc.driver=driver.class.name +org.apache.oodt.cas.filemgr.catalog.datasource.quoteFields=false +org.apache.oodt.cas.filemgr.catalog.datasource.pageSize=20 +org.apache.oodt.cas.filemgr.catalog.datasource.cacheUpdateMinutes=5 +org.apache.oodt.cas.filemgr.catalog.datasource.orderedValues=false +# set the following property to 'true' to allow dynamic metadata fields, +# effectively bypassing the validation layer. +# by default the property is false +#org.apache.oodt.cas.filemgr.catalog.datasource.lenientFields=true +# set the following property to true to enable the column "product_id" +# in table "products" to be of type string +#org.apache.oodt.cas.filemgr.catalog.datasource.productId.string=false + + # mapped data source catalog configuration -#org.apache.oodt.cas.filemgr.catalog.mappeddatasource.mapFile=/path/to/ops.catalog.typemap.properties +org.apache.oodt.cas.filemgr.catalog.mappeddatasource.mapFile=/path/to/ops.catalog.typemap.properties + +# science data catalog configuration +org.apache.oodt.cas.filemgr.catalog.science.jdbc.url=some_jdbc_url +org.apache.oodt.cas.filemgr.catalog.science.jdbc.user=user +org.apache.oodt.cas.filemgr.catalog.science.jdbc.pass=pass +org.apache.oodt.cas.filemgr.catalog.science.jdbc.driver=driver.class.name # lucene catalog configuration -org.apache.oodt.cas.filemgr.catalog.lucene.idxPath=/usr/local/coal-sds-deploy/filemgr/data/catalog +org.apache.oodt.cas.filemgr.catalog.lucene.idxPath=/tmp/test org.apache.oodt.cas.filemgr.catalog.lucene.pageSize=20 org.apache.oodt.cas.filemgr.catalog.lucene.commitLockTimeout.seconds=60 org.apache.oodt.cas.filemgr.catalog.lucene.writeLockTimeout.seconds=60 org.apache.oodt.cas.filemgr.catalog.lucene.mergeFactor=20 -# XML repository manager configuration -org.apache.oodt.cas.filemgr.repositorymgr.dirs=file:///usr/local/coal-sds-deploy/filemgr/policy/oodt +# solr catalog configuration +org.apache.oodt.cas.filemgr.catalog.solr.url=http://localhost:8983/solr +#org.apache.oodt.cas.filemgr.catalog.solr.url=http://localhost:8080/solr +org.apache.oodt.cas.filemgr.catalog.solr.productSerializer=org.apache.oodt.cas.filemgr.catalog.solr.DefaultProductSerializer +org.apache.oodt.cas.filemgr.catalog.solr.productIdGenerator=org.apache.oodt.cas.filemgr.catalog.solr.UUIDProductIdGenerator +#org.apache.oodt.cas.filemgr.catalog.solr.productIdGenerator=org.apache.oodt.cas.filemgr.catalog.solr.NameProductIdGenerator -# XML validation layer configuration -org.apache.oodt.cas.filemgr.validation.dirs=file:///usr/local/coal-sds-deploy/filemgr/policy/oodt -# remote data transfer configuration -org.apache.oodt.cas.filemgr.datatransfer.remote.chunkSize=1024 - -# location of Mime-Type repository -org.apache.oodt.cas.filemgr.mime.type.repository=/usr/local/coal-sds-deploy/filemgr/etc/mime-types.xml +# data source repository manager configuration +org.apache.oodt.cas.filemgr.repositorymgr.datasource.jdbc.url=some_jdbc_url +org.apache.oodt.cas.filemgr.repositorymgr.datasource.jdbc.user=user +org.apache.oodt.cas.filemgr.repositorymgr.datasource.jdbc.pass=pass +org.apache.oodt.cas.filemgr.repositorymgr.datasource.jdbc.driver=driver.class.name +# science data repository manager configuration +org.apache.oodt.cas.filemgr.repositorymgr.science.jdbc.url=some_jdbc_url +org.apache.oodt.cas.filemgr.repositorymgr.science.jdbc.user=user +org.apache.oodt.cas.filemgr.repositorymgr.science.jdbc.pass=pass +org.apache.oodt.cas.filemgr.repositorymgr.science.jdbc.driver=driver.class.name -############ data source configuration ################################## -# -# These 3 blocks of config properties can be used to setup a catalog, -# repository manager, and validation layer based on a RDBMS such as -# Oracle, MySQL, PostgreSQL, or any others that support a jdbc connection. -# Just un-comment the following blocks of properties and configure as -# needed. -# -######################################################################### +# XML repository manager configuration +org.apache.oodt.cas.filemgr.repositorymgr.dirs=file:///dir1,file:///dir2 +# uncomment the following line and set the value to true to recursively parse policy directories +#org.apache.oodt.cas.filemgr.repositorymgr.dirs.recursive=false -# datasource catalog config -#org.apache.oodt.cas.filemgr.catalog.datasource.jdbc.url=some_jdbc_url -#org.apache.oodt.cas.filemgr.catalog.datasource.jdbc.user=user -#org.apache.oodt.cas.filemgr.catalog.datasource.jdbc.pass=pass -#org.apache.oodt.cas.filemgr.catalog.datasource.jdbc.driver=driver.class.name -#org.apache.oodt.cas.filemgr.catalog.datasource.quoteFields=false -#org.apache.oodt.cas.filemgr.catalog.datasource.pageSize=20 -#org.apache.oodt.cas.filemgr.catalog.datasource.cacheUpdateMinutes=5 +# XML validation layer configuration +org.apache.oodt.cas.filemgr.validation.dirs=file:///dir1,file:///dir2 +# uncomment the following line and set the value to true to recursively parse validation directories +#org.apache.oodt.cas.filemgr.validation.dirs.recursive=false -# data source repository manager configuration -#org.apache.oodt.cas.filemgr.repositorymgr.datasource.jdbc.url=some_jdbc_url -#org.apache.oodt.cas.filemgr.repositorymgr.datasource.jdbc.user=user -#org.apache.oodt.cas.filemgr.repositorymgr.datasource.jdbc.pass=pass -#org.apache.oodt.cas.filemgr.repositorymgr.datasource.jdbc.driver=driver.class.name +# set the following property to 'true' to allow dynamic metadata fields, +# effectively bypassing the validation layer. +# by default the property is false +#org.apache.oodt.cas.filemgr.catalog.lucene.lenientFields=true # data source validation layer configuration -#org.apache.oodt.cas.filemgr.validation.datasource.jdbc.url=some_jdbc_url -#org.apache.oodt.cas.filemgr.validation.datasource.jdbc.user=user -#org.apache.oodt.cas.filemgr.validation.datasource.jdbc.pass=pass -#org.apache.oodt.cas.filemgr.validation.datasource.jdbc.driver=driver.class.name -#org.apache.oodt.cas.filemgr.validation.datasource.quoteFields=false +org.apache.oodt.cas.filemgr.validation.datasource.jdbc.url=some_jdbc_url +org.apache.oodt.cas.filemgr.validation.datasource.jdbc.user=user +org.apache.oodt.cas.filemgr.validation.datasource.jdbc.pass=pass +org.apache.oodt.cas.filemgr.validation.datasource.jdbc.driver=driver.class.name +org.apache.oodt.cas.filemgr.validation.datasource.quoteFields=false + +# science data validation layer configuration +org.apache.oodt.cas.filemgr.validation.science.jdbc.url=some_jdbc_url +org.apache.oodt.cas.filemgr.validation.science.jdbc.user=user +org.apache.oodt.cas.filemgr.validation.science.jdbc.pass=pass +org.apache.oodt.cas.filemgr.validation.science.jdbc.driver=driver.class.name +# remote data transfer configuration +org.apache.oodt.cas.filemgr.datatransfer.remote.chunkSize=1024 + +# Amazon S3 data transfer configuration. +# Region can be any of the values found here: +# http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/regions/Regions.html +org.apache.oodt.cas.filemgr.datatransfer.s3.bucket.name=some_bucket_name +org.apache.oodt.cas.filemgr.datatransfer.s3.region=EU_WEST_1 +org.apache.oodt.cas.filemgr.datatransfer.s3.access.key=s3_access_key +org.apache.oodt.cas.filemgr.datatransfer.s3.secret.key=s3_secret_key +org.apache.oodt.cas.filemgr.datatransfer.s3.encrypt=false + +# location of Mime-Type repository +org.apache.oodt.cas.filemgr.mime.type.repository=/path/to/mime-types.xml # tells the file manager system layer to include product instance metadata # NOTE: here are the expected field mappings # -# product.getProductId() -> ProductId -# product.getProductName() -> ProductName -# product.getProductStructure() -> ProductStructure -# product.getTransferStatus() -> ProductTransferStatus -# product.getRootRef() -> ProductRootReference - -# for the references returned by product.getProductReferences() the following -# metadata fields will be added (order will be maintained, such that data store +# product.getProductId() -> ProductId +# product.getProductName() -> ProductName +# product.getProductStructure() -> ProductStructure +# product.getTransferStatus() -> ProductTransferStatus +# product.getRootRef() -> ProductRootReference + +# for the references returned by product.getProductReferences() the following +# metadata fields will be added (order will be maintained, such that data store # ref at index 0 will map to orig ref at index 0, etc.) # -# ProductDataStoreReferences (list of all data store references: -# note already translated into path, not URI) -# ProductOrigReferences (list of all orig references: -# note already translated into path, not URI) -# ProductMimeType (list of all references' mime-types) -# ProductFileSize (list of all references' file sizes) +# ProductDataStoreReferences (list of all data store references: +# note already translated into path, not URI) +# ProductOrigReferences (list of all orig references: +# note already translated into path, not URI) +# ProductMimeType (list of all references' mime-types) +# ProductFileSize (list of all references' file sizes) org.apache.oodt.cas.filemgr.metadata.expandProduct=false + +# ConfigurableMetadataBasedVersioning configuration +#org.apache.oodt.cas.filemgr.versioning.configuration.all_product_types=[Year]/[Month]/[Filename] +#org.apache.oodt.cas.filemgr.versioning.configuration.=[Year]/[Month]/[Filename] + +# Enable to use only server-side versioning. +#org.apache.oodt.cas.filemgr.serverside.versioning=true \ No newline at end of file diff --git a/pcs/src/main/resources/policy/pcs-crawlers.xml b/pcs/src/main/resources/policy/pcs-crawlers.xml index d1bfee2..af391c5 100644 --- a/pcs/src/main/resources/policy/pcs-crawlers.xml +++ b/pcs/src/main/resources/policy/pcs-crawlers.xml @@ -44,7 +44,7 @@ This file is used in the PCSHealthMonitor tool. monitor is running on. --> - 9020 + 9003 - \ No newline at end of file + diff --git a/pge/src/main/resources/policy/pycoal-environment-pge-config.xml b/pge/src/main/resources/policy/pycoal-environment-pge-config.xml new file mode 100644 index 0000000..8e34ea0 --- /dev/null +++ b/pge/src/main/resources/policy/pycoal-environment-pge-config.xml @@ -0,0 +1,43 @@ + + + + + + + [PGE_ROOT]/bin/example-script.sh > /tmp/HelloWorldPGE-[ProductionDateTime].output.txt + + + + + + + + + + + + + + + + + + + + + + + diff --git a/pge/src/main/resources/policy/pycoal-mineral-pge-config.xml b/pge/src/main/resources/policy/pycoal-mineral-pge-config.xml new file mode 100644 index 0000000..5aba9f2 --- /dev/null +++ b/pge/src/main/resources/policy/pycoal-mineral-pge-config.xml @@ -0,0 +1,42 @@ + + + + + + + pycoal-mineral -i + + + + + + + + + + diff --git a/pge/src/main/resources/policy/pycoal-mining-pge-config.xml b/pge/src/main/resources/policy/pycoal-mining-pge-config.xml new file mode 100644 index 0000000..8e34ea0 --- /dev/null +++ b/pge/src/main/resources/policy/pycoal-mining-pge-config.xml @@ -0,0 +1,43 @@ + + + + + + + [PGE_ROOT]/bin/example-script.sh > /tmp/HelloWorldPGE-[ProductionDateTime].output.txt + + + + + + + + + + + + + + + + + + + + + + + diff --git a/pom.xml b/pom.xml index a00bbe4..5fd622c 100644 --- a/pom.xml +++ b/pom.xml @@ -27,8 +27,8 @@ UTF-8 UTF-8 - 1.2.5 - 1.2.5 + 1.9-SNAPSHOT + 1.9-SNAPSHOT 2.0.0-SNAPSHOT 2.9.1 diff --git a/resmgr/src/main/resources/bin/resmgr b/resmgr/src/main/resources/bin/resmgr index 6737e0c..438f0e8 100644 --- a/resmgr/src/main/resources/bin/resmgr +++ b/resmgr/src/main/resources/bin/resmgr @@ -91,7 +91,7 @@ if [ "$1" = "start" ]; then -Djava.util.logging.config.file="$RESMGR_HOME"/etc/logging.properties \ -Dorg.apache.oodt.cas.resource.properties="$RESMGR_HOME"/etc/resource.properties \ -Djava.io.tmpdir="$OODT_TMPDIR" \ - org.apache.oodt.cas.resource.system.XmlRpcResourceManager \ + org.apache.oodt.cas.resource.system.ResourceManagerMain \ --portNum "$RESMGR_PORT" 2>&1 & if [ ! -z "$RESMGR_PID" ]; then diff --git a/resmgr/src/main/resources/bin/resmgr-client b/resmgr/src/main/resources/bin/resmgr-client index 3f6758c..3fd7e45 100644 --- a/resmgr/src/main/resources/bin/resmgr-client +++ b/resmgr/src/main/resources/bin/resmgr-client @@ -31,4 +31,4 @@ $JAVA_HOME/bin/java \ -Djava.util.logging.config.file=../etc/logging.properties \ -Dorg.apache.oodt.cas.cli.action.spring.config=../policy/cmd-line-actions.xml \ -Dorg.apache.oodt.cas.cli.option.spring.config=../policy/cmd-line-options.xml \ - org.apache.oodt.cas.resource.system.XmlRpcResourceManagerClient $* + org.apache.oodt.cas.resource.system.ResourceManagerClientMain $* diff --git a/resmgr/src/main/resources/etc/resource.properties b/resmgr/src/main/resources/etc/resource.properties index 5520c7b..537012b 100644 --- a/resmgr/src/main/resources/etc/resource.properties +++ b/resmgr/src/main/resources/etc/resource.properties @@ -16,6 +16,9 @@ # # Properties required to configure the Resource Manager +resmgr.manager=org.apache.oodt.cas.resource.system.AvroRpcResourceManager +resmgr.manager.client=org.apache.oodt.cas.resource.system.AvroRpcResourceManagerClient + # resource batchmgr factory resource.batchmgr.factory = org.apache.oodt.cas.resource.batchmgr.XmlRpcBatchMgrFactory diff --git a/startup.sh b/startup.sh new file mode 100644 index 0000000..7a45e64 --- /dev/null +++ b/startup.sh @@ -0,0 +1,21 @@ +#!/bin/bash +#Licensed to the Apache Software Foundation (ASF) under one or more contributor +#license agreements. See the NOTICE.txt file distributed with this work for +#additional information regarding copyright ownership. The ASF licenses this +#file to you under the Apache License, Version 2.0 (the "License"); you may not +#use this file except in compliance with the License. You may obtain a copy of +#the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +#Unless required by applicable law or agreed to in writing, software +#distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +#WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +#License for the specific language governing permissions and limitations under +#the License. + +mkdir /usr/local/coal-sds-deploy +mvn clean package -Pfm-solr-catalog -DskipTests +tar -zxvf distribution/target/coal-sds-distribution-*-bin.tar.gz -C /usr/local/coal-sds-deploy/ +pip install pycoal +/usr/local/coal-sds-deploy/bin/oodt start \ No newline at end of file diff --git a/workflow/pom.xml b/workflow/pom.xml index e62b412..b1f25bb 100644 --- a/workflow/pom.xml +++ b/workflow/pom.xml @@ -60,12 +60,12 @@ ${project.parent.version} jar runtime - + org.apache.oodt @@ -82,16 +82,16 @@ org.apache.oodt cas-pge ${oodt.pge.version} - + xerces diff --git a/workflow/src/main/resources/bin/wmgr b/workflow/src/main/resources/bin/wmgr index 76cbccb..7719746 100644 --- a/workflow/src/main/resources/bin/wmgr +++ b/workflow/src/main/resources/bin/wmgr @@ -86,7 +86,7 @@ if [ "$1" = "start" ]; then -Djava.io.tmpdir="$OODT_TMPDIR" \ -Dorg.apache.oodt.cas.pge.task.metkeys.legacyMode="true" \ -Dorg.apache.oodt.cas.pge.task.status.legacyMode="true" \ - org.apache.oodt.cas.workflow.system.XmlRpcWorkflowManager \ + org.apache.oodt.cas.workflow.system.WorkflowManagerStarter \ --portNum "$WORKFLOW_PORT" 2>&1 & if [ ! -z "$WORKFLOW_PID" ]; then diff --git a/workflow/src/main/resources/bin/wmgr-client b/workflow/src/main/resources/bin/wmgr-client index 4807516..16ce8a4 100644 --- a/workflow/src/main/resources/bin/wmgr-client +++ b/workflow/src/main/resources/bin/wmgr-client @@ -72,4 +72,4 @@ cd "$WORKFLOW_HOME"/bin -Djava.util.logging.config.file="$WORKFLOW_HOME"/etc/logging.properties \ -Dorg.apache.oodt.cas.cli.action.spring.config=file:"$WORKFLOW_HOME"/policy/cmd-line-actions.xml \ -Dorg.apache.oodt.cas.cli.option.spring.config=file:"$WORKFLOW_HOME"/policy/cmd-line-options.xml \ - org.apache.oodt.cas.workflow.system.XmlRpcWorkflowManagerClient "$@" + org.apache.oodt.cas.workflow.system.WorkflowManagerClientStarter "$@" diff --git a/workflow/src/main/resources/etc/workflow.properties b/workflow/src/main/resources/etc/workflow.properties index dbb0dba..9a4ef1e 100644 --- a/workflow/src/main/resources/etc/workflow.properties +++ b/workflow/src/main/resources/etc/workflow.properties @@ -15,6 +15,9 @@ # # Properties required to configure the Workflow Manager +workflow.server.factory = org.apache.oodt.cas.workflow.system.rpc.AvroRpcWorkflowManagerFactory +workflow.client.factory = org.apache.oodt.cas.workflow.system.rpc.AvroRpcWorkflowManagerFactory + # workflow repository factory workflow.repo.factory = org.apache.oodt.cas.workflow.repository.XMLWorkflowRepositoryFactory @@ -23,6 +26,7 @@ workflow.engine.factory = org.apache.oodt.cas.workflow.engine.ThreadPoolWorkflow # workflow instance repository factory workflow.engine.instanceRep.factory = org.apache.oodt.cas.workflow.instrepo.LuceneWorkflowInstanceRepositoryFactory +#workflow.engine.instanceRep.factory = org.apache.oodt.cas.workflow.instrepo.MemoryWorkflowInstanceRepositoryFactory # thread pool workflow engine properties org.apache.oodt.cas.workflow.engine.queueSize= @@ -33,18 +37,27 @@ org.apache.oodt.cas.workflow.engine.unlimitedQueue=true org.apache.oodt.cas.workflow.engine.preConditionWaitTime=10 # set this if you want the workflow manager to submit jobs through the resource mgr -org.apache.oodt.cas.workflow.engine.resourcemgr.url=http://localhost:9002 +org.apache.oodt.cas.workflow.engine.resourcemgr.url= # if you use the resource mgr submission, you can specify how many seconds the # workflow manager should wait inbetween checking to see if a job is complete org.apache.oodt.cas.workflow.engine.resourcemgr.pollingWaitTime=10 +# wengine properties +# define workflow prioritizer class to use for sorting workflow tasks +org.apache.oodt.cas.workflow.wengine.prioritizer=org.apache.oodt.cas.workflow.structs.FILOPrioritySorter +# the default amount of time (in seconds) that the task querier waits before dispositioning processors +org.apache.oodt.cas.workflow.wengine.taskquerier.waitSeconds=2 + +# the maximum number of threads to be used by the asynchronous engine runner +org.apache.oodt.cas.workflow.wengine.asynchronous.runner.num.threads=25 + # workflow instance repository general properties # default page size to page through WorkflowInstances with org.apache.oodt.cas.workflow.instanceRep.pageSize=20 # lucene workflow instance repository properties -org.apache.oodt.cas.workflow.instanceRep.lucene.idxPath=[OODT_HOME]/data/workflow +org.apache.oodt.cas.workflow.instanceRep.lucene.idxPath=/usr/local/coal-sds-deploy/data/workflow # data source workflow instance repository properties # org.apache.oodt.cas.workflow.instanceRep.datasource.jdbc.url=jdbc:url @@ -54,7 +67,7 @@ org.apache.oodt.cas.workflow.instanceRep.lucene.idxPath=[OODT_HOME]/data/workflo # org.apache.oodt.cas.workflow.instanceRep.datasource.quoteFields=false # XML workflow repository properties -org.apache.oodt.cas.workflow.repo.dirs=file://[WORKFLOW_HOME]/policy +org.apache.oodt.cas.workflow.repo.dirs=file:///usr/local/coal-sds-deploy/workflow/policy # wengine-style packaged workflow repo properties #org.apache.oodt.cas.workflow.wengine.packagedRepo.dir.path = /path/to/wengine/workflow/files @@ -66,5 +79,5 @@ org.apache.oodt.cas.workflow.repo.dirs=file://[WORKFLOW_HOME]/policy # org.apache.oodt.cas.workflow.repo.datasource.jdbc.driver=your.jdbc.Driver # Spring command line option and action store properties -org.apache.oodt.cas.cli.action.spring.config=src/main/resources/cmd-line-actions.xml -org.apache.oodt.cas.cli.option.spring.config=src/main/resources/cmd-line-options.xml +org.apache.oodt.cas.cli.action.spring.config=file:///usr/local/coal-sds-deploy/workflow/policy/cmd-line-actions.xml +org.apache.oodt.cas.cli.option.spring.config=file:///usr/local/coal-sds-deploy/workflow/policy/cmd-line-options.xml diff --git a/workflow/src/main/resources/policy/events.xml b/workflow/src/main/resources/policy/events.xml index 84a02ae..fc04963 100644 --- a/workflow/src/main/resources/policy/events.xml +++ b/workflow/src/main/resources/policy/events.xml @@ -15,7 +15,7 @@ limitations under the License. --> - + + + diff --git a/workflow/src/main/resources/policy/pycoal.pge.workflow.xml b/workflow/src/main/resources/policy/pycoal.pge.workflow.xml new file mode 100644 index 0000000..5128566 --- /dev/null +++ b/workflow/src/main/resources/policy/pycoal.pge.workflow.xml @@ -0,0 +1,23 @@ + + + + + + + + \ No newline at end of file diff --git a/workflow/src/main/resources/policy/tasks.xml b/workflow/src/main/resources/policy/tasks.xml index 86aa67e..9052609 100644 --- a/workflow/src/main/resources/policy/tasks.xml +++ b/workflow/src/main/resources/policy/tasks.xml @@ -15,7 +15,72 @@ limitations under the License. --> - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/workflow/src/main/resources/policy/workflow-instance-met.xml b/workflow/src/main/resources/policy/workflow-instance-met.xml index 2905a46..b94485a 100644 --- a/workflow/src/main/resources/policy/workflow-instance-met.xml +++ b/workflow/src/main/resources/policy/workflow-instance-met.xml @@ -17,7 +17,7 @@ the License. --> - + + + + +