Hc 158 refactor (#13) (#14) (#15)

* Hc 158 refactor (#13)

* refactored retry job

added gitignore

* purge.py refactor, TODO: not sure what this linne of code means query_obj["query"]

* refactored purge job to use context insntead of positional and cleaned up code

* refactored retry.py

removed retry.sh, moved job_spec destinations to context
cleaned up code

* refactored notify by email job

added more logging to purge job

* deleting by index and id in retry.py, fixed bugs in retry.py

refactored job_spec for revoke json to be context not positional

* added setup.shh and hysds_commons in DockerFile

* refactored reprioritize jobs PGE

* using specfic branch for hysds_commons in setup.sh

* fixed pip command in dockerfile

* fixed pip command in dockerfile

* fixed pip command in dockerfile

* fixed Dockerfile copy command

* fixed Dockerfile copy command

* fixed Dockerfile copy command

* fixed Dockerfile copy command

* fixed Dockerfile copy command

* fixed Dockerfile copy command

* fixed Dockerfile copy command

* fixed Dockerfile copy command

* fixed Dockerfile copy command

* fixed Dockerfile copy command

* fixed command in job_spec from .sh to .py

* fixed command job_spec in purge

* fixed command job_spec in purge

* fixed command job_spec in purge

* fixed command job_spec in purge

* edited job_spec commands for retry and purge

* removed sudo from Dockerfile to maybe get retry job to work properly

error: "ImportError: /lib64/libstdc++.so.6: version `CXXABI_1.3.8' not found"

* . ~/.bash_profiles to activate conda virtual environment

* . ~/.bash_profiles to activate conda virtual environment

* . ~/.bash_profiles to activate conda virtual environment

* . ~/.bash_profiles to activate conda virtual environment

* . ~/.bash_profiles to activate conda virtual environment

* . ~/.bash_profiles to activate conda virtual environment

* wrapped retry.py in a shell script instead

* wrapped retry.py in a shell script instead

* fixed command in job_specs

* changed branch to develop-es7 in setup.sh

* removed setup.sh

.gitignore

@@ -0,0 +1,160 @@
+*.pyc
+build
+*.egg-info
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+.idea
+.idea/
+
+# General
+.DS_Store
+.AppleDouble
+.LSOverride
+
+# Icon must end with two \r
+Icon
+
+
+# Thumbnails
+._*
+
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+.com.apple.timemachine.donotpresent
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk

docker/Dockerfile

@@ -5,10 +5,21 @@ LABEL description="Lightweight System Jobs"
 
 # provision lightweight-jobs PGE
 USER ops
-COPY . /home/ops/verdi/ops/lightweight-jobs
+COPY . /home/ops/lightweight-jobs
+
+RUN sudo mv /home/ops/verdi/ops/lightweight-jobs /home/ops/verdi/ops/lightweight-jobs.orig
+
+RUN unlink /home/ops/verdi/ops/hysds_commons
+RUN sudo mv /home/ops/verdi/ops/hysds_commons-0.2.4 /home/ops/verdi/ops/hysds_commons-0.2.4.orig
+RUN ln -s /home/ops/verdi/ops/hysds_commons /home/ops/verdi/ops/hysds_commons-0.2.4
+
+RUN sudo mv /home/ops/lightweight-jobs/hysds_commons /home/ops/verdi/ops/
+
+RUN ~/verdi/bin/pip install 'elasticsearch>=7.0.0,<8.0.0'
 
 # set entrypoint
 ENTRYPOINT ["/entrypoint-pge-with-stats.sh"]
 
 WORKDIR /home/ops
+
 CMD ["/bin/bash", "--login"]

docker/hysds-io.json.lw-mozart-reprioritize

@@ -1,9 +1,9 @@
 {
-  "label":"Reprioritize Jobs/Tasks",
-  "component":"figaro",
-  "submission_type":"individual",
+  "label": "Reprioritize Jobs/Tasks",
+  "component": "figaro",
+  "submission_type": "individual",
   "enable_dedup": false,
-  "params" : [
+  "params": [
     {
       "name": "retry_job_id",
       "type": "text",
@@ -23,10 +23,10 @@
     },
     {
       "name": "new_job_priority",
-      "from": "submitter",
       "type": "text",
+      "from": "submitter",
       "default": "0",
       "lambda": "lambda x: int(x)"
     }
   ]
-}
+}

docker/hysds-io.json.lw-mozart-retry

@@ -1,9 +1,9 @@
 {
-  "label":"Retry Jobs/Tasks",
-  "component":"figaro",
-  "submission_type":"individual",
+  "label": "Retry Jobs/Tasks",
+  "component": "figaro",
+  "submission_type": "individual",
   "enable_dedup": false,
-  "params" : [
+  "params": [
     {
       "name": "retry_job_id",
       "type": "text",
@@ -25,9 +25,13 @@
       "name": "job_priority_increment",
       "type": "enum",
       "from": "submitter",
-      "enumerables": ["-1","0","+1"],
+      "enumerables": [
+        "-1",
+        "0",
+        "+1"
+      ],
       "default": "0",
       "lambda": "lambda x: int(x)"
     }
   ]
-}
+}

docker/hysds-io.json.lw-mozart-revoke

@@ -1,8 +1,8 @@
 {
-  "label":"_Revoke / _Stop Job",
-  "component":"figaro",
-  "submission_type":"individual",
-  "params" : [
+  "label": "_Revoke / _Stop Job",
+  "component": "figaro",
+  "submission_type": "individual",
+  "params": [
     {
       "name": "query",
       "type": "text",
@@ -21,4 +21,4 @@
       "value": "revoke"
     }
   ]
-}
+}

docker/hysds-io.json.lw-tosca-purge

@@ -1,8 +1,8 @@
 {
-  "label":"Purge datasets",
-  "component":"tosca",
-  "submission_type":"individual",
-  "params" : [
+  "label": "Purge datasets",
+  "component": "tosca",
+  "submission_type": "individual",
+  "params": [
     {
       "name": "query",
       "type": "text",
@@ -21,4 +21,4 @@
       "value": "purge"
     }
   ]
-}
+}

docker/job-spec.json.lw-mozart-notify-by-email

@@ -7,23 +7,23 @@
   "params" : [
     {
       "name": "id",
-      "destination": "positional"
+      "destination": "context"
     },
     {
       "name": "url",
-      "destination": "positional"
+      "destination": "context"
     },
     {
       "name": "emails",
-      "destination": "positional"
+      "destination": "context"
     },
     {
       "name": "name",
-      "destination": "positional"
+      "destination": "context"
     },
     {
       "name": "component",
-      "destination": "positional"
+      "destination": "context"
     }
 	]
 }

docker/job-spec.json.lw-mozart-purge

@@ -1,21 +1,21 @@
 {
   "required_queues":["system-jobs-queue"],
-  "command":"/home/ops/verdi/ops/lightweight-jobs/purge.sh",
+  "command":"python /home/ops/lightweight-jobs/purge.py",
   "disk_usage":"3GB",
   "soft_time_limit": 86400,
   "time_limit": 86700,
   "params" : [
     {
       "name": "query",
-      "destination": "positional"
+      "destination": "context"
     },
     {
       "name": "component",
-      "destination": "positional"
+      "destination": "context"
     },
     {
       "name": "operation",
-      "destination": "positional"
+      "destination": "context"
     }
    ]
 }

docker/job-spec.json.lw-mozart-reprioritize

@@ -1,17 +1,19 @@
 {
-  "required_queues":["system-jobs-queue"],
-  "command":"/home/ops/verdi/ops/lightweight-jobs/retry.sh",
-  "disk_usage":"3GB",
+  "required_queues": [
+    "system-jobs-queue"
+  ],
+  "command": "/home/ops/lightweight-jobs/retry.sh",
+  "disk_usage": "3GB",
   "soft_time_limit": 86400,
   "time_limit": 86700,
-  "params" : [
+  "params": [
     {
       "name": "retry_job_id",
       "destination": "context"
     },
     {
       "name": "type",
-      "destination": "positional"
+      "destination": "context"
     },
     {
       "name": "retry_count_max",
@@ -22,4 +24,4 @@
       "destination": "context"
     }
   ]
-}
+}