Initial cleanup and restructuring for beam 2.35

Dockerfile-scheduler

@@ -0,0 +1,16 @@
+FROM gcr.io/world-fishing-827/github.com/globalfishingwatch/gfw-pipeline:latest-python3.7
+
+# Setup scheduler-specific dependencies
+COPY ./requirements-scheduler.txt ./
+RUN pip install -r requirements-scheduler.txt
+
+# Setup worker dependencies
+COPY ./requirements-worker.txt ./
+RUN pip install -r requirements-worker.txt
+
+# Setup local packages
+COPY . /opt/project
+RUN pip install -e .
+
+# Setup the entrypoint for quickly executing the pipelines
+ENTRYPOINT ["scripts/run.sh"]

Dockerfile-worker

@@ -0,0 +1,6 @@
+FROM apache/beam_python3.7_sdk:2.35.0
+
+# Setup local application dependencies
+COPY ./requirements-worker.txt ./
+RUN pip install -r requirements-worker.txt
+

LICENSE

@@ -172,30 +172,3 @@
       defend, and hold each Contributor harmless for any liability
       incurred by, or claims asserted against, such Contributor by reason
       of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.

MANIFEST.in

@@ -3,5 +3,4 @@ include CHANGES.md
 include MANIFEST.in
 include README.md
 include setup.py
-include requirements.txt
-recursive-include  assets *
+recursive-include  assets *

cloudbuild.yaml

@@ -1,41 +1,69 @@
 steps:
-
 - name: 'gcr.io/cloud-builders/docker'
-  id: build
+  id: build-scheduler
   args: [
     'build',
-    '-t',
-    'gcr.io/$PROJECT_ID/github-globalfishingwatch-pipe-segment:latest',
-    '-t',
-    'gcr.io/$PROJECT_ID/github-globalfishingwatch-pipe-segment:$TAG_NAME',
-    '.'
+    '-t', '${_SCHEDULER_IMAGE_NAME}:${TAG_NAME}',
+    '-t', '${_SCHEDULER_IMAGE_NAME}:latest',
+    '-f', 'Dockerfile-scheduler',
+    '.',
   ]
 
 - name: 'gcr.io/cloud-builders/docker'
   id: test
   args: [
     'run',
     '--rm',
-    '--entrypoint',
-    'py.test',
-    'gcr.io/$PROJECT_ID/github-globalfishingwatch-pipe-segment:latest'
+    '--entrypoint', 'py.test',
+    '${_SCHEDULER_IMAGE_NAME}:latest',
   ]
-  waitFor: [ 'build' ]
 
 - name: 'gcr.io/cloud-builders/docker'
-  id: push_latest
+  id: push-scheduler-tag
   args: [
     'push',
-    'gcr.io/$PROJECT_ID/github-globalfishingwatch-pipe-segment:latest'
+    '${_SCHEDULER_IMAGE_NAME}:${TAG_NAME}'
   ]
-  waitFor: [ 'build', 'test' ]
 
 - name: 'gcr.io/cloud-builders/docker'
-  id: push_tagged
+  id: push-scheduler-latest
   args: [
     'push',
-    'gcr.io/$PROJECT_ID/github-globalfishingwatch-pipe-segment:$TAG_NAME'
+    '${_SCHEDULER_IMAGE_NAME}:latest'
+  ]
+
+- name: 'gcr.io/cloud-builders/docker'
+  id: build-worker
+  waitFor: ['test']
+  args: [
+    'build',
+    '-t', '${_WORKER_IMAGE_NAME}:${TAG_NAME}',
+    '-t', '${_WORKER_IMAGE_NAME}:latest',
+    '-f', 'Dockerfile-worker',
+    '.',
+  ]
+
+- name: 'gcr.io/cloud-builders/docker'
+  id: push-worker-tag
+  waitFor: ['build-worker']
+  args: [
+    'push',
+    '${_WORKER_IMAGE_NAME}:${TAG_NAME}'
+  ]
+
+- name: 'gcr.io/cloud-builders/docker'
+  id: push-worker-latest
+  waitFor: ['build-worker']
+  args: [
+    'push',
+    '${_WORKER_IMAGE_NAME}:latest'
   ]
-  waitFor: [ 'build', 'test' ]
 
 timeout: 600s
+substitutions:
+  _BASE_IMAGE_NAME: 'gcr.io/${PROJECT_ID}/pipe-segment'
+  _SCHEDULER_IMAGE_NAME: '${_BASE_IMAGE_NAME}/scheduler'
+  _WORKER_IMAGE_NAME: '${_BASE_IMAGE_NAME}/worker'
+options:
+  dynamic_substitutions: true
+