Merge pull request #3727 from Yelp/u/cuza/improvements-to-paasta-play…

…ground

Improvements to PaaSTA Playground

.gitignore

@@ -50,6 +50,7 @@ general_itests/fake_etc_paasta/clusters.json
 pip-wheel-metadata
 debian/debhelper-build-stamp
 unique-run
+.vault-token
 
 # Coverage artifacts
 .coverage

Makefile

@@ -158,13 +158,33 @@ setup-kubernetes-job: k8s_fake_cluster generate_deployments_for_service
 	export PAASTA_TEST_CLUSTER=kind-${USER}-k8s-test;\
 	.tox/py38-linux/bin/python -m paasta_tools.list_kubernetes_service_instances -d ./soa_config_playground --shuffle --group-lines 1 | xargs --no-run-if-empty .tox/py38-linux/bin/python -m paasta_tools.setup_kubernetes_job -d ./soa_config_playground -c kind-${USER}-k8s-test
 
+.PHONY: cleanup-kubernetes-jobs
+cleanup-kubernetes-jobs:
+	export KUBECONFIG=./k8s_itests/kubeconfig;\
+	export PAASTA_SYSTEM_CONFIG_DIR=./etc_paasta_playground/;\
+	export PAASTA_TEST_CLUSTER=kind-${USER}-k8s-test;\
+	.tox/py38-linux/bin/python -m paasta_tools.cleanup_kubernetes_jobs -d ./soa_config_playground -c kind-${USER}-k8s-test --force
+
 .PHONY: paasta-secrets-sync
 paasta-secrets-sync: setup-kubernetes-job .vault-token
 	export KUBECONFIG=./k8s_itests/kubeconfig;\
 	export PAASTA_SYSTEM_CONFIG_DIR=./etc_paasta_playground/;\
 	export PAASTA_TEST_CLUSTER=kind-${USER}-k8s-test;\
 	{ .tox/py38-linux/bin/python -m paasta_tools.list_kubernetes_service_instances -d ./soa_config_playground ; echo -n \ _shared; } | cut -f1 -d"." | uniq | shuf | xargs .tox/py38-linux/bin/python -m paasta_tools.kubernetes.bin.paasta_secrets_sync -v -d ./soa_config_playground -t ./.vault-token
 
+define ANNOUNCE_CRONS_BODY
+The following PaaSTA cron jobs will run on an infinite loop using the PaaSTA Playground k8s cluster:
+- setup-kubernetes-job
+- cleanup-kubernetes-job
+- paasta-secrets-sync
+- generate_deployments_for_service
+endef
+export ANNOUNCE_CRONS_BODY
+.PHONY: paasta-crons
+make paasta-cronjobs:
+	@echo "$$ANNOUNCE_CRONS_BODY"
+	while true; do make paasta-secrets-sync && make cleanup-kubernetes-jobs; sleep 5; done
+
 .vault-token:
 	export VAULT_ADDR=https://vault-devc.yelpcorp.com:8200 ;\
 	export VAULT_SKIP_VERIFY=true ;\

paasta_tools/contrib/create_paasta_playground.py

@@ -46,6 +46,7 @@ def main():
         src="./k8s_itests/deployments/paasta/fake_soa_config",
         dst="soa_config_playground",
         values=values_path,
+        overwrite=False,
     )
 
 

paasta_tools/contrib/render_template.py

@@ -31,23 +31,25 @@ def render_file(src, dst, values):
             new.write(replace(old.read(), values))
 
 
-def render(src, dst, values={}, exclude={}):
+def render(src, dst, values={}, exclude={}, overwrite=True):
     if os.path.isfile(src):
-        render_file(src, dst, values)
+        if overwrite:
+            render_file(src, dst, values)
         return
     for f in os.scandir(src):
         if f.name.startswith(".") or f.path in exclude:
             continue
         if os.path.isfile(f.path):
-            render_file(f.path, dst, values)
+            if overwrite:
+                render_file(f.path, dst, values)
         else:
             new_dst = replace(f"{dst}/{f.name}", values)
             try:
                 os.makedirs(new_dst, exist_ok=True)
             except OSError as e:
                 if e.errno != os.errno.EEXIST:
                     raise
-            render(f.path, new_dst, values, exclude)
+            render(f.path, new_dst, values, exclude, overwrite)
 
 
 def parse_args():
@@ -82,7 +84,7 @@ def parse_args():
     return args
 
 
-def render_values(src: str, dst: str, values: str) -> None:
+def render_values(src: str, dst: str, values: str, overwrite=True) -> None:
     if values is not None:
         values = os.path.abspath(values)
     # Validate src and values. Dst needs to be a directory. src can be either a valid folder of directory. values need to be valid file if provided.
@@ -108,7 +110,7 @@ def render_values(src: str, dst: str, values: str) -> None:
             ),
             v,
         )
-    render(src, dst, config_dict, {values})
+    render(src, dst, config_dict, {values}, overwrite)
 
 
 def main():

requirements-dev-minimal.txt

@@ -1,6 +1,9 @@
 astroid
 asynctest
 coverage
+# VSCode debugging requirement
+# See https://code.visualstudio.com/docs/python/debugging#_local-script-debugging
+debugpy
 docutils
 flake8
 freezegun

requirements-dev.txt

@@ -5,6 +5,7 @@ asynctest==0.12.0
 Babel==2.9.1
 cfgv==2.0.1
 coverage==6.5.0
+debugpy==1.8.0
 distlib==0.3.4
 exceptiongroup==1.1.2
 filelock==3.0.12