From 8f9609a78d05368c834623a669f228a142773f57 Mon Sep 17 00:00:00 2001 From: Liana Date: Thu, 21 Nov 2024 11:04:00 -0600 Subject: [PATCH] --wip-- [skipci] --- .../apps/machine-learning/boltz/README.md | 5 + .../boltz/app/helmrelease.yaml | 92 +++++++++++++++++++ .../boltz/app/kustomization.yaml | 6 ++ .../arc1/apps/machine-learning/boltz/ks.yaml | 24 +++++ .../colabfold/app/helmrelease.yaml | 7 +- .../apps/machine-learning/kustomization.yaml | 1 + .../apps/machine-learning/mmseqs2/README.md | 3 + .../mmseqs2/app/helmrelease.yaml | 2 +- .../apps/machine-learning/qlora/README.md | 7 ++ 9 files changed, 143 insertions(+), 4 deletions(-) create mode 100644 kubernetes/arc1/apps/machine-learning/boltz/README.md create mode 100644 kubernetes/arc1/apps/machine-learning/boltz/app/helmrelease.yaml create mode 100644 kubernetes/arc1/apps/machine-learning/boltz/app/kustomization.yaml create mode 100644 kubernetes/arc1/apps/machine-learning/boltz/ks.yaml create mode 100644 kubernetes/arc1/apps/machine-learning/mmseqs2/README.md create mode 100644 kubernetes/arc1/apps/machine-learning/qlora/README.md diff --git a/kubernetes/arc1/apps/machine-learning/boltz/README.md b/kubernetes/arc1/apps/machine-learning/boltz/README.md new file mode 100644 index 0000000..5f9f473 --- /dev/null +++ b/kubernetes/arc1/apps/machine-learning/boltz/README.md @@ -0,0 +1,5 @@ + + +We have containerized [Boltz-1](https://github.com/jwohlwend/boltz) for testing and development purposes + +Boltz-1 is an open-source model which predicts the 3D structure of proteins, RNA, DNA and small molecules; it handles modified residues, covalent ligands and glycans, as well as condition the generation on pocket residues. diff --git a/kubernetes/arc1/apps/machine-learning/boltz/app/helmrelease.yaml b/kubernetes/arc1/apps/machine-learning/boltz/app/helmrelease.yaml new file mode 100644 index 0000000..0e7a276 --- /dev/null +++ b/kubernetes/arc1/apps/machine-learning/boltz/app/helmrelease.yaml @@ -0,0 +1,92 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/bjw-s/helm-charts/main/charts/other/app-template/schemas/helmrelease-helm-v2.schema.json +# TODO: Finish this +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: &app boltz +spec: + interval: 30m + chart: + spec: + chart: app-template + version: 3.5.1 + sourceRef: + kind: HelmRepository + name: bjw-s + namespace: flux-system + install: + remediation: + retries: 3 + upgrade: + cleanupOnFail: true + remediation: + strategy: rollback + retries: 3 + values: + controllers: + boltz: + type: deployment + annotations: + reloader.stakater.com/auto: "true" + pod: + runtimeClassName: nvidia + terminationGracePeriodSeconds: 1 + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: nvidia.com/gpu.present + operator: In + values: + - "true" + containers: + app: + image: + repository: ghcr.io/rarecompute/boltz-docker-ci + # Base image + #tag: main@sha256:e2258b1a2ade6c5e92ee0b5e314fc8a073ca15d06bc4b6b6f32b0699ff958b96 + + # CUDA 12.1 + #tag: cuda12-1@sha256:c4da98989ade648a5158b01acc0d86648410ba9e73bfc281564aa8e000bd67ed + + # CUDA 12.4 + tag: cuda12-4@sha256:342069ec496df0027c355a5f7418732b8b0e86931e7ed2dfc3bc679b73b43e0b + command: ["tail", "-f", "/dev/null"] + env: + TZ: ${TIMEZONE} + NVIDIA_VISIBLE_DEVICES: all + NVIDIA_DRIVER_CAPABILITIES: all + securityContext: + capabilities.drop: ["ALL"] + resources: + requests: + cpu: 200m + memory: 8Gi + limits: + cpu: 16 + memory: 48Gi + nvidia.com/gpu: 4 + service: + app: + controller: *app + annotations: + ports: + http: + port: &port 80 + persistence: + # app: + # storageClass: local-nvme + # accessMode: ReadWriteOnce + # size: 1Gi + # globalMounts: + # - path: /app + workspace: + existingClaim: qlora-workspace + globalMounts: + - path: /workspace + tmp: + type: emptyDir + globalMounts: + - path: /tmp diff --git a/kubernetes/arc1/apps/machine-learning/boltz/app/kustomization.yaml b/kubernetes/arc1/apps/machine-learning/boltz/app/kustomization.yaml new file mode 100644 index 0000000..17cbc72 --- /dev/null +++ b/kubernetes/arc1/apps/machine-learning/boltz/app/kustomization.yaml @@ -0,0 +1,6 @@ +--- +# yaml-language-server: $schema=https://json.schemastore.org/kustomization +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./helmrelease.yaml diff --git a/kubernetes/arc1/apps/machine-learning/boltz/ks.yaml b/kubernetes/arc1/apps/machine-learning/boltz/ks.yaml new file mode 100644 index 0000000..901d624 --- /dev/null +++ b/kubernetes/arc1/apps/machine-learning/boltz/ks.yaml @@ -0,0 +1,24 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/fluxcd-community/flux2-schemas/main/kustomization-kustomize-v1.json +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: &app boltz + namespace: flux-system +spec: + targetNamespace: machine-learning + commonMetadata: + labels: + app.kubernetes.io/name: *app + path: ./kubernetes/arc1/apps/machine-learning/boltz/app + prune: true + sourceRef: + kind: GitRepository + name: k8s-gitops + wait: false + interval: 30m + retryInterval: 1m + timeout: 5m + postBuild: + substitute: + APP: *app diff --git a/kubernetes/arc1/apps/machine-learning/colabfold/app/helmrelease.yaml b/kubernetes/arc1/apps/machine-learning/colabfold/app/helmrelease.yaml index b198781..4b7f857 100644 --- a/kubernetes/arc1/apps/machine-learning/colabfold/app/helmrelease.yaml +++ b/kubernetes/arc1/apps/machine-learning/colabfold/app/helmrelease.yaml @@ -26,7 +26,7 @@ spec: retries: 3 values: controllers: - ollama: + colabfold: type: deployment annotations: reloader.stakater.com/auto: "true" @@ -56,11 +56,11 @@ spec: requests: cpu: 200m memory: 4Gi - gpu.intel.com/i915: "4" + nvidia.com/gpu: 2 limits: cpu: 32000m memory: 64Gi - gpu.intel.com/i915: "4" + nvidia.com/gpu: 4 service: app: controller: *app @@ -88,6 +88,7 @@ spec: - secretName: colabfold-tls hosts: [*host] persistence: + # TODO: Replace with existing PVC data: storageClass: local-nvme accessMode: ReadWriteMany diff --git a/kubernetes/arc1/apps/machine-learning/kustomization.yaml b/kubernetes/arc1/apps/machine-learning/kustomization.yaml index b4cfb5f..52d428e 100644 --- a/kubernetes/arc1/apps/machine-learning/kustomization.yaml +++ b/kubernetes/arc1/apps/machine-learning/kustomization.yaml @@ -6,4 +6,5 @@ resources: - ./namespace.yaml - ./qlora/ks.yaml - ./mmseqs2/ks.yaml + - ./boltz/ks.yaml #- ./jupyterhub/ks.yaml diff --git a/kubernetes/arc1/apps/machine-learning/mmseqs2/README.md b/kubernetes/arc1/apps/machine-learning/mmseqs2/README.md new file mode 100644 index 0000000..f0d56fa --- /dev/null +++ b/kubernetes/arc1/apps/machine-learning/mmseqs2/README.md @@ -0,0 +1,3 @@ + + +MMseqs2 (Many-against-Many sequence searching) is a software suite to search and cluster huge protein and nucleotide sequence sets. MMseqs2 is free and open source software implemented in C++ for Linux, MacOS, and (as beta version, via cygwin) Windows. The software is designed to run on multiple cores and servers and exhibits very good scalability. MMseqs2 can run 10000 times faster than BLAST. At 100 times its speed it achieves almost the same sensitivity. It can perform profile searches with the same sensitivity as PSI-BLAST at over 400 times its speed. diff --git a/kubernetes/arc1/apps/machine-learning/mmseqs2/app/helmrelease.yaml b/kubernetes/arc1/apps/machine-learning/mmseqs2/app/helmrelease.yaml index a480dbc..c0a4cfc 100644 --- a/kubernetes/arc1/apps/machine-learning/mmseqs2/app/helmrelease.yaml +++ b/kubernetes/arc1/apps/machine-learning/mmseqs2/app/helmrelease.yaml @@ -60,7 +60,7 @@ spec: resources: requests: cpu: 200m - memory: 8Gi + memory: 4Gi limits: cpu: 16 memory: 32Gi diff --git a/kubernetes/arc1/apps/machine-learning/qlora/README.md b/kubernetes/arc1/apps/machine-learning/qlora/README.md new file mode 100644 index 0000000..bd3ba0c --- /dev/null +++ b/kubernetes/arc1/apps/machine-learning/qlora/README.md @@ -0,0 +1,7 @@ +We have containerized [QLoRA](https://arxiv.org/abs/2305.14314) for testing and development purposes + +From [artidoro/qlora](https://github.com/artidoro/qlora) + +> QLoRA uses bitsandbytes for quantization and is integrated with Hugging Face's PEFT and transformers libraries. QLoRA was developed by members of the University of Washington's UW NLP group. +> +> We present QLoRA, an efficient finetuning approach that reduces memory usage enough to finetune a 65B parameter model on a single 48GB GPU while preserving full 16-bit finetuning task performance. QLoRA backpropagates gradients through a frozen, 4-bit quantized pretrained language model into Low Rank Adapters (LoRA). Our best model family, which we name Guanaco, outperforms all previous openly released models on the Vicuna benchmark, reaching 99.3% of the performance level of ChatGPT while only requiring 24 hours of finetuning on a single GPU. QLoRA introduces a number of innovations to save memory without sacrificing performance: (a) 4-bit NormalFloat (NF4), a new data type that is information theoretically optimal for normally distributed weights (b) Double Quantization to reduce the average memory footprint by quantizing the quantization constants, and (c) Paged Optimizers to manage memory spikes. We use QLoRA to finetune more than 1,000 models, providing a detailed analysis of instruction following and chatbot performance across 8 instruction datasets, multiple model types (LLaMA, T5), and model scales that would be infeasible to run with regular finetuning (e.g. 33B and 65B parameter models). Our results show that QLoRA finetuning on a small high-quality dataset leads to state-of-the-art results, even when using smaller models than the previous SoTA. We provide a detailed analysis of chatbot performance based on both human and GPT-4 evaluations showing that GPT-4 evaluations are a cheap and reasonable alternative to human evaluation. Furthermore, we find that current chatbot benchmarks are not trustworthy to accurately evaluate the performance levels of chatbots. We release all of our models and code, including CUDA kernels for 4-bit training.