sbatch updates

mansueto-institute · Apr 10, 2024 · e434371 · e434371
1 parent 874838a
commit e434371
Show file tree

Hide file tree

Showing 4 changed files with 13 additions and 7 deletions.
diff --git a/deployments/deploy_etl.err b/deployments/deploy_etl.err
@@ -0,0 +1,4 @@
+Traceback (most recent call last):
+  File "/project/crberry/firstamerican-etl/fa-etl.py", line 4, in <module>
+    import psutil
+ModuleNotFoundError: No module named 'psutil'
diff --git a/deployments/deploy_etl.out b/deployments/deploy_etl.out
diff --git a/fa-etl.py b/fa-etl.py
@@ -2,7 +2,6 @@
 from pathlib import Path
 import logging
 import psutil
-
 import polars as pl
 
 def mem_profile() -> str:

diff --git a/fa-etl.sbatch b/fa-etl.sbatch
@@ -13,15 +13,18 @@
 #SBATCH --account=pi-crberry
 
 module load python/anaconda-2022.05
-conda activate fa_etl_env
+
+# CREATE CONDA ENVIRONMENT IF NOT ALREADY CREATED
+# conda create --name fa_etl_env python=3.11.5 --yes
+# source activate fa_etl_env
+# conda install -c conda-forge polars --yes
+# conda install -c conda-forge psutil --yes
+
+source activate fa_etl_env
 
 working_directory=/Users/claireboyd/internships/mansueto/firstamerican-etl/dev/36061
 
 log_file_arg=$working_directory/deployments/deploy_etl.log
 input_dir_arg=$working_directory/
 
-python fa-etl.py --input_dir $input_dir_arg --log_file $log_file_arg
-
-
-
-
+python fa-etl.py --input_dir $input_dir_arg --log_file $log_file_arg