diff --git a/deployments/deploy_etl.err b/deployments/deploy_etl.err new file mode 100644 index 0000000..d9e5022 --- /dev/null +++ b/deployments/deploy_etl.err @@ -0,0 +1,4 @@ +Traceback (most recent call last): + File "/project/crberry/firstamerican-etl/fa-etl.py", line 4, in + import psutil +ModuleNotFoundError: No module named 'psutil' diff --git a/deployments/deploy_etl.out b/deployments/deploy_etl.out new file mode 100644 index 0000000..e69de29 diff --git a/fa-etl.py b/fa-etl.py index 3ca523f..e3e638a 100644 --- a/fa-etl.py +++ b/fa-etl.py @@ -2,7 +2,6 @@ from pathlib import Path import logging import psutil - import polars as pl def mem_profile() -> str: diff --git a/fa-etl.sbatch b/fa-etl.sbatch index e2cd5f6..d51ed59 100644 --- a/fa-etl.sbatch +++ b/fa-etl.sbatch @@ -13,15 +13,18 @@ #SBATCH --account=pi-crberry module load python/anaconda-2022.05 -conda activate fa_etl_env + +# CREATE CONDA ENVIRONMENT IF NOT ALREADY CREATED +# conda create --name fa_etl_env python=3.11.5 --yes +# source activate fa_etl_env +# conda install -c conda-forge polars --yes +# conda install -c conda-forge psutil --yes + +source activate fa_etl_env working_directory=/Users/claireboyd/internships/mansueto/firstamerican-etl/dev/36061 log_file_arg=$working_directory/deployments/deploy_etl.log input_dir_arg=$working_directory/ -python fa-etl.py --input_dir $input_dir_arg --log_file $log_file_arg - - - - +python fa-etl.py --input_dir $input_dir_arg --log_file $log_file_arg \ No newline at end of file