From fcef4c958db62fcb7afcc94caaca80e2a87e2cce Mon Sep 17 00:00:00 2001 From: Joel Mathew Thomas <90510078+joelmathewthomas@users.noreply.github.com> Date: Sun, 26 Jan 2025 16:03:24 +0530 Subject: [PATCH] fix dependency hell setup a new env structure. Use a common env, and separate one for tensorflow due to dependency hell --- .gitignore | 4 + requirements.txt | 131 -------------------------------- requirements/env.txt | 66 ++++++++++++++++ requirements/env_tensorflow.txt | 51 +++++++++++++ setup_env.sh | 25 ++++++ 5 files changed, 146 insertions(+), 131 deletions(-) delete mode 100644 requirements.txt create mode 100644 requirements/env.txt create mode 100644 requirements/env_tensorflow.txt create mode 100755 setup_env.sh diff --git a/.gitignore b/.gitignore index 06b7abc..a1aa0c5 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ __pycache__/ # Ignore virtual environment folder venv/ +envs/ # Ignore other common files *.pyc @@ -14,3 +15,6 @@ venv/ # Ignore VSCode config .vscode/ + +# site-packages.pth +site-packages.pth diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 97e2a49..0000000 --- a/requirements.txt +++ /dev/null @@ -1,131 +0,0 @@ -absl-py==2.1.0 -amqp==5.3.1 -antlr4-python3-runtime==4.9.3 -asgiref==3.8.1 -asttokens==3.0.0 -astunparse==1.6.3 -audioread==3.0.1 -billiard==4.2.1 -celery==5.4.0 -certifi==2024.12.14 -cffi==1.17.1 -charset-normalizer==3.4.0 -click==8.1.8 -click-didyoumean==0.3.1 -click-plugins==1.1.1 -click-repl==0.3.0 -cloudpickle==3.1.0 -contourpy==1.3.1 -cycler==0.12.1 -decorator==5.1.1 -demucs==4.0.1 -Django==5.1.4 -dora_search==0.1.12 -einops==0.8.0 -executing==2.1.0 -filelock==3.16.1 -flatbuffers==24.12.23 -fonttools==4.55.3 -fsspec==2024.12.0 -gast==0.6.0 -google-pasta==0.2.0 -grpcio==1.68.1 -h5py==3.12.1 -idna==3.10 -iniconfig==2.0.0 -jedi==0.19.2 -Jinja2==3.1.5 -joblib==1.4.2 -julius==0.2.7 -keras==3.7.0 -kiwisolver==1.4.8 -kombu==5.4.2 -lameenc==1.7.0 -lazy_loader==0.4 -libclang==18.1.1 -librosa==0.10.2.post1 -llvmlite==0.43.0 -Markdown==3.7 -markdown-it-py==3.0.0 -MarkupSafe==3.0.2 -matplotlib==3.10.0 -matplotlib-inline==0.1.7 -mdurl==0.1.2 -ml-dtypes==0.4.1 -mpmath==1.3.0 -msgpack==1.1.0 -namex==0.0.8 -networkx==3.4.2 -numba==0.60.0 -numpy==2.0.2 -nvidia-cublas-cu12==12.4.5.8 -nvidia-cuda-cupti-cu12==12.4.127 -nvidia-cuda-nvrtc-cu12==12.4.127 -nvidia-cuda-runtime-cu12==12.4.127 -nvidia-cudnn-cu12==9.1.0.70 -nvidia-cufft-cu12==11.2.1.3 -nvidia-curand-cu12==10.3.5.147 -nvidia-cusolver-cu12==11.6.1.9 -nvidia-cusparse-cu12==12.3.1.170 -nvidia-nccl-cu12==2.21.5 -nvidia-nvjitlink-cu12==12.4.127 -nvidia-nvtx-cu12==12.4.127 -omegaconf==2.3.0 -openunmix==1.3.0 -opt_einsum==3.4.0 -optree==0.13.1 -packaging==24.2 -parso==0.8.4 -pexpect==4.9.0 -pillow==11.0.0 -platformdirs==4.3.6 -pluggy==1.5.0 -pooch==1.8.2 -prompt_toolkit==3.0.48 -protobuf==5.29.2 -ptyprocess==0.7.0 -pure_eval==0.2.3 -pycparser==2.22 -Pygments==2.18.0 -pyparsing==3.2.0 -PyQt6==6.8.0 -PyQt6-Qt6==6.8.1 -PyQt6_sip==13.9.1 -pytest==8.3.4 -python-dateutil==2.9.0.post0 -PyYAML==6.0.2 -redis==5.2.1 -requests==2.32.3 -retrying==1.3.4 -rich==13.9.4 -scikit-learn==1.6.0 -scipy==1.14.1 -setuptools==75.6.0 -six==1.17.0 -soundfile==0.12.1 -soxr==0.5.0.post1 -sqlparse==0.5.3 -stack-data==0.6.3 -submitit==1.5.2 -sympy==1.13.1 -tensorboard==2.18.0 -tensorboard-data-server==0.7.2 -tensorflow==2.18.0 -tensorflow-hub==0.16.1 -termcolor==2.5.0 -tf_keras==2.18.0 -threadpoolctl==3.5.0 -torch==2.5.1 -torchaudio==2.5.1 -tqdm==4.67.1 -traitlets==5.14.3 -treetable==0.2.5 -triton==3.1.0 -typing_extensions==4.12.2 -tzdata==2024.2 -urllib3==2.3.0 -vine==5.1.0 -wcwidth==0.2.13 -Werkzeug==3.1.3 -wheel==0.45.1 -wrapt==1.17.0 diff --git a/requirements/env.txt b/requirements/env.txt new file mode 100644 index 0000000..072019e --- /dev/null +++ b/requirements/env.txt @@ -0,0 +1,66 @@ +antlr4-python3-runtime==4.9.3 +audioread==3.0.1 +certifi==2024.12.14 +cffi==1.17.1 +charset-normalizer==3.4.1 +cloudpickle==3.1.1 +decorator==5.1.1 +demucs==4.0.1 +dora_search==0.1.12 +einops==0.8.0 +filelock==3.17.0 +fsspec==2024.12.0 +idna==3.10 +iniconfig==2.0.0 +Jinja2==3.1.5 +joblib==1.4.2 +julius==0.2.7 +lameenc==1.8.1 +lazy_loader==0.4 +librosa==0.10.2.post1 +llvmlite==0.44.0 +MarkupSafe==3.0.2 +mpmath==1.3.0 +msgpack==1.1.0 +networkx==3.4.2 +numba==0.61.0 +numpy==2.1.3 +nvidia-cublas-cu12==12.4.5.8 +nvidia-cuda-cupti-cu12==12.4.127 +nvidia-cuda-nvrtc-cu12==12.4.127 +nvidia-cuda-runtime-cu12==12.4.127 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.2.1.3 +nvidia-curand-cu12==10.3.5.147 +nvidia-cusolver-cu12==11.6.1.9 +nvidia-cusparse-cu12==12.3.1.170 +nvidia-nccl-cu12==2.21.5 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-nvtx-cu12==12.4.127 +omegaconf==2.3.0 +openunmix==1.3.0 +packaging==24.2 +platformdirs==4.3.6 +pluggy==1.5.0 +pooch==1.8.2 +pycparser==2.22 +pytest==8.3.4 +PyYAML==6.0.2 +requests==2.32.3 +retrying==1.3.4 +scikit-learn==1.6.1 +scipy==1.15.1 +setuptools==75.8.0 +six==1.17.0 +soundfile==0.13.1 +soxr==0.5.0.post1 +submitit==1.5.2 +sympy==1.13.1 +threadpoolctl==3.5.0 +torch==2.5.1 +torchaudio==2.5.1 +tqdm==4.67.1 +treetable==0.2.5 +triton==3.1.0 +typing_extensions==4.12.2 +urllib3==2.3.0 diff --git a/requirements/env_tensorflow.txt b/requirements/env_tensorflow.txt new file mode 100644 index 0000000..6c319b0 --- /dev/null +++ b/requirements/env_tensorflow.txt @@ -0,0 +1,51 @@ +absl-py==2.1.0 +astunparse==1.6.3 +certifi==2024.12.14 +charset-normalizer==3.4.1 +flatbuffers==25.1.24 +gast==0.6.0 +google-pasta==0.2.0 +grpcio==1.70.0 +h5py==3.12.1 +idna==3.10 +keras==3.8.0 +libclang==18.1.1 +Markdown==3.7 +markdown-it-py==3.0.0 +MarkupSafe==3.0.2 +mdurl==0.1.2 +ml-dtypes==0.4.1 +namex==0.0.8 +numpy==2.0.2 +nvidia-cublas-cu12==12.5.3.2 +nvidia-cuda-cupti-cu12==12.5.82 +nvidia-cuda-nvcc-cu12==12.5.82 +nvidia-cuda-nvrtc-cu12==12.5.82 +nvidia-cuda-runtime-cu12==12.5.82 +nvidia-cudnn-cu12==9.3.0.75 +nvidia-cufft-cu12==11.2.3.61 +nvidia-curand-cu12==10.3.6.82 +nvidia-cusolver-cu12==11.6.3.83 +nvidia-cusparse-cu12==12.5.1.3 +nvidia-nccl-cu12==2.21.5 +nvidia-nvjitlink-cu12==12.5.82 +opt_einsum==3.4.0 +optree==0.14.0 +packaging==24.2 +protobuf==5.29.3 +Pygments==2.19.1 +requests==2.32.3 +rich==13.9.4 +setuptools==75.8.0 +six==1.17.0 +tensorboard==2.18.0 +tensorboard-data-server==0.7.2 +tensorflow==2.18.0 +tensorflow-hub==0.16.1 +termcolor==2.5.0 +tf_keras==2.18.0 +typing_extensions==4.12.2 +urllib3==2.3.0 +Werkzeug==3.1.3 +wheel==0.45.1 +wrapt==1.17.2 diff --git a/setup_env.sh b/setup_env.sh new file mode 100755 index 0000000..3eab299 --- /dev/null +++ b/setup_env.sh @@ -0,0 +1,25 @@ +echo "Setting up virtual environments" + +mkdir envs + +#Create common env +echo "Creating common virtual environment env" +python -m venv envs/env + +#Create env for tensorflow +echo "Creating virtual environment env_tensorflow" +python -m venv envs/env_tensorflow + +source envs/env/bin/activate +echo "Installing dependencies in virtual environment env" +pip install -r requirements/env.txt +deactivate + +source envs/env_tensorflow/bin/activate +pip install -r requirements/env_tensorflow.txt +deactivate + +echo "Creating site-packages.pth" +touch site-packages.pth +echo "env/env/lib/python3.12/site-packages" >> site-packages.pth +echo "env/env_tensorflow/lib/python3.12/site-packages" >> site-packages.pth