From 1db74eee4f07369ba8a05df6a37fda9459c25f7a Mon Sep 17 00:00:00 2001 From: Xu Zhao Date: Wed, 19 Jun 2024 15:41:20 -0400 Subject: [PATCH] Fix pip install requirements --- install.py | 18 ++--------- requirements.txt | 5 +-- .../models/Background_Matting/install.py | 7 ++-- .../Background_Matting/requirements.txt | 5 +-- torchbenchmark/models/dcgan/install.py | 7 +--- torchbenchmark/models/demucs/install.py | 15 +-------- torchbenchmark/models/hf_T5_base/install.py | 7 +--- torchbenchmark/models/hf_T5_large/install.py | 7 +--- torchbenchmark/models/hf_Whisper/install.py | 6 +--- torchbenchmark/util/framework/gnn/__init__.py | 13 ++------ utils/python_utils.py | 32 ++++++++++++++++++- 11 files changed, 44 insertions(+), 78 deletions(-) diff --git a/install.py b/install.py index 63930a33b7..195e2bdcec 100644 --- a/install.py +++ b/install.py @@ -6,25 +6,11 @@ from userbenchmark import list_userbenchmarks from utils import get_pkg_versions, TORCH_DEPS, generate_pkg_constraints +from utils.python_utils import pip_install_requirements REPO_ROOT = Path(__file__).parent -def pip_install_requirements(requirements_txt="requirements.txt"): - try: - subprocess.run( - [sys.executable, "-m", "pip", "install", "-q", "-r", requirements_txt], - check=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - ) - except subprocess.CalledProcessError as e: - return (False, e.output) - except Exception as e: - return (False, e) - return True, None - - if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( @@ -87,7 +73,7 @@ def pip_install_requirements(requirements_txt="requirements.txt"): ) sys.exit(0) - success, errmsg = pip_install_requirements() + success, errmsg = pip_install_requirements(continue_on_fail=True) if not success: print("Failed to install torchbenchmark requirements:") print(errmsg) diff --git a/requirements.txt b/requirements.txt index e38ba85136..b9d52c5b24 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,10 +15,7 @@ transformers==4.38.1 MonkeyType psutil pyyaml -# We need to pin numpy version to the same as the torch testing environment -# which still supports python 3.8 -numpy==1.21.2; python_version < '3.11' -numpy==1.26.0; python_version >= '3.11' +numpy opencv-python submitit pynvml diff --git a/torchbenchmark/models/Background_Matting/install.py b/torchbenchmark/models/Background_Matting/install.py index 97b247920d..66c1bff7fb 100644 --- a/torchbenchmark/models/Background_Matting/install.py +++ b/torchbenchmark/models/Background_Matting/install.py @@ -1,10 +1,7 @@ -import subprocess -import sys -from utils import s3_utils +from utils import s3_utils, python_utils def pip_install_requirements(): - subprocess.check_call([sys.executable, '-m', 'pip', - 'install', '-q', '-r', 'requirements.txt']) + python_utils.pip_install_requirements('requirements.txt') if __name__ == '__main__': pip_install_requirements() diff --git a/torchbenchmark/models/Background_Matting/requirements.txt b/torchbenchmark/models/Background_Matting/requirements.txt index 3250404275..188e465c72 100644 --- a/torchbenchmark/models/Background_Matting/requirements.txt +++ b/torchbenchmark/models/Background_Matting/requirements.txt @@ -1,7 +1,4 @@ -# We need to pin numpy version to the same as the torch testing environment -# which still supports python 3.8 -numpy==1.21.2; python_version < '3.11' -numpy==1.26.0; python_version >= '3.11' +numpy opencv-python pandas Pillow diff --git a/torchbenchmark/models/dcgan/install.py b/torchbenchmark/models/dcgan/install.py index 76fc6193d5..4ae5bae9c5 100644 --- a/torchbenchmark/models/dcgan/install.py +++ b/torchbenchmark/models/dcgan/install.py @@ -1,9 +1,4 @@ -import subprocess -import sys - - -def pip_install_requirements(): - subprocess.check_call([sys.executable, '-m', 'pip', 'install', '-q', '-r', 'requirements.txt']) +from utils.python_utils import pip_install_requirements if __name__ == '__main__': pip_install_requirements() diff --git a/torchbenchmark/models/demucs/install.py b/torchbenchmark/models/demucs/install.py index b550c34e8a..44bd90cc2c 100644 --- a/torchbenchmark/models/demucs/install.py +++ b/torchbenchmark/models/demucs/install.py @@ -1,17 +1,4 @@ -import subprocess -import sys - - -def pip_install_requirements(): - subprocess.check_call([sys.executable, '-m', 'pip', 'install', '-q', '-r', 'requirements.txt']) - -def spacy_download(language): - pass - -def preprocess(): - pass +from utils.python_utils import pip_install_requirements if __name__ == '__main__': pip_install_requirements() - spacy_download('') - preprocess() diff --git a/torchbenchmark/models/hf_T5_base/install.py b/torchbenchmark/models/hf_T5_base/install.py index c7855f3fef..2748b1da40 100644 --- a/torchbenchmark/models/hf_T5_base/install.py +++ b/torchbenchmark/models/hf_T5_base/install.py @@ -1,12 +1,7 @@ - -import subprocess -import sys import os +from utils.python_utils import pip_install_requirements from torchbenchmark.util.framework.huggingface.patch_hf import patch_transformers, cache_model -def pip_install_requirements(): - subprocess.check_call([sys.executable, '-m', 'pip', 'install', '-q', '-r', 'requirements.txt']) - if __name__ == '__main__': pip_install_requirements() patch_transformers() diff --git a/torchbenchmark/models/hf_T5_large/install.py b/torchbenchmark/models/hf_T5_large/install.py index c7855f3fef..81d8d0492c 100644 --- a/torchbenchmark/models/hf_T5_large/install.py +++ b/torchbenchmark/models/hf_T5_large/install.py @@ -1,11 +1,6 @@ - -import subprocess -import sys import os from torchbenchmark.util.framework.huggingface.patch_hf import patch_transformers, cache_model - -def pip_install_requirements(): - subprocess.check_call([sys.executable, '-m', 'pip', 'install', '-q', '-r', 'requirements.txt']) +from utils.python_utils import pip_install_requirements if __name__ == '__main__': pip_install_requirements() diff --git a/torchbenchmark/models/hf_Whisper/install.py b/torchbenchmark/models/hf_Whisper/install.py index 1a49905932..b0fe419716 100644 --- a/torchbenchmark/models/hf_Whisper/install.py +++ b/torchbenchmark/models/hf_Whisper/install.py @@ -1,11 +1,7 @@ -import subprocess -import sys import os +from utils.python_utils import pip_install_requirements from torchbenchmark.util.framework.huggingface.patch_hf import patch_transformers, cache_model -def pip_install_requirements(): - subprocess.check_call([sys.executable, '-m', 'pip', 'install', '-q', '-r', 'requirements.txt']) - if __name__ == '__main__': pip_install_requirements() patch_transformers() diff --git a/torchbenchmark/util/framework/gnn/__init__.py b/torchbenchmark/util/framework/gnn/__init__.py index 960c115cc1..a37c4896f4 100644 --- a/torchbenchmark/util/framework/gnn/__init__.py +++ b/torchbenchmark/util/framework/gnn/__init__.py @@ -1,16 +1,7 @@ import os.path -import subprocess -import sys +from utils.python_utils import pip_install_requirements CURRENT_DIR = os.path.dirname(os.path.realpath(__file__)) - def install_pytorch_geometric(): - pip_install_requirements() - - -def pip_install_requirements(): - requirements_file = os.path.join(CURRENT_DIR, "requirements.txt") - subprocess.check_call( - [sys.executable, "-m", "pip", "install", "-q", "-r", requirements_file] - ) + pip_install_requirements(os.path.join(CURRENT_DIR, "requirements.txt")) diff --git a/utils/python_utils.py b/utils/python_utils.py index ca8a1d0965..9d92bb3de8 100644 --- a/utils/python_utils.py +++ b/utils/python_utils.py @@ -1,4 +1,5 @@ -import argparse +import warnings +from pathlib import Path import subprocess DEFAULT_PYTHON_VERSION = "3.11" @@ -14,6 +15,7 @@ "pytorch_url": "cp311", }, } +REPO_DIR = Path(__file__).parent.parent def create_conda_env(pyver: str, name: str): @@ -21,7 +23,35 @@ def create_conda_env(pyver: str, name: str): subprocess.check_call(command) +def pip_install_requirements(requirements_txt="requirements.txt", continue_on_fail=False): + import sys + constraints_file = REPO_DIR.joinpath("build", "contraints.txt") + if not constraints_file.exists(): + warnings.warn("contraints.txt could not be found, please rerun install.py.") + constraints_parameters = [] + else: + constraints_parameters = ["-c", str(constraints_file.resolve())] + if not continue_on_fail: + subprocess.check_call( + [sys.executable, "-m", "pip", "install", "-r", requirements_txt] + constraints_parameters, + ) + return True, None + try: + subprocess.run( + [sys.executable, "-m", "pip", "install", "-r", requirements_txt] + constraints_parameters, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + except subprocess.CalledProcessError as e: + return (False, e.output) + except Exception as e: + return (False, e) + return True, None + + if __name__ == "__main__": + import argparse parser = argparse.ArgumentParser() parser.add_argument( "--pyver",