Add moondream and llava from huggingface transformers (#2176)

Summary:
For llava model:

* The CPU accuracy test fail because of CI runner CPU memory OOM
* The GPU accuracy test fail because of GPU CUDA OOM

Pull Request resolved: #2176

Reviewed By: aaronenyeshi

Differential Revision: D54075890

Pulled By: xuzhao9

fbshipit-source-id: 99a256616c228303e366734d3f33bcf2b63933a6

torchbenchmark/models/llava/__init__.py

@@ -0,0 +1,17 @@
+from torchbenchmark.tasks import NLP
+from torchbenchmark.util.framework.huggingface.model_factory import HuggingFaceModel
+
+class Model(HuggingFaceModel):
+    task = NLP.LANGUAGE_MODELING
+    # DEFAULT_TRAIN_BSIZE not specified since we're not implementing a train test
+    # DEFAULT_TRAIN_BSIZE = 1
+    DEFAULT_EVAL_BSIZE = 1
+
+    def __init__(self, test, device, batch_size=None, extra_args=[]):
+        super().__init__(name="llava", test=test, device=device, batch_size=batch_size, extra_args=extra_args)
+
+    def train(self):
+        return NotImplementedError("Not implemented")
+
+    def eval(self):
+        super().eval()

torchbenchmark/models/llava/install.py

@@ -0,0 +1,13 @@
+import subprocess
+import sys
+import os
+from torchbenchmark.util.framework.huggingface.patch_hf import patch_transformers, cache_model
+
+def pip_install_requirements():
+    subprocess.check_call([sys.executable, '-m', 'pip', 'install', '-q', '-r', 'requirements.txt'])
+
+if __name__ == '__main__':
+    pip_install_requirements()
+    patch_transformers()
+    model_name = os.path.basename(os.path.dirname(os.path.abspath(__file__)))
+    cache_model(model_name)

torchbenchmark/models/llava/metadata.yaml

@@ -0,0 +1,15 @@
+devices:
+  NVIDIA A100-SXM4-40GB:
+    eval_batch_size: 1
+eval_benchmark: false
+eval_deterministic: false
+eval_nograd: true
+train_benchmark: false
+train_deterministic: false
+not_implemented:
+  - device: NVIDIA A10G
+  # CPU OOM on the CI runner
+  - device: cpu
+  # accuracy test OOM on CUDA
+  - device: cuda
+    test: example

torchbenchmark/models/llava/requirements.txt

@@ -0,0 +1 @@
+einops

torchbenchmark/models/moondream/__init__.py

@@ -0,0 +1,17 @@
+from torchbenchmark.tasks import NLP
+from torchbenchmark.util.framework.huggingface.model_factory import HuggingFaceModel
+
+class Model(HuggingFaceModel):
+    task = NLP.LANGUAGE_MODELING
+    # DEFAULT_TRAIN_BSIZE not specified since we're not implementing a train test
+    # DEFAULT_TRAIN_BSIZE = 1
+    DEFAULT_EVAL_BSIZE = 1
+
+    def __init__(self, test, device, batch_size=None, extra_args=[]):
+        super().__init__(name="moondream", test=test, device=device, batch_size=batch_size, extra_args=extra_args)
+
+    def train(self):
+        return NotImplementedError("Not implemented")
+
+    def eval(self):
+        super().eval()

torchbenchmark/models/moondream/install.py

@@ -0,0 +1,13 @@
+import subprocess
+import sys
+import os
+from torchbenchmark.util.framework.huggingface.patch_hf import patch_transformers, cache_model
+
+def pip_install_requirements():
+    subprocess.check_call([sys.executable, '-m', 'pip', 'install', '-q', '-r', 'requirements.txt'])
+
+if __name__ == '__main__':
+    pip_install_requirements()
+    patch_transformers()
+    model_name = os.path.basename(os.path.dirname(os.path.abspath(__file__)))
+    cache_model(model_name)

torchbenchmark/models/moondream/metadata.yaml

@@ -0,0 +1,10 @@
+devices:
+  NVIDIA A100-SXM4-40GB:
+    eval_batch_size: 1
+eval_benchmark: false
+eval_deterministic: false
+eval_nograd: true
+train_benchmark: false
+train_deterministic: false
+not_implemented:
+  - device: NVIDIA A10G

torchbenchmark/models/moondream/requirements.txt

@@ -0,0 +1 @@
+einops

torchbenchmark/util/framework/huggingface/model_factory.py

@@ -9,6 +9,11 @@
 from torchbenchmark.tasks import NLP
 import transformers
 from transformers import AutoConfig, ReformerConfig, BertConfig, GenerationConfig, WhisperConfig, LlamaConfig
+# PhiConfig is only available in newer version of transformers
+try:
+    from transformers import PhiConfig
+except:
+    pass
 from typing import Tuple
 
 class_models = {
@@ -33,11 +38,13 @@
     # default num_hidden_layers=32 but that OOMs, feel free to change this config to something more real
     'llama_v2_7b_16h' : (128,512, 'LlamaConfig(num_hidden_layers=16)', 'AutoModelForCausalLM'),
     'hf_MPT_7b_instruct': (512, 512, 'AutoConfig.from_pretrained("mosaicml/mpt-7b-instruct", trust_remote_code=True)', 'AutoModelForCausalLM'),
+    'llava' : (512,512, 'AutoConfig.from_pretrained("liuhaotian/llava-v1.5-13b")', 'LlavaForConditionalGeneration'),
     'llama_v2_7b' : (512,512, 'AutoConfig.from_pretrained("meta-llama/Llama-2-7b-hf")', 'AutoModelForCausalLM'),
     'llama_v2_13b' : (512,512, 'AutoConfig.from_pretrained("meta-llama/Llama-2-13b-hf")', 'AutoModelForCausalLM'),
     'llama_v2_70b' : (512, 512, 'AutoConfig.from_pretrained("meta-llama/Llama-2-70b-hf")', 'AutoModelForMaskedLM'),
     'phi_1_5' : (512, 512, 'AutoConfig.from_pretrained("microsoft/phi-1_5", trust_remote_code=True)', 'AutoModelForCausalLM'),
     'phi_2' : (512, 512, 'AutoConfig.from_pretrained("microsoft/phi-2", trust_remote_code=True)', 'AutoModelForCausalLM'),
+    'moondream' : (512, 512, 'PhiConfig.from_pretrained("vikhyatk/moondream1")', 'PhiForCausalLM'),
     # as per this page https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.1 trust_remote_code=True is not required
     'mistral_7b_instruct' : (128, 128, 'AutoConfig.from_pretrained("mistralai/Mistral-7B-Instruct-v0.1")', 'AutoModelForCausalLM'),
     'hf_Yi' : (512, 512, 'AutoConfig.from_pretrained("01-ai/Yi-6B", trust_remote_code=True)', 'AutoModelForCausalLM'),
@@ -96,7 +103,10 @@ def __init__(self, name, test, device, batch_size=None, extra_args=[]):
         hugging_face_models_requiring_trust_remote_code = ["hf_Falcon_7b", "hf_MPT_7b_instruct", "phi_1_5", "phi_2", "hf_Yi"]
         if name in hugging_face_models_requiring_trust_remote_code:
             kwargs["trust_remote_code"] = True
-        self.model = class_ctor.from_config(config, **kwargs).to(device)
+        if hasattr(class_ctor, "from_config"):
+            self.model = class_ctor.from_config(config, **kwargs).to(device)
+        else:
+            self.model = class_ctor(config, **kwargs).to(device)
         self.optimizer = optim.Adam(
             self.model.parameters(),
             lr=0.001,

torchbenchmark/util/framework/huggingface/patch_hf.py

@@ -5,16 +5,20 @@
 import subprocess
 import sys
 from .model_factory import class_models
-from transformers import AutoConfig, ReformerConfig, BigBirdConfig, BertConfig, WhisperConfig, LlamaConfig
-
+from transformers import AutoConfig, ReformerConfig, BigBirdConfig, BertConfig, WhisperConfig, LlamaConfig, PhiConfig
+import inspect
 
 PATCH_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "patches")
 
 def cache_model(name: str, **kwargs):
     import transformers
     model_config = eval(class_models[name][2])
     model_ctor = getattr(transformers, class_models[name][3])
-    model_ctor.from_config(model_config, **kwargs)
+    if not hasattr(model_ctor, "from_config"):
+        model_ctor(model_config, **kwargs)
+    else:
+        model_ctor.from_config(model_config, **kwargs)
+
 
 def patch_transformers():
     import transformers