Add 13/09/2025 results

.gitignore

@@ -1 +1,2 @@
-__pycache__/
+__pycache__/
+tests/

evaluation.py

@@ -3,15 +3,29 @@ import argparse
 import json
 import time
 import os
+from copy import deepcopy
 from fvcore.nn import FlopCountAnalysis, parameter_count_table
 from torch.utils.data import DataLoader
 from datasets import load_dataset
-from transformers import AutoModelForImageClassification, AutoImageProcessor
+from transformers import AutoModel, AutoModelForImageClassification, AutoImageProcessor
 from tqdm import tqdm
+from PIL import Image
 
 
+def get_image_size(processor):
+    dummy_image = Image.new("RGB", (256, 256), color="white")
+    dummy_image = processor(dummy_image, return_tensors="pt")
+
+    image_size = {
+        'height': dummy_image['pixel_values'].shape[-2],
+        'width': dummy_image['pixel_values'].shape[-1]
+    }
+
+    return image_size
+
 def benchmark_inference(model, processor, device, runs=20, warmup=5):
-    dummy_image = torch.randn(1, 3, processor.size['height'], processor.size['width'], device=device)
+    image_size = get_image_size(processor)
+    dummy_image = torch.randn(1, 3, image_size['height'], image_size['width'], device=device)
     model.eval()
     
     # Warmup
@@ -49,15 +63,16 @@ def load_dataloader(args):
 def evaluate_model(args, dataloader, model_info):
     """Evaluate a model on ImageNet-1k validation set"""
     device = torch.device(args.device if torch.cuda.is_available() else "cpu")
-    
+
+    #model = AutoModel.from_pretrained(model_info["path"], trust_remote_code=True)
     model = AutoModelForImageClassification.from_pretrained(
-        model_info["path"], 
-        torch_dtype=torch.float32  # Use float32 for consistency
+        model_info["path"],
+        #trust_remote_code=True
     )
     
     processor = AutoImageProcessor.from_pretrained(model_info["path"])
-    print(dir(processor))
-    print(processor.size)
+    image_size = get_image_size(processor)
+
     model.to(device)
     model.eval()
     
@@ -181,7 +196,7 @@ if __name__ == "__main__":
                     epilog='Results will be saved to JSONL file')
     parser.add_argument('--data-path', default="ILSVRC/imagenet-1k", 
                        help='Path to ImageNet-1k dataset')
-    parser.add_argument('--device', default="cuda", 
+    parser.add_argument('--device', default="cuda:6", 
                        help='Device to use for evaluation (cuda/cpu)')
     parser.add_argument('--batch-size', type=int, default=128, 
                        help='Batch size for evaluation')
@@ -192,7 +207,7 @@ if __name__ == "__main__":
     args = parser.parse_args()
 
     # Override data path with absolute path
-    args.device = "cuda:6"
+    #args.device = "cuda:6"
     args.data_path = "/data3/salah/datasets/imagenet-1k"
 
     # Load models list

imagenet_results.jsonl

@@ -17,3 +17,14 @@
 {"model": "facebook/convnext-xlarge-224-22k-1k", "top1_accuracy": 86.98599999999999, "top5_accuracy": 98.202, "parameters": 350196968, "flops": 60967704576, "inference_time": 22.357892990112305, "model_size": 1400787872, "license": "Open"}
 {"model": "facebook/convnext-xlarge-384-22k-1k", "top1_accuracy": 87.764, "top5_accuracy": 98.55199999999999, "parameters": 350196968, "flops": 179166406656, "inference_time": 22.17559814453125, "model_size": 1400787872, "license": "Open"}
 {"model": "facebook/deit-base-distilled-patch16-224", "top1_accuracy": 83.39800000000001, "top5_accuracy": 96.43599999999999, "parameters": 87338192, "flops": 16953211392, "inference_time": 5.678451061248779, "model_size": 349352768, "license": "Open"}
+{"model": "google/efficientnet-b0", "top1_accuracy": 75.472, "top5_accuracy": 92.552, "parameters": 5288548, "flops": 17061920, "inference_time": 6.572115421295166, "model_size": 21154192, "license": "Apache-2.0"}
+{"model": "google/efficientnet-b1", "top1_accuracy": 77.05, "top5_accuracy": 93.514, "parameters": 7794184, "flops": 27127664, "inference_time": 9.38577651977539, "model_size": 31176736, "license": "Apache-2.0"}
+{"model": "google/efficientnet-b2", "top1_accuracy": 78.25999999999999, "top5_accuracy": 94.192, "parameters": 9109994, "flops": 34311200, "inference_time": 13.025212287902832, "model_size": 36439976, "license": "Apache-2.0"}
+{"model": "google/efficientnet-b3", "top1_accuracy": 80.43, "top5_accuracy": 95.346, "parameters": 12233232, "flops": 58144576, "inference_time": 10.713708400726318, "model_size": 48932928, "license": "Apache-2.0"}
+{"model": "google/efficientnet-b4", "top1_accuracy": 82.53, "top5_accuracy": 96.136, "parameters": 19341616, "flops": 120258128, "inference_time": 13.012278079986572, "model_size": 77366464, "license": "Apache-2.0"}
+{"model": "google/efficientnet-b5", "top1_accuracy": 83.358, "top5_accuracy": 96.612, "parameters": 30389784, "flops": 240274768, "inference_time": 21.18891477584839, "model_size": 121559136, "license": "Apache-2.0"}
+{"model": "google/efficientnet-b6", "top1_accuracy": 83.928, "top5_accuracy": 96.892, "parameters": 43040704, "flops": 406106768, "inference_time": 18.17166805267334, "model_size": 172162816, "license": "Apache-2.0"}
+{"model": "microsoft/cvt-13", "top1_accuracy": 81.404, "top5_accuracy": 95.616, "parameters": 19997480, "flops": 4584573392.0, "inference_time": 11.934351921081543, "model_size": 79989920, "license": "Open"}
+{"model": "microsoft/cvt-21-384", "top1_accuracy": 82.648, "top5_accuracy": 95.882, "parameters": 31622696, "flops": 24932082688.0, "inference_time": 18.59004497528076, "model_size": 126490784, "license": "Open"}
+{"model": "microsoft/cvt-21", "top1_accuracy": 81.54, "top5_accuracy": 95.438, "parameters": 31622696, "flops": 7206283808.0, "inference_time": 19.456958770751953, "model_size": 126490784, "license": "Open"}
+{"model": "microsoft/cvt-13-384", "top1_accuracy": 82.788, "top5_accuracy": 96.308, "parameters": 19997480, "flops": 16323617952.0, "inference_time": 16.565978527069092, "model_size": 79989920, "license": "Open"}

models_list.json

@@ -5,6 +5,11 @@
     {"path": "microsoft/resnet-50", "paper": "https://arxiv.org/abs/1512.03385", "license": "Open", "year": 2015},
     {"path": "microsoft/resnet-101", "paper": "https://arxiv.org/abs/1512.03385", "license": "Open", "year": 2015},
     {"path": "microsoft/resnet-152", "paper": "https://arxiv.org/abs/1512.03385", "license": "Open", "year": 2015},
+    
+    {"path": "microsoft/cvt-13", "paper": "https://arxiv.org/abs/2103.15808", "license": "Open", "year": 2021},
+    {"path": "microsoft/cvt-21-384", "paper": "https://arxiv.org/abs/2103.15808", "license": "Open", "year": 2021},
+    {"path": "microsoft/cvt-21", "paper": "https://arxiv.org/abs/2103.15808", "license": "Open", "year": 2021},
+    {"path": "microsoft/cvt-13-384", "paper": "https://arxiv.org/abs/2103.15808", "license": "Open", "year": 2021},
 
     {"path": "facebook/convnext-base-224-22k-1k", "paper": "https://arxiv.org/abs/2201.09792", "license": "Open", "year": 2022},
     {"path": "facebook/convnext-base-224", "paper": "https://arxiv.org/abs/2201.09792", "license": "Open", "year": 2022},
@@ -115,9 +120,6 @@
     {"path": "google/vit-large-patch32-384", "paper": "https://arxiv.org/abs/2010.11929", "license": "Apache-2.0", "year": 2020},
     {"path": "google/vit-hybrid-base-bit-384", "paper": "https://arxiv.org/abs/2010.11929", "license": "Apache-2.0", "year": 2020},
 
-    {"path": "google/cxr-foundation", "paper": "https://arxiv.org/abs/2311.18775", "license": "Apache-2.0", "year": 2023},
-    {"path": "google/derm-foundation", "paper": "https://arxiv.org/abs/2311.18775", "license": "Apache-2.0", "year": 2023},
-
     {"path": "google/mobilenet_v2_1.4_224", "paper": "https://arxiv.org/abs/1801.04381", "license": "Apache-2.0", "year": 2018},
     {"path": "google/mobilenet_v2_1.0_224", "paper": "https://arxiv.org/abs/1801.04381", "license": "Apache-2.0", "year": 2018},
     {"path": "google/mobilenet_v2_0.75_160", "paper": "https://arxiv.org/abs/1801.04381", "license": "Apache-2.0", "year": 2018},
@@ -137,41 +139,23 @@
     {"path": "google/efficientnet-b6", "paper": "https://arxiv.org/abs/1905.11946", "license": "Apache-2.0", "year": 2019},
     {"path": "google/efficientnet-b7", "paper": "https://arxiv.org/abs/1905.11946", "license": "Apache-2.0", "year": 2019},
 
-
-    {"path": "qualcomm/MobileNet-v3-Small", "paper": "https://arxiv.org/abs/1905.02244", "license": "Open", "year": 2019},
-    {"path": "qualcomm/Swin-Base", "paper": "https://arxiv.org/abs/2103.14030", "license": "Open", "year": 2021},
-    {"path": "qualcomm/DenseNet-121", "paper": "https://arxiv.org/abs/1608.06993", "license": "Open", "year": 2016},
-    {"path": "qualcomm/Inception-v3", "paper": "https://arxiv.org/abs/1512.00567", "license": "Open", "year": 2015},
-    {"path": "qualcomm/WideResNet50", "paper": "https://arxiv.org/abs/1605.07146", "license": "Open", "year": 2016},
-    {"path": "qualcomm/RegNet", "paper": "https://arxiv.org/abs/2003.13678", "license": "Open", "year": 2020},
-    {"path": "qualcomm/ResNet18", "paper": "https://arxiv.org/abs/1512.03385", "license": "Open", "year": 2015},
-    {"path": "qualcomm/MobileNet-v2", "paper": "https://arxiv.org/abs/1801.04381", "license": "Open", "year": 2018},
-    {"path": "qualcomm/ConvNext-Tiny", "paper": "https://arxiv.org/abs/2201.03545", "license": "Open", "year": 2022},
-    {"path": "qualcomm/ResNet101", "paper": "https://arxiv.org/abs/1512.03385", "license": "Open", "year": 2015},
-    {"path": "qualcomm/OpenAI-Clip", "paper": "https://arxiv.org/abs/2103.00020", "license": "Open", "year": 2021},
-    {"path": "qualcomm/Swin-Tiny", "paper": "https://arxiv.org/abs/2103.14030", "license": "Open", "year": 2021},
-    {"path": "qualcomm/ResNeXt101", "paper": "https://arxiv.org/abs/1611.05431", "license": "Open", "year": 2016},
-    {"path": "qualcomm/EfficientNet-B0", "paper": "https://arxiv.org/abs/1905.11946", "license": "Open", "year": 2019},
-    {"path": "qualcomm/GoogLeNet", "paper": "https://arxiv.org/abs/1409.4842", "license": "Open", "year": 2014},
-    {"path": "qualcomm/MNASNet05", "paper": "https://arxiv.org/abs/1807.11626", "license": "Open", "year": 2018},
-    {"path": "qualcomm/Shufflenet-v2", "paper": "https://arxiv.org/abs/1807.11164", "license": "Open", "year": 2018},
-    {"path": "qualcomm/MobileNet-v3-Large", "paper": "https://arxiv.org/abs/1905.02244", "license": "Open", "year": 2019},
-    {"path": "qualcomm/Swin-Small", "paper": "https://arxiv.org/abs/2103.14030", "license": "Open", "year": 2021},
-    {"path": "qualcomm/ResNet50", "paper": "https://arxiv.org/abs/1512.03385", "license": "Open", "year": 2015},
-    {"path": "qualcomm/ResNeXt50", "paper": "https://arxiv.org/abs/1611.05431", "license": "Open", "year": 2016},
-    {"path": "qualcomm/VIT", "paper": "https://arxiv.org/abs/2010.11929", "license": "Open", "year": 2020},
-    {"path": "qualcomm/EfficientViT-l2-cls", "paper": "https://arxiv.org/abs/2205.14756", "license": "Open", "year": 2022},
-    {"path": "qualcomm/EfficientNet-B4", "paper": "https://arxiv.org/abs/1905.11946", "license": "Open", "year": 2019},
-    {"path": "qualcomm/EfficientViT-b2-cls", "paper": "https://arxiv.org/abs/2205.14756", "license": "Open", "year": 2022},
-    {"path": "qualcomm/EfficientNet-V2-s", "paper": "https://arxiv.org/abs/2104.00298", "license": "Open", "year": 2021},
-    {"path": "qualcomm/ConvNext-Base", "paper": "https://arxiv.org/abs/2201.03545", "license": "Open", "year": 2022},
-    {"path": "qualcomm/Beit", "paper": "https://arxiv.org/abs/2106.08254", "license": "Open", "year": 2021},
-    {"path": "qualcomm/LeViT", "paper": "https://arxiv.org/abs/2104.01136", "license": "Open", "year": 2021},
-    {"path": "qualcomm/NASNet", "paper": "https://arxiv.org/abs/1707.07012", "license": "Open", "year": 2017},
-    {"path": "qualcomm/DLA-102-X", "paper": "https://arxiv.org/abs/1707.06484", "license": "Open", "year": 2017},
-    {"path": "qualcomm/Mobile-VIT", "paper": "https://arxiv.org/abs/2110.02178", "license": "Open", "year": 2021},
-    {"path": "qualcomm/SqueezeNet-1.1", "paper": "https://arxiv.org/abs/1602.07360", "license": "Open", "year": 2016},
-    {"path": "qualcomm/EfficientFormer", "paper": "https://arxiv.org/abs/2206.01191", "license": "Open", "year": 2022},
-
-    {"path": "Intel/vit-base-patch16-224-int8-static-inc", "paper": "https://arxiv.org/abs/2010.11929", "license": "Apache-2.0", "year": 2020}
+    {"path": "nvidia/mamba_vision_160k_ade20k-512x512_base", "paper": "https://arxiv.org/abs/2407.08083", "license": "NVIDIA Source Code License-NC", "year": 2024},
+    {"path": "nvidia/mamba_vision_160k_ade20k-512x512_small", "paper": "https://arxiv.org/abs/2407.08083", "license": "NVIDIA Source Code License-NC", "year": 2024},
+    {"path": "nvidia/mamba_vision_160k_ade20k-512x512_tiny", "paper": "https://arxiv.org/abs/2407.08083", "license": "NVIDIA Source Code License-NC", "year": 2024},
+    {"path": "nvidia/MambaVision-L2-1K", "paper": "https://arxiv.org/abs/2407.08083", "license": "NVIDIA Source Code License-NC", "year": 2024},
+    {"path": "nvidia/MambaVision-L-1K", "paper": "https://arxiv.org/abs/2407.08083", "license": "NVIDIA Source Code License-NC", "year": 2024},
+    {"path": "nvidia/MambaVision-B-1K", "paper": "https://arxiv.org/abs/2407.08083", "license": "NVIDIA Source Code License-NC", "year": 2024},
+    {"path": "nvidia/MambaVision-T-1K", "paper": "https://arxiv.org/abs/2407.08083", "license": "NVIDIA Source Code License-NC", "year": 2024},
+    {"path": "nvidia/MambaVision-T2-1K", "paper": "https://arxiv.org/abs/2407.08083", "license": "NVIDIA Source Code License-NC", "year": 2024},
+    {"path": "nvidia/MambaVision-S-1K", "paper": "https://arxiv.org/abs/2407.08083", "license": "NVIDIA Source Code License-NC", "year": 2024},
+
+    {"path": "OpenGVLab/internimage_t_1k_224", "paper": "https://arxiv.org/abs/2211.05778", "license": "Open", "year": 2022},
+    {"path": "OpenGVLab/internimage_s_1k_224", "paper": "https://arxiv.org/abs/2211.05778", "license": "Open", "year": 2022},
+    {"path": "OpenGVLab/internimage_b_1k_224", "paper": "https://arxiv.org/abs/2211.05778", "license": "Open", "year": 2022},
+    {"path": "OpenGVLab/internimage_l_22kto1k_384", "paper": "https://arxiv.org/abs/2211.05778", "license": "Open", "year": 2022},
+    {"path": "OpenGVLab/internimage_xl_22kto1k_384", "paper": "https://arxiv.org/abs/2211.05778", "license": "Open", "year": 2022},
+    {"path": "OpenGVLab/internimage_h_22kto1k_640", "paper": "https://arxiv.org/abs/2211.05778", "license": "Open", "year": 2022},
+    {"path": "OpenGVLab/internimage_g_22kto1k_512", "paper": "https://arxiv.org/abs/2211.05778", "license": "Open", "year": 2022}
+
+    
 ]