Initial commit

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,84 @@
+---
+license: mit
+pipeline_tag: video-classification
+tags:
+- video
+---
+
+# V-JEPA 2
+
+A frontier video understanding model developed by FAIR, Meta, which extends the pretraining objectives of [VJEPA](https://ai.meta.com/blog/v-jepa-yann-lecun-ai-model-video-joint-embedding-predictive-architecture/), resulting in state-of-the-art video understanding capabilities, leveraging data and model sizes at scale.
+The code is released [in this repository](https://github.com/facebookresearch/vjepa2).
+
+<img src="https://dl.fbaipublicfiles.com/vjepa2/vjepa2-pretrain.gif">&nbsp;
+
+## Installation
+
+To run V-JEPA 2 model, ensure you have installed the latest transformers:
+
+```bash
+pip install -U git+https://github.com/huggingface/transformers
+```
+
+## Intended Uses
+
+V-JEPA 2 is intended to represent any video (and image) to perform video classification, retrieval, or as a video encoder for VLMs.
+
+```python
+from transformers import AutoVideoProcessor, AutoModel
+
+hf_repo = "facebook/vjepa2-vith-fpc64-256"
+
+model = AutoModel.from_pretrained(hf_repo)
+processor = AutoVideoProcessor.from_pretrained(hf_repo)
+```
+
+To load a video, sample the number of frames according to the model. For this model, we use 64.
+
+```python
+import torch
+from torchcodec.decoders import VideoDecoder
+import numpy as np
+
+video_url = "https://huggingface.co/datasets/nateraw/kinetics-mini/resolve/main/val/archery/-Qz25rXdMjE_000014_000024.mp4"
+vr = VideoDecoder(video_url)
+frame_idx = np.arange(0, 64) # choosing some frames. here, you can define more complex sampling strategy
+video = vr.get_frames_at(indices=frame_idx).data  # T x C x H x W
+video = processor(video, return_tensors="pt").to(model.device)
+with torch.no_grad():
+    video_embeddings = model.get_vision_features(**video)
+
+print(video_embeddings.shape)
+```
+
+To load an image, simply copy the image to the desired number of frames.
+
+```python
+from transformers.image_utils import load_image
+
+image = load_image("https://huggingface.co/datasets/merve/coco/resolve/main/val2017/000000000285.jpg")
+pixel_values = processor(image, return_tensors="pt").to(model.device)["pixel_values_videos"]
+pixel_values = pixel_values.repeat(1, 16, 1, 1, 1) # repeating image 16 times
+
+with torch.no_grad():
+    image_embeddings = model.get_vision_features(pixel_values)    
+
+print(image_embeddings.shape)
+```
+
+For more code examples, please refer to the V-JEPA 2 documentation.
+
+### Citation
+
+```
+@techreport{assran2025vjepa2,
+  title={V-JEPA~2: Self-Supervised Video Models Enable Understanding, Prediction and Planning},
+  author={Assran, Mahmoud and Bardes, Adrien and Fan, David and Garrido, Quentin and Howes, Russell and
+Komeili, Mojtaba and Muckley, Matthew and Rizvi, Ammar and Roberts, Claire and Sinha, Koustuv and Zholus, Artem and
+Arnaud, Sergio and Gejji, Abha and Martin, Ada and Robert Hogan, Francois and Dugas, Daniel and
+Bojanowski, Piotr and Khalidov, Vasil and Labatut, Patrick and Massa, Francisco and Szafraniec, Marc and
+Krishnakumar, Kapil and Li, Yong and Ma, Xiaodong and Chandar, Sarath and Meier, Franziska and LeCun, Yann and
+Rabbat, Michael and Ballas, Nicolas},
+  institution={FAIR at Meta},
+  year={2025}
+}

config.json

@@ -0,0 +1,33 @@
+{
+  "architectures": [
+    "VJEPA2Model"
+  ],
+  "attention_probs_dropout_prob": 0.0,
+  "crop_size": 256,
+  "drop_path_rate": 0.0,
+  "frames_per_clip": 64,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.0,
+  "hidden_size": 1280,
+  "image_size": 256,
+  "in_chans": 3,
+  "initializer_range": 0.02,
+  "layer_norm_eps": 1e-06,
+  "mlp_ratio": 4,
+  "model_type": "vjepa2",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 32,
+  "patch_size": 16,
+  "pred_hidden_size": 384,
+  "pred_mlp_ratio": 4.0,
+  "pred_num_attention_heads": 12,
+  "pred_num_hidden_layers": 12,
+  "pred_num_mask_tokens": 10,
+  "pred_zero_init_mask_tokens": true,
+  "qkv_bias": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.53.0.dev0",
+  "tubelet_size": 2,
+  "use_SiLU": false,
+  "wide_SiLU": true
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ad0d49df6565ffeb7f837023ee149e27362d24c06332fa4a504e169d86b16eb0
+size 2615800136

notebook.ipynb

@@ -0,0 +1,157 @@
+{
+  "nbformat": 4,
+  "nbformat_minor": 0,
+  "metadata": {
+    "colab": {
+      "provenance": [],
+      "gpuType": "T4"
+    },
+    "kernelspec": {
+      "name": "python3",
+      "display_name": "Python 3"
+    },
+    "language_info": {
+      "name": "python"
+    },
+    "accelerator": "GPU"
+  },
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "source": [
+        "# Use VJEPA 2"
+      ],
+      "metadata": {
+        "id": "02ruu54h4yLc"
+      }
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "V-JEPA 2 is a new open 1.2B video embedding model by Meta, which attempts to capture the physical world modelling through video ⏯️\n",
+        "\n",
+        "The model can be used for various tasks for video: fine-tuning for downstream tasks like video classification, or any task involving embeddings (similarity, retrieval and more!).\n",
+        "\n",
+        "You can check all V-JEPA 2 checkpoints and the datasets that come with this release [in this collection](https://huggingface.co/collections/facebook/v-jepa-2-6841bad8413014e185b497a6)."
+      ],
+      "metadata": {
+        "id": "ol0IGYCd4hg4"
+      }
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "We need to install transformers' release specific branch."
+      ],
+      "metadata": {
+        "id": "kIIBxYOA41Ga"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "!pip install -q git+https://github.com/huggingface/[email protected]"
+      ],
+      "metadata": {
+        "id": "4D4D1hC940yX"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "from huggingface_hub import login # to later push the model\n",
+        "\n",
+        "login()"
+      ],
+      "metadata": {
+        "id": "Ne2rU68Ep1On"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "As of now, Colab supports torchcodec==0.2.1 which supports torch==2.6.0."
+      ],
+      "metadata": {
+        "id": "dJWXmFu53Ap6"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "!pip install -q torch==2.6.0 torchvision==0.21.0\n",
+        "!pip install -q torchcodec==0.2.1\n",
+        "\n",
+        "import torch\n",
+        "print(\"Torch:\", torch.__version__)\n",
+        "from torchcodec.decoders import VideoDecoder # verify"
+      ],
+      "metadata": {
+        "id": "JIoq84ze2_Ls"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "## Initialize the model and the processor"
+      ],
+      "metadata": {
+        "id": "-7OATf5S20U_"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "from transformers import AutoVideoProcessor, AutoModel\n",
+        "\n",
+        "hf_repo = \"facebook/vjepa2-vith-fpc64-256\"\n",
+        "\n",
+        "model = AutoModel.from_pretrained(hf_repo).to(\"cuda\")\n",
+        "processor = AutoVideoProcessor.from_pretrained(hf_repo)"
+      ],
+      "metadata": {
+        "id": "K8oSsy7Y2zQK"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "## Extract video embeddings from the model"
+      ],
+      "metadata": {
+        "id": "ZJ_DUR9f22Uc"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "import torch\n",
+        "from torchcodec.decoders import VideoDecoder\n",
+        "import numpy as np\n",
+        "\n",
+        "video_url = \"https://huggingface.co/datasets/nateraw/kinetics-mini/resolve/main/val/archery/-Qz25rXdMjE_000014_000024.mp4\"\n",
+        "vr = VideoDecoder(video_url)\n",
+        "frame_idx = np.arange(0, 64) # choosing some frames. here, you can define more complex sampling strategy\n",
+        "video = vr.get_frames_at(indices=frame_idx).data  # T x C x H x W\n",
+        "video = processor(video, return_tensors=\"pt\").to(model.device)\n",
+        "with torch.no_grad():\n",
+        "    video_embeddings = model.get_vision_features(**video)\n",
+        "\n",
+        "print(video_embeddings.shape)"
+      ],
+      "metadata": {
+        "id": "kAgWZJHt24px"
+      },
+      "execution_count": null,
+      "outputs": []
+    }
+  ]
+}

original/model.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f18bab855c310d75fac12bfdb64f2dc2e2e048979f3fb9312c1d97a713b33cd
+size 10374445514

video_preprocessor_config.json

@@ -0,0 +1,71 @@
+{
+  "_valid_kwargs_names": [
+    "do_convert_rgb",
+    "do_resize",
+    "size",
+    "size_divisor",
+    "default_to_square",
+    "resample",
+    "do_rescale",
+    "rescale_factor",
+    "do_normalize",
+    "image_mean",
+    "image_std",
+    "do_pad",
+    "do_center_crop",
+    "crop_size",
+    "data_format",
+    "input_data_format",
+    "device"
+  ],
+  "crop_size": {
+    "height": 256,
+    "width": 256
+  },
+  "data_format": "channels_first",
+  "default_to_square": true,
+  "device": null,
+  "do_center_crop": true,
+  "do_convert_rgb": null,
+  "do_normalize": true,
+  "do_pad": null,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.485,
+    0.456,
+    0.406
+  ],
+  "image_std": [
+    0.229,
+    0.224,
+    0.225
+  ],
+  "input_data_format": null,
+  "model_valid_processing_keys": [
+    "do_convert_rgb",
+    "do_resize",
+    "size",
+    "size_divisor",
+    "default_to_square",
+    "resample",
+    "do_rescale",
+    "rescale_factor",
+    "do_normalize",
+    "image_mean",
+    "image_std",
+    "do_pad",
+    "do_center_crop",
+    "crop_size",
+    "data_format",
+    "input_data_format",
+    "device"
+  ],
+  "resample": 2,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "shortest_edge": 292
+  },
+  "size_divisor": null,
+  "video_processor_type": "VJEPA2VideoProcessor"
+}