Upload 6 files

README.md

@@ -1,3 +1,60 @@
----
-license: mit
----
+---
+license: mit
+base_model:
+- black-forest-labs/FLUX.1-dev
+---
+
+# DiT360: High-Fidelity Panoramic Image Generation via Hybrid Training
+
+<p align="center"><a href="https://arxiv.org/abs/2412.01506"><img src='https://img.shields.io/badge/arXiv-Paper-red?logo=arxiv&logoColor=white' alt='arXiv'></a>
+<a href='https://fenghora.github.io/DiT360-Page/'><img src='https://img.shields.io/badge/Project_Page-Website-green?logo=insta360&logoColor=white' alt='Project Page'></a>
+<a href='https://huggingface.co/spaces/Insta360-Research/DiT360'><img src='https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Live_Demo-blue'></a>
+</p>
+
+![teaser](assets/teaser.jpg)
+
+**DiT360** is a framework for high-quality panoramic image generation, leveraging both **perspective** and **panoramic** data in a hybrid training scheme.
+It adopts a two-level strategy—**image-level cross-domain guidance** and **token-level hybrid supervision**—to enhance perceptual realism and geometric fidelity.
+
+## 🔨 Installation
+
+Clone the repo first:
+
+```Bash
+git clone https://github.com/Insta360-Research-Team/DiT360.git
+cd DiT360
+```
+
+(Optional) Create a fresh conda env:
+
+```Bash
+conda create -n dit360 python=3.12
+conda activate dit360
+```
+
+Install necessary packages (torch > 2):
+
+```Bash
+# pytorch (select correct CUDA version, we test our code on torch==2.6.0 and torchvision==0.21.0)
+pip install torch==2.6.0 torchvision==0.21.0
+
+# other dependencies
+pip install -r requirements.txt
+```
+
+## 📒 Quick Start
+
+```Bash
+python inference.py
+```
+
+## 🤝 Acknowledgement
+
+We appreciate the open source of the following projects:
+
+* [diffusers](https://github.com/huggingface/diffusers)
+
+## Citation
+```
+
+```

adapter_config.json

@@ -0,0 +1,39 @@
+{
+  "task_type": null,
+  "peft_type": "LORA",
+  "auto_mapping": null,
+  "base_model_name_or_path": null,
+  "revision": null,
+  "inference_mode": false,
+  "r": 64,
+  "target_modules": [
+    "attn.to_q",
+    "attn.to_v",
+    "attn.to_k",
+    "attn.to_out.0"
+  ],
+  "exclude_modules": null,
+  "lora_alpha": 64,
+  "lora_dropout": 0.05,
+  "fan_in_fan_out": false,
+  "bias": "none",
+  "use_rslora": false,
+  "modules_to_save": null,
+  "init_lora_weights": "gaussian",
+  "layers_to_transform": null,
+  "layers_pattern": null,
+  "rank_pattern": {},
+  "alpha_pattern": {},
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "trainable_token_indices": null,
+  "loftq_config": {},
+  "eva_config": null,
+  "corda_config": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "qalora_group_size": 16,
+  "layer_replication": null,
+  "lora_bias": false,
+  "target_parameters": null
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:130b1f09b1c35f97a06c15f3ee9677289e426303db9501aa39271fe7db329380
+size 149472912

gitattributes

@@ -0,0 +1,38 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+adapter_model.safetensors filter=lfs diff=lfs merge=lfs -text
+*.jpg filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text

inference.py

@@ -0,0 +1,16 @@
+from src.pipeline import DiT360Pipeline
+import torch
+
+device = torch.device("cuda:0")
+pipe = DiT360Pipeline.from_pretrained("black-forest-labs/FLUX.1-dev", dtype=torch.float16).to(device)
+pipe.load_lora_weights("fenghora/DiT360-Panorama-Image-Generation")
+
+image = pipe(
+    "This is a panorama. The image shows a medieval castle stands proudly on a hilltop surrounded by autumn forests, with golden light spilling across the landscape.",
+    width=2048,
+    height=1024,
+    num_inference_steps=28,
+    guidance_scale=2.8,
+    generator=torch.Generator(device=device).manual_seed(0),
+).images[0]
+image.save("result.png")

requirements.txt

@@ -0,0 +1,6 @@
+git+https://ghfast.top/https://github.com/huggingface/diffusers # to avoid https://github.com/huggingface/diffusers/issues/12436
+transformers
+accelerate
+peft
+protobuf
+sentencepiece