add-example-fix-bug

VADER-VideoCrafter/scripts/main/train_t2v_lora.py

@@ -573,54 +573,11 @@ def run_training(args, peft_model, **kwargs):
         gradient_accumulation_steps=args.gradient_accumulation_steps,
         mixed_precision=args.mixed_precision,
         project_dir=args.project_dir
-        
     )
     output_dir = args.project_dir
 
     # Make one log on every process with the configuration for debugging.
     create_logging(logging, logger, accelerator)
-
-    # ## ------------------------step 2: model config-----------------------------
-    # # download the checkpoint for VideoCrafter2 model
-    # ckpt_dir = args.ckpt_path.split('/')    # args.ckpt='checkpoints/base_512_v2/model.ckpt' -> 'checkpoints/base_512_v2'
-    # ckpt_dir = '/'.join(ckpt_dir[:-1])
-    # snapshot_download(repo_id='VideoCrafter/VideoCrafter2', local_dir =ckpt_dir)
-    
-    # # load the model
-    # config = OmegaConf.load(args.config)
-    # model_config = config.pop("model", OmegaConf.create())
-    # model = instantiate_from_config(model_config)
-
-    # assert os.path.exists(args.ckpt_path), f"Error: checkpoint [{args.ckpt_path}] Not Found!"
-    # model = load_model_checkpoint(model, args.ckpt_path)
-
-
-    # # convert first_stage_model and cond_stage_model to torch.float16 if mixed_precision is True
-    # if args.mixed_precision != 'no':
-    #     model.first_stage_model = model.first_stage_model.half()
-    #     model.cond_stage_model = model.cond_stage_model.half()
-
-    # # step 2.1: add LoRA using peft
-    # config = peft.LoraConfig(
-    #         r=args.lora_rank,
-    #         target_modules=["to_k", "to_v", "to_q"],        # only diffusion_model has these modules
-    #         lora_dropout=0.01,
-    #     )
-    
-    # peft_model = peft.get_peft_model(model, config)
-
-    # peft_model.print_trainable_parameters()
-
-    # # load the pretrained LoRA model
-    # if args.lora_ckpt_path is not None:
-    #     if args.lora_ckpt_path == "huggingface-hps-aesthetic":  # download the pretrained LoRA model from huggingface
-    #         snapshot_download(repo_id='zheyangqin/VADER', local_dir ='VADER-VideoCrafter/checkpoints/pretrained_lora')
-    #         args.lora_ckpt_path = 'VADER-VideoCrafter/checkpoints/pretrained_lora/vader_videocrafter_hps_aesthetic.pt'
-    #     elif args.lora_ckpt_path == "huggingface-pickscore":    # download the pretrained LoRA model from huggingface
-    #         snapshot_download(repo_id='zheyangqin/VADER', local_dir ='VADER-VideoCrafter/checkpoints/pretrained_lora')
-    #         args.lora_ckpt_path = 'VADER-VideoCrafter/checkpoints/pretrained_lora/vader_videocrafter_pickscore.pt'
-    #     # load the pretrained LoRA model
-    #     peft.set_peft_model_state_dict(peft_model, torch.load(args.lora_ckpt_path))
     
     # Inference Step: only do inference and save the videos. Skip this step if it is training
     # ==================================================================
@@ -749,7 +706,7 @@ def setup_model(lora_ckpt_path="huggingface-pickscore", lora_rank=16):
     # download the checkpoint for VideoCrafter2 model
     ckpt_dir = args.ckpt_path.split('/')    # args.ckpt='checkpoints/base_512_v2/model.ckpt' -> 'checkpoints/base_512_v2'
     ckpt_dir = '/'.join(ckpt_dir[:-1])
-    snapshot_download(repo_id='VideoCrafter/VideoCrafter2', local_dir =ckpt_dir)
+    snapshot_download(repo_id='VideoCrafter/VideoCrafter2', local_dir=ckpt_dir)
     
     # load the model
     config = OmegaConf.load(args.config)
@@ -766,7 +723,7 @@ def setup_model(lora_ckpt_path="huggingface-pickscore", lora_rank=16):
     
     # step 2.1: add LoRA using peft
     config = peft.LoraConfig(
-            r=args.lora_rank,
+            r=lora_rank,
             target_modules=["to_k", "to_v", "to_q"],        # only diffusion_model has these modules
             lora_dropout=0.01,
         )
@@ -783,6 +740,14 @@ def setup_model(lora_ckpt_path="huggingface-pickscore", lora_rank=16):
         elif lora_ckpt_path == "huggingface-pickscore":    # download the pretrained LoRA model from huggingface
             snapshot_download(repo_id='zheyangqin/VADER', local_dir ='VADER-VideoCrafter/checkpoints/pretrained_lora')
             lora_ckpt_path = 'VADER-VideoCrafter/checkpoints/pretrained_lora/vader_videocrafter_pickscore.pt'
+        elif lora_ckpt_path == "peft_model_532":
+            lora_ckpt_path = 'VADER-VideoCrafter/checkpoints/pretrained_lora/peft_model_532.pt'
+        elif lora_ckpt_path == "peft_model_548":
+            lora_ckpt_path = 'VADER-VideoCrafter/checkpoints/pretrained_lora/peft_model_548.pt'
+        elif lora_ckpt_path == "peft_model_536":
+            lora_ckpt_path = 'VADER-VideoCrafter/checkpoints/pretrained_lora/peft_model_536.pt'
+        elif lora_ckpt_path == "peft_model_400":
+            lora_ckpt_path = 'VADER-VideoCrafter/checkpoints/pretrained_lora/peft_model_400.pt'
         # load the pretrained LoRA model
         peft.set_peft_model_state_dict(peft_model, torch.load(lora_ckpt_path))
 

app.py

@@ -1,15 +1,27 @@
 import gradio as gr
 import os
-
+import spaces
 import sys
 sys.path.append('./VADER-VideoCrafter/scripts/main')
 sys.path.append('./VADER-VideoCrafter/scripts')
 sys.path.append('./VADER-VideoCrafter')
 
+
 from train_t2v_lora import main_fn, setup_model
 
+examples = [
+    ["A fairy tends to enchanted, glowing flowers.", 'huggingface-hps-aesthetic', 8, 400, 384, 512, 12.0, 25, 1.0, 24, 10],
+    ["A cat playing an electric guitar in a loft with industrial-style decor and soft, multicolored lights.", 'huggingface-hps-aesthetic', 8, 206, 384, 512, 12.0, 25, 1.0, 24, 10],
+    ["A raccoon playing a guitar under a blossoming cherry tree.", 'huggingface-hps-aesthetic', 8, 204, 384, 512, 12.0, 25, 1.0, 24, 10],
+    ["A mermaid with flowing hair and a shimmering tail discovers a hidden underwater kingdom adorned with coral palaces, glowing pearls, and schools of colorful fish, encountering both wonders and dangers along the way.", 
+     "huggingface-pickscore", 16, 205, 384, 512, 12.0, 25, 1.0, 24, 10],
+    ["A talking bird with shimmering feathers and a melodious voice leads an adventure to find a legendary treasure, guiding through enchanted forests, ancient ruins, and mystical challenges.",
+     "huggingface-pickscore", 16, 204, 384, 512, 12.0, 25, 1.0, 24, 10]
+]
+
 model = None # Placeholder for model
 
+@spaces.GPU(duration=70)
 def gradio_main_fn(prompt, seed, height, width, unconditional_guidance_scale, ddim_steps, ddim_eta,
                    frames, savefps):
     global model
@@ -30,7 +42,7 @@ def gradio_main_fn(prompt, seed, height, width, unconditional_guidance_scale, dd
 
 def reset_fn():
     return ("A mermaid with flowing hair and a shimmering tail discovers a hidden underwater kingdom adorned with coral palaces, glowing pearls, and schools of colorful fish, encountering both wonders and dangers along the way.", 
-            200, 320, 512, 12.0, 25, 1.0, 24, 16, 10, "huggingface-pickscore")
+            200, 384, 512, 12.0, 25, 1.0, 24, 16, 10, "huggingface-pickscore")
 
 def update_lora_rank(lora_model):
     if lora_model == "huggingface-pickscore":
@@ -38,7 +50,7 @@ def update_lora_rank(lora_model):
     elif lora_model == "huggingface-hps-aesthetic":
         return gr.update(value=8)
     else: # "Base Model"
-        return gr.update(value=0)
+        return gr.update(value=8)
 
 def update_dropdown(lora_rank):
     if lora_rank == 16:
@@ -48,7 +60,7 @@ def update_dropdown(lora_rank):
     else: # 0
         return gr.update(value="Base Model")
 
-
+@spaces.GPU(duration=120)
 def setup_model_progress(lora_model, lora_rank):
     global model
 
@@ -60,15 +72,58 @@ def setup_model_progress(lora_model, lora_rank):
     # Enable buttons after loading and update indicator
     yield (gr.update(interactive=True), gr.update(interactive=True), gr.update(interactive=True), "Model loaded successfully")
 
-css = """
-.centered {
-    display: flex;
-    justify-content: center;
-}
-"""
+@spaces.GPU(duration=120)
+def generate_example(prompt, lora_model, lora_rank, seed, height, width, unconditional_guidance_scale, ddim_steps, ddim_eta,
+                   frames, savefps):
+    global model
+    model = setup_model(lora_model, lora_rank)
+    video_path = main_fn(prompt=prompt,
+                        seed=int(seed),
+                        height=int(height), 
+                        width=int(width), 
+                        unconditional_guidance_scale=float(unconditional_guidance_scale), 
+                        ddim_steps=int(ddim_steps), 
+                        ddim_eta=float(ddim_eta), 
+                        frames=int(frames),  
+                        savefps=int(savefps),
+                        model=model)
+    return video_path
 
+custom_css = """
+    #centered {
+        display: flex;
+        justify-content: center;
+    }
+    .column-centered {
+        display: flex;
+        flex-direction: column;
+        align-items: center;
+        width: 60%;
+    }
+    #image-upload {
+        flex-grow: 1;
+    }
+    #params .tabs {
+        display: flex;
+        flex-direction: column;
+        flex-grow: 1;
+    }
+    #params .tabitem[style="display: block;"] {
+        flex-grow: 1;
+        display: flex !important;
+    }
+    #params .gap {
+        flex-grow: 1;
+    }
+    #params .form {
+        flex-grow: 1 !important;
+    }
+    #params .form > :last-child{
+        flex-grow: 1;
+    }
+"""
 
-with gr.Blocks(css=css) as demo:
+with gr.Blocks(css=custom_css) as demo:
     with gr.Row():
         with gr.Column():
             gr.HTML(
@@ -152,21 +207,23 @@ with gr.Blocks(css=css) as demo:
                 """
             )
 
-    with gr.Row(elem_classes="centered"):
-        with gr.Column(scale=0.6):
-            output_video = gr.Video()
-
-            with gr.Row():
-                lora_model = gr.Dropdown(
-                    label="VADER Model",
-                    choices=["huggingface-pickscore", "huggingface-hps-aesthetic", "Base Model"],
-                    value="huggingface-pickscore"
-                )
-                lora_rank = gr.Slider(minimum=0, maximum=16, label="LoRA Rank", step = 8, value=16)
+    with gr.Row(elem_id="centered"):
+        with gr.Column(scale=0.3, elem_id="params"):
+            lora_model = gr.Dropdown(
+                label="VADER Model",
+                choices=["huggingface-pickscore", "huggingface-hps-aesthetic", "Base Model"],
+                value="huggingface-pickscore"
+            )
+            lora_rank = gr.Slider(minimum=8, maximum=16, label="LoRA Rank", step = 8, value=16)
             load_btn = gr.Button("Load Model")
             # Add a label to show the loading indicator
             loading_indicator = gr.Label(value="", label="Loading Indicator")
+
+        with gr.Column(scale=0.3):
+            output_video = gr.Video(elem_id="image-upload")
             
+    with gr.Row(elem_id="centered"):
+        with gr.Column(scale=0.6):      
             prompt = gr.Textbox(placeholder="Enter prompt text here", lines=4, label="Text Prompt",
                                 value="A mermaid with flowing hair and a shimmering tail discovers a hidden underwater kingdom adorned with coral palaces, glowing pearls, and schools of colorful fish, encountering both wonders and dangers along the way.")
 
@@ -176,7 +233,7 @@ with gr.Blocks(css=css) as demo:
 
             
             with gr.Row():
-                height = gr.Slider(minimum=0, maximum=1024, label="Height", step = 16, value=320)
+                height = gr.Slider(minimum=0, maximum=1024, label="Height", step = 16, value=384)
                 width = gr.Slider(minimum=0, maximum=1024, label="Width", step = 16, value=512)
 
             with gr.Row():
@@ -205,6 +262,12 @@ with gr.Blocks(css=css) as demo:
             lora_model.change(fn=update_lora_rank, inputs=lora_model, outputs=lora_rank)
             lora_rank.change(fn=update_dropdown, inputs=lora_rank, outputs=lora_model)
 
-demo.launch()
+            gr.Examples(examples=examples,
+                    inputs=[prompt, lora_model, lora_rank, seed, height, width, unconditional_guidance_scale, DDIM_Steps, DDIM_Eta, frames, savefps],
+                    outputs=output_video,
+                    fn=generate_example,
+                    run_on_click=False,
+                    cache_examples=True,
+                    )
 
-# main_fn(prompt="A mermaid with flowing hair and a shimmering tail discovers a hidden underwater kingdom adorned with coral palaces, glowing pearls, and schools of colorful fish, encountering both wonders and dangers along the way.",)
+demo.launch(share=True)

gradio_cached_examples/34/log.csv

@@ -0,0 +1,6 @@
+component 0,flag,username,timestamp
+"{""video"": {""path"": ""gradio_cached_examples/34/component 0/098dac4a3713d5d7c6a8/temporal.mp4"", ""url"": ""/file=/tmp/gradio/4bc133becbc469de8da700250f7f7df1103c6f56/temporal.mp4"", ""size"": null, ""orig_name"": ""temporal.mp4"", ""mime_type"": null, ""is_stream"": false, ""meta"": {""_type"": ""gradio.FileData""}}, ""subtitles"": null}",,,2024-07-18 22:50:14.868519
+"{""video"": {""path"": ""gradio_cached_examples/34/component 0/b32c2706faa4801becfc/temporal.mp4"", ""url"": ""/file=/tmp/gradio/7f62f2e865f6a6eef4c27968ad35c3102d6ba5a4/temporal.mp4"", ""size"": null, ""orig_name"": ""temporal.mp4"", ""mime_type"": null, ""is_stream"": false, ""meta"": {""_type"": ""gradio.FileData""}}, ""subtitles"": null}",,,2024-07-18 22:51:57.454233
+"{""video"": {""path"": ""gradio_cached_examples/34/component 0/0ced86d109f80abd1456/temporal.mp4"", ""url"": ""/file=/tmp/gradio/2af48d5977a6b60b9c91982ef479e44a2ce2bd42/temporal.mp4"", ""size"": null, ""orig_name"": ""temporal.mp4"", ""mime_type"": null, ""is_stream"": false, ""meta"": {""_type"": ""gradio.FileData""}}, ""subtitles"": null}",,,2024-07-18 22:53:33.714132
+"{""video"": {""path"": ""gradio_cached_examples/34/component 0/b3018d4fa1632c5c33d3/temporal.mp4"", ""url"": ""/file=/tmp/gradio/50c4df5d030c66ff3f75b5f427bb6ef42eb20597/temporal.mp4"", ""size"": null, ""orig_name"": ""temporal.mp4"", ""mime_type"": null, ""is_stream"": false, ""meta"": {""_type"": ""gradio.FileData""}}, ""subtitles"": null}",,,2024-07-18 22:55:14.236468
+"{""video"": {""path"": ""gradio_cached_examples/34/component 0/73648e9d504425f92839/temporal.mp4"", ""url"": ""/file=/tmp/gradio/469d6c7ffc22a14449337ee8c966b3a517d581a3/temporal.mp4"", ""size"": null, ""orig_name"": ""temporal.mp4"", ""mime_type"": null, ""is_stream"": false, ""meta"": {""_type"": ""gradio.FileData""}}, ""subtitles"": null}",,,2024-07-18 22:56:46.543720

requirements.txt

@@ -9,7 +9,7 @@ Pillow==9.5.0
 pytorch_lightning==2.3.1
 PyYAML==6.0
 setuptools==65.6.3
-tqdm==4.65.0
+tqdm>=4.66.3
 transformers==4.25.1
 moviepy==1.0.3
 av==12.2.0
@@ -27,4 +27,5 @@ wandb==0.17.3
 ipdb==0.13.13
 huggingface-hub==0.23.4
 gradio
+spaces
 -e git+https://github.com/tgxs002/HPSv2.git#egg=hpsv2