update theme

app.py

@@ -12,6 +12,7 @@ from ultralytics import YOLO
 from pathlib import Path
 import argparse
 import json
+import trimesh
 from torchvision import transforms
 from typing import Dict, Optional
 from PIL import Image, ImageDraw
@@ -175,13 +176,17 @@ def run_model(image, conf, IoU_threshold=0.5):
             
             reconstructions = {'verts': verts, 'palm': palm, 'objtrans': objtrans, 'objpcs': pointclouds_up, 'cam_t': cam_t, 'right': is_right, 'img_size': 224, 'focal': scaled_focal_length}
 
-        return crop_img_cv2[..., ::-1].astype(np.float32) / 255.0, len(detections), reconstructions
-    else: 
-        return crop_img_cv2[..., ::-1].astype(np.float32) / 255.0, len(detections), None       
+            camera_translation = cam_t.copy()
+            hand_mesh = renderer.mesh(verts, camera_translation, LIGHT_PURPLE, is_right=is_right)
+            obj_pcd = trimesh.PointCloud(reconstructions['objpcs'] + reconstructions['palm'] + reconstructions['objtrans'] + camera_translation, colors=[70, 130, 180, 255])
+            scene = trimesh.Scene([hand_mesh, obj_pcd])
+            scene_path = "/tmp/test.glb"
+            scene.export(scene_path)
 
+        return crop_img_cv2[..., ::-1].astype(np.float32) / 255.0, len(detections), reconstructions, scene_path
 
 def render_reconstruction(image, conf, IoU_threshold=0.3): 
-    input_img, num_dets, reconstructions = run_model(image, conf, IoU_threshold=0.5)
+    input_img, num_dets, reconstructions, scene_path = run_model(image, conf, IoU_threshold=0.5)
     # Render front view
     misc_args = dict(mesh_base_color=LIGHT_PURPLE, point_base_color=STEEL_BLUE, scene_bg_color=(1, 1, 1), focal_length=reconstructions['focal'])
     cam_view = renderer.render_rgba(reconstructions['verts'], reconstructions['objpcs'] + reconstructions['palm'] + reconstructions['objtrans'], cam_t=reconstructions['cam_t'], render_res=(224, 224), is_right=True, **misc_args)
@@ -190,7 +195,7 @@ def render_reconstruction(image, conf, IoU_threshold=0.3):
     input_img = np.concatenate([input_img, np.ones_like(input_img[:,:,:1])], axis=2) # Add alpha channel
     input_img_overlay = input_img[:,:,:3] * (1-cam_view[:,:,3:]) + cam_view[:,:,:3] * cam_view[:,:,3:]
 
-    return input_img_overlay, f'{num_dets} hands detected'  
+    return input_img_overlay, f'{num_dets} hands detected', scene_path  
 
 
 header = ('''
@@ -215,8 +220,12 @@ header = ('''
 <a href='https://github.com/zerchen/hort'><img src='https://img.shields.io/badge/GitHub-Code-black?style=flat&logo=github&logoColor=white'></a> 
 ''')
 
-
-with gr.Blocks(title="HORT: Monocular Hand-held Objects Reconstruction with Transformers", css=".gradio-container") as demo:
+theme = gr.themes.Ocean()
+theme.set(
+    checkbox_label_background_fill_selected="*button_primary_background_fill",
+    checkbox_label_text_color_selected="*button_primary_text_color",
+)
+with gr.Blocks(theme=theme, title="HORT: Monocular Hand-held Objects Reconstruction with Transformers", css=".gradio-container") as demo:
 
     gr.Markdown(header)
 
@@ -229,9 +238,10 @@ with gr.Blocks(title="HORT: Monocular Hand-held Objects Reconstruction with Tran
         
         with gr.Column():
             reconstruction = gr.Image(label="Reconstructions", type="numpy")
+            output_meshes = gr.Model3D(height=300, zoom_speed=0.5, pan_speed=0.5)
             hands_detected = gr.Textbox(label="Hands Detected")
     
-        submit.click(fn=render_reconstruction, inputs=[input_image, threshold], outputs=[reconstruction, hands_detected])
+        submit.click(fn=render_reconstruction, inputs=[input_image, threshold], outputs=[reconstruction, hands_detected, output_meshes])
 
     with gr.Row():
         example_images = gr.Examples([

hort/utils/renderer.py

@@ -280,6 +280,14 @@ class Renderer:
         mesh.apply_transform(rot)
         return mesh
 
+    def mesh(self, vertices, camera_translation, mesh_base_color=(1.0, 1.0, 0.9), rot_axis=[1,0,0], rot_angle=0, is_right=1):
+        vertex_colors = np.array([(*mesh_base_color, 1.0)] * vertices.shape[0])
+        if is_right:
+            mesh = trimesh.Trimesh(vertices.copy() + camera_translation, self.faces.copy(), vertex_colors=vertex_colors)
+        else:
+            mesh = trimesh.Trimesh(vertices.copy() + camera_translation, self.faces_left.copy(), vertex_colors=vertex_colors)
+        return mesh
+
     def render_rgba(
             self,
             vertices: np.array,