update

app.py

@@ -102,7 +102,7 @@ def viewcrafter_demo(opts):
             )
             # step 2 - camera trajectory generation
             gr.Markdown("---\n## Step 2: Input camera trajectory", show_label=False, visible=True)
-            gr.Markdown("<div align='left' style='font-size:18px;color: #000000'> Input a d_phi sequence, a d_theta sequence, and a d_r sequence to generate a camera trajectory. In the sequences, a positive d_phi moves the camera to the right, a negative d_theta moves the camera up, and a negative d_r moves the camera forward. Ensure that each sequence starts with 0 and contains at least two elements (a start and an end). </div>")
+            gr.Markdown("<div align='left' style='font-size:18px;color: #000000'> Input a d_phi sequence, a d_theta sequence, and a d_r sequence to generate a camera trajectory. In the sequences, a positive d_phi moves the camera to the right, a negative d_theta moves the camera up, and a negative d_r moves the camera forward. Ensure that each sequence starts with 0 and contains at least two elements (a start and an end). If you upload a new image, remember to conduct this step again. </div>")
             with gr.Row():
                 with gr.Column():
                     # camera_mode = gr.Radio(choices=CAMERA_MOTION_MODE, value=CAMERA_MOTION_MODE[0], label="Camera Motion Control Mode", interactive=True, visible=False)
@@ -113,7 +113,6 @@ def viewcrafter_demo(opts):
                     # camera_info = gr.Button(value="Proceed", visible=False)      
                 with gr.Column():
                     i2v_traj_video = gr.Video(label="Camera Trajectory",elem_id="traj_vid",autoplay=True,show_share_button=True)          
-            
             gr.Examples(examples=traj_examples,
                         inputs=[i2v_d_phi, i2v_d_theta, i2v_d_r],
             )
@@ -135,7 +134,7 @@ def viewcrafter_demo(opts):
         i2v_start_btn.click(inputs=[i2v_input_image, i2v_elevation, i2v_center_scale, i2v_d_phi, i2v_d_theta, i2v_d_r],
                         outputs=[i2v_traj_video],
                         fn = image2video.run_traj
-        )        
+        )
         
         i2v_end_btn.click(inputs=[i2v_steps, i2v_seed],
                         outputs=[i2v_output_video],
@@ -148,5 +147,5 @@ def viewcrafter_demo(opts):
 viewcrafter_iface = viewcrafter_demo(opts)
 viewcrafter_iface.queue(max_size=10)
 viewcrafter_iface.launch() #fixme
-# viewcrafter_iface.launch(server_name='11.220.92.96', server_port=80, max_threads=1,debug=False)
+# viewcrafter_iface.launch(server_name='11.220.92.96', server_port=80, max_threads=10,debug=False)
 

viewcrafter.py

@@ -158,14 +158,12 @@ class ViewCrafter:
         if self.opts.mode == 'single_view_txt':
             if phi[-1]==0. and theta[-1]==0. and r[-1]==0.:
                 render_results[-1] = self.img_ori
-                
-        save_video(render_results, os.path.join(self.opts.save_dir, 'render0.mp4'))
-        self.render_results = render_results
+        # torch.Size([25, 576, 1024, 3]), [0,1]
         # save_pointcloud_with_normals([imgs[-1]], [pcd[-1]], msk=None, save_path=os.path.join(self.opts.save_dir,'pcd0.ply') , mask_pc=False, reduce_pc=False)
         # diffusion_results = self.run_diffusion(render_results)
         # save_video((diffusion_results + 1.0) / 2.0, os.path.join(self.opts.save_dir, 'diffusion0.mp4'))
 
-        return None
+        return render_results
 
     def nvs_sparse_view(self,iter):
 
@@ -373,14 +371,25 @@ class ViewCrafter:
         # self.img_ori:  torch.Size([576, 1024, 3]), [0,1]
         # self.images, self.img_ori = self.load_initial_images(image_dir=i2v_input_image)
         self.run_dust3r(input_images=self.images)
-        self.nvs_single_view(gradio=True)
+        render_results = self.nvs_single_view(gradio=True)
+        save_video(render_results, os.path.join(self.opts.save_dir, 'render0.mp4'))
         traj_dir = os.path.join(self.opts.save_dir, "viz_traj.mp4")
         return traj_dir
     
     def run_gen(self,i2v_steps, i2v_seed):
         self.opts.ddim_steps = i2v_steps
         seed_everything(i2v_seed)
+        render_dir = os.path.join(self.opts.save_dir, 'render0.mp4')
+        video = imageio.get_reader(render_dir, 'ffmpeg')
+        frames = []
+        for frame in video:
+            frame = frame / 255.0
+            frames.append(frame)
+        frames = np.array(frames)
+        ##torch.Size([25, 576, 1024, 3])
+        render_results = torch.from_numpy(frames).to(self.device).half()
+
         gen_dir = os.path.join(self.opts.save_dir, "diffusion0.mp4")
-        diffusion_results = self.run_diffusion(self.render_results)
+        diffusion_results = self.run_diffusion(render_results)
         save_video((diffusion_results + 1.0) / 2.0, os.path.join(self.opts.save_dir, 'diffusion0.mp4'))
         return gen_dir