update config

.vscode/settings.json

@@ -26,8 +26,8 @@
         "**/exporters": true,
         "**/prompt_processors": true,
 
-        "**/configs/[^cgid]*": true,
-        "**/configs/dream*": true,
+        "configs/[^cgidw]*": true,
+        "configs/dream*": true,
 
         "**/scripts/run_[^g]*": true,
         "**/scripts/[^r]*": true,

configs/dynamic_gaussian_t4d_du.yaml

@@ -0,0 +1,126 @@
+name: "dynamic-gaussian-reconstruct"
+tag: "${rmspace:${system.prompt_processor.prompt},_}"
+exp_root_dir: "outputs"
+seed: 0
+
+data_type: "dynamic-multiview-camera-datamodule"
+data:
+  train_downsample_resolution: 2
+  eval_downsample_resolution: 2
+  eval_data_interval: 1
+  camera_layout: "default"
+  initial_t0_step: 2000
+  dataroot: ???
+
+system_type: "dynamic-gaussian-splatting-du-system"
+system:
+
+  start_editing_step: 0
+  per_editing_step: 40
+
+  extent: 1.0
+  num_pts: 5000
+  invert_bg_prob: 0.5
+
+  geometry_type: "dynamic-gaussian"
+  geometry:
+    gaussian_name: "gaussian-residual"
+    gaussian:
+      position_lr_init: 0.00016
+      position_lr_final:  0.0000016
+      position_lr_delay_mult: 0.01
+      position_lr_max_steps: 30000
+      feature_lr: 0.0025
+      opacity_lr: 0.005
+      scaling_lr: 0.005
+      rotation_lr: 0.001
+      force_sphere: False
+    dynamic_flow_name: "dynamic-flow-tensor4d"
+    dynamic_flow_config:
+      n_feature_dims: 7
+      need_normalization: True
+      bbox_min: [-1, -1, -1]
+      bbox_max: [1, 1, 1]
+    flow_residual: True
+
+
+  renderer_type: "dynamic-gaussian-rasterizer"
+  renderer:
+    debug: false
+    invert_bg_prob: 0.5
+
+  material_type: "no-material" # unused
+  material:
+    n_output_dims: 0
+
+  background_type: "solid-color-background" # unused
+
+  guidance_type: "web-guidance"
+  guidance:
+    # json_path: "/root/autodl-tmp/ComfyUI/config/n3d_sdxl_elon_musk_api.json"
+    # json_path: "/root/autodl-tmp/ComfyUI/config/n3d_jugger_elon_musk_api.json"
+    json_path: "/root/autodl-tmp/ComfyUI/config/n3d_nvidia_api.json"
+    server_address: "127.0.0.1:8188"
+
+    rgb_name: "rgb.jpg"
+    cond_rgb_name: "cond_rgb.jpg"
+    server_path: "/root/autodl-tmp/ComfyUI/input"
+
+    min_step_percent: [0, 0.5, 0.2, 2000]
+    max_step_percent: [0, 0.98, 0.7, 2000]
+
+  prompt_processor_type: "stable-diffusion-prompt-processor"
+  prompt_processor:
+    pretrained_model_name_or_path: "runwayml/stable-diffusion-v1-5"
+    prompt: "Elon Musk is cooking something"
+
+  loggers:
+    wandb:
+      enable: false
+      project: 'threestudio'
+      name: None
+
+  loss:
+    lambda_l1: 1.0
+    lambda_p: 1.0
+    lambda_xyz_residual: 1.0
+    lambda_scaling_residual: 1.0
+    lambda_flow_residual: 1.0
+    lambda_G_p: 1.0
+    lambda_G_l1: 1.0
+    lambda_G_dis: 0.001
+    lambda_D: 1.0
+
+  optimizer:
+    name: Adam
+    args:
+      lr: 0.01
+      betas: [0.9, 0.99]
+      eps: 1.e-15
+    params:
+      geometry.dynamic_flow_residual:
+        lr: 0.0001
+      geometry.refine_net:
+        lr: 0.0002
+    optimizer_dis:
+      name: Adam
+      args:
+        lr: 0.01
+        betas: [0.9, 0.99]
+        eps: 1.e-15
+      params:
+        geometry.discriminator:
+          lr: 0.00001
+
+trainer:
+  max_steps: 30000
+  log_every_n_steps: 1
+  num_sanity_val_steps: 0
+  val_check_interval: 500
+  enable_progress_bar: true
+  precision: 32-true
+
+checkpoint:
+  save_last: true # save at each validation time
+  save_top_k: -1
+  every_n_train_steps: ${trainer.max_steps}

configs/dynamic_gaussian_t4d_gan_reconstruct.yaml

@@ -51,6 +51,7 @@ system:
   renderer:
     debug: false
     invert_bg_prob: 0.5
+    need_refine: false
 
   material_type: "no-material" # unused
   material:

configs/dynamic_gaussian_t4d_reconstruct.yaml

@@ -67,8 +67,9 @@ system:
 
   loss:
     lambda_l1: 1.0
-    lambda_G_p: 1.0
-    lambda_G_l1: 1.0
+    lambda_G_p: 0.0
+    lambda_G_l1: 0.0
+    lambda_flow: 1.0
 
   optimizer:
     name: Adam

configs/web_guidance/dynamic_gaussian_t4d_instruct_web.yaml

@@ -0,0 +1,17 @@
+system:
+  web_dataset:
+    downsample_resolution: 2
+    dataroot: "/root/autodl-tmp/threestudio/dataset/n3d_video_dynamic_pose_editing"
+
+  web_guidance:
+    # json_path: "/root/autodl-tmp/ComfyUI/config/n3d_sdxl_elon_musk_api.json"
+    # json_path: "/root/autodl-tmp/ComfyUI/config/n3d_jugger_elon_musk_api.json"
+    json_path: "/root/autodl-tmp/ComfyUI/config/n3d_nvidia_api.json"
+    server_address: "127.0.0.1:8188"
+
+    rgb_name: "rgb.jpg"
+    cond_rgb_name: "cond_rgb.jpg"
+    server_path: "/root/autodl-tmp/ComfyUI/input"
+
+    min_step_percent: [0, 0.5, 0.2, 2000]
+    max_step_percent: [0, 0.98, 0.7, 2000]

configs/web_guidance/dynamic_web_video.yaml

@@ -0,0 +1,16 @@
+system:
+  web_dataset:
+    downsample_resolution: 1
+    dataroot: "/root/autodl-tmp/threestudio/dataset/enerf_actor1_dynamic_editing2"
+
+  web_guidance:
+    json_path: "/root/autodl-tmp/ComfyUI/config/enerf_sdxl_temporal_ironman_api.json"
+    static_json_path: "/root/autodl-tmp/ComfyUI/config/enerf_sdxl_temporal_ironman_api_static.json"
+    server_address: "127.0.0.1:8188"
+
+    rgb_name: "/root/autodl-tmp/threestudio/.threestudio_cache/images_20231110171243"
+    cond_rgb_name: "/root/autodl-tmp/threestudio/.threestudio_cache/cond_images_20231110171243"
+    server_path: "/root/autodl-tmp/ComfyUI/input"
+
+    min_step_percent: [0, 0.5, 0.2, 2000]
+    max_step_percent: [0, 0.98, 0.7, 2000]

threestudio/scripts/run_gaussian.sh

@@ -73,9 +73,26 @@ python launch.py --config configs/dynamic_gaussian_t4d_reconstruct.yaml --train
 # dynamic recon with gan
 python launch.py --config configs/dynamic_gaussian_t4d_gan_reconstruct.yaml --train --gpu 0 data.dataroot="dataset/n3d_video_dynamic_pose" name=dynamic_gaussian_gan_reconstruct tag=n3d_dynamic_pose data.camera_layout="default" system.prompt_processor.prompt="Elon Musk wearing white shirt" system.geometry.geometry_convert_from=/root/autodl-tmp/threestudio/outputs/dynamic_gaussian_reconstruct/n3d_dynamic_pose@20231104-075533/ckpts/last.ckpt data.eval_interpolation=[0,7,50] data.eval_time_interpolation=[0.0,1.0] system.renderer.near=8.0 system.geometry.gaussian.min_opac_prune=0.005 data.online_load_image=True system.geometry.dynamic_flow_config.bbox_min=[-23,-13,4] system.geometry.dynamic_flow_config.bbox_max=[62,15,42] trainer.max_steps=10000
 # dynamic edit with gan
-python launch.py --config configs/dynamic_gaussian_t4d_instruct_web.yaml --train --gpu 0 data.dataroot="dataset/n3d_video_dynamic_pose_editing" name=dynamic_gaussian_instruct tag=n3d_dynamic_pose_editing data.camera_layout="default" system.prompt_processor.prompt="Elon Musk wearing white shirt" system.geometry.geometry_convert_from=/root/autodl-tmp/threestudio/outputs/dynamic_gaussian_gan_reconstruct/n3d_dynamic_pose@20231104-085250/ckpts/last.ckpt data.eval_interpolation=[0,7,50] data.eval_time_interpolation=[0.0,1.0] system.renderer.near=8.0 data.online_load_image=False system.geometry.dynamic_flow_config.bbox_min=[-23,-13,4] system.geometry.dynamic_flow_config.bbox_max=[62,15,42]
+python launch.py --config configs/dynamic_gaussian_t4d_instruct_web.yaml --train --gpu 1 data.dataroot="dataset/n3d_video_dynamic_pose_editing" name=dynamic_gaussian_instruct tag=n3d_dynamic_pose_editing data.camera_layout="default" system.prompt_processor.prompt="Elon Musk wearing white shirt" system.geometry.geometry_convert_from=/root/autodl-tmp/threestudio/outputs/dynamic_gaussian_gan_reconstruct/n3d_dynamic_pose@20231104-085250/ckpts/last.ckpt data.eval_interpolation=[0,7,50] data.eval_time_interpolation=[0.0,1.0] system.renderer.near=8.0 data.online_load_image=False system.geometry.dynamic_flow_config.bbox_min=[-23,-13,4] system.geometry.dynamic_flow_config.bbox_max=[62,15,42]
 # checkpoint
-python launch.py --config configs/dynamic_gaussian_t4d_instruct_web.yaml --train --gpu 0 data.dataroot="dataset/n3d_video_dynamic_pose_editing" name=dynamic_gaussian_instruct_web tag=n3d_dynamic_pose_editing data.camera_layout="default" system.prompt_processor.prompt="Elon Musk wearing white shirt" resume=/root/autodl-tmp/threestudio/outputs/dynamic_gaussian_instruct/n3d_dynamic_pose_editing@20231104-095951_1500iter/ckpts/last.ckpt system.edit_path=/root/autodl-tmp/threestudio/outputs/dynamic_gaussian_instruct/n3d_dynamic_pose_editing@20231104-102942/save/edit data.eval_interpolation=[0,7,100] data.eval_time_interpolation=[0.0,1.0] system.renderer.near=8.0 data.online_load_image=False system.geometry.dynamic_flow_config.bbox_min=[-23,-13,4] system.geometry.dynamic_flow_config.bbox_max=[62,15,42]
+python launch.py --config configs/dynamic_gaussian_t4d_instruct_web.yaml --train --gpu 1 data.dataroot="dataset/n3d_video_dynamic_pose_editing" name=dynamic_gaussian_instruct_web tag=n3d_dynamic_pose_editing data.camera_layout="default" system.prompt_processor.prompt="Elon Musk wearing white shirt" resume=/root/autodl-tmp/threestudio/outputs/dynamic_gaussian_instruct/n3d_dynamic_pose_editing@20231104-120856/ckpts/last.ckpt system.edit_path=/root/autodl-tmp/threestudio/outputs/dynamic_gaussian_instruct/n3d_dynamic_pose_editing@20231104-120856/save/edit data.eval_interpolation=[0,7,100] data.eval_time_interpolation=[0.0,1.0] system.renderer.near=8.0 data.online_load_image=False system.geometry.dynamic_flow_config.bbox_min=[-23,-13,4] system.geometry.dynamic_flow_config.bbox_max=[62,15,42]
 
 
-python launch.py --config configs/dynamic_gaussian_t4d_reconstruct.yaml --train --gpu 0 data.dataroot="dataset/enerf_actor1" name=dynamic_gaussian_reconstruct tag=enerf_actor1 data.camera_layout="default" system.prompt_processor.prompt="Elon Musk wearing white shirt" system.geometry.geometry_convert_from=/root/autodl-tmp/threestudio/dataset/enerf/actor1/background.ply data.eval_interpolation=[0,7,50] data.eval_time_interpolation=[0.0,0.0] system.renderer.near=1.0 system.geometry.gaussian.min_opac_prune=0.05 data.online_load_image=False system.geometry.dynamic_flow_config.bbox_min=[-5,-5,-5] system.geometry.dynamic_flow_config.bbox_max=[5,5,5]
+# edit yebinliu
+python launch.py --config configs/dynamic_gaussian_t4d_instruct_web.yaml --train --gpu 1 data.dataroot="dataset/n3d_video_dynamic_pose_editing" name=dynamic_gaussian_instruct tag=n3d_dynamic_pose_editing data.camera_layout="default" system.prompt_processor.prompt="Elon Musk wearing white shirt" system.geometry.geometry_convert_from=/root/autodl-tmp/threestudio/outputs/dynamic_gaussian_gan_reconstruct/n3d_dynamic_pose@20231104-085250/ckpts/last.ckpt data.eval_interpolation=[0,7,50] data.eval_time_interpolation=[0.0,1.0] system.renderer.near=8.0 data.online_load_image=False system.geometry.dynamic_flow_config.bbox_min=[-23,-13,4] system.geometry.dynamic_flow_config.bbox_max=[62,15,42] system.guidance.json_path=/root/autodl-tmp/ComfyUI/config/n3d_ybl_api.json
+python launch.py --config configs/dynamic_gaussian_t4d_instruct_web.yaml --train --gpu 1 data.dataroot="dataset/n3d_video_dynamic_pose_editing" name=dynamic_gaussian_instruct_web tag=n3d_dynamic_pose_editing data.camera_layout="default" system.prompt_processor.prompt="Elon Musk wearing white shirt" resume=/root/autodl-tmp/threestudio/outputs/dynamic_gaussian_instruct/n3d_dynamic_pose_editing@20231104-143031/ckpts/last.ckpt system.edit_path=/root/autodl-tmp/threestudio/outputs/dynamic_gaussian_instruct/n3d_dynamic_pose_editing@20231104-120856/save/edit data.eval_interpolation=[18,6,100] data.eval_time_interpolation=[0.0,1.0] system.renderer.near=8.0 data.online_load_image=False system.geometry.dynamic_flow_config.bbox_min=[-23,-13,4] system.geometry.dynamic_flow_config.bbox_max=[62,15,42] trainer.max_steps=1
+
+# edit du
+python launch.py --config configs/dynamic_gaussian_t4d_du.yaml --train --gpu 0 data.dataroot="dataset/n3d_video_dynamic_pose_editing" name=dynamic_gaussian_du tag=n3d_dynamic_pose_editing data.camera_layout="default" system.prompt_processor.prompt="Elon Musk wearing white shirt" system.geometry.geometry_convert_from=/root/autodl-tmp/threestudio/outputs/dynamic_gaussian_gan_reconstruct/n3d_dynamic_pose@20231104-085250/ckpts/last.ckpt data.eval_interpolation=[0,7,50] data.eval_time_interpolation=[0.0,1.0] system.renderer.near=8.0 data.online_load_image=True system.geometry.dynamic_flow_config.bbox_min=[-23,-13,4] system.geometry.dynamic_flow_config.bbox_max=[62,15,42] system.guidance.json_path=/root/autodl-tmp/ComfyUI/config/n3d_ybl_api.json system.get_patch_size=512 data.train_downsample_resolution=2 data.eval_downsample_resolution=2 system.renderer.need_refine=False
+
+# enerf static recon
+python launch.py --config configs/dynamic_gaussian_t4d_reconstruct.yaml --train --gpu 0 data.dataroot="dataset/enerf_actor1" name=dynamic_gaussian_reconstruct tag=enerf_actor1 data.camera_layout="default" system.prompt_processor.prompt="Elon Musk wearing white shirt" system.geometry.geometry_convert_from=/root/autodl-tmp/threestudio/dataset/enerf/actor1/background.ply data.eval_interpolation=[0,7,50] data.eval_time_interpolation=[0.0,0.0] system.renderer.near=1.0 system.geometry.gaussian.min_opac_prune=0.005 data.online_load_image=False system.geometry.dynamic_flow_config.bbox_min=[-5,-5,-5] system.geometry.dynamic_flow_config.bbox_max=[5,5,5] data.train_downsample_resolution=1 data.eval_downsample_resolution=1
+# enerf dynamic gan recon
+python launch.py --config configs/dynamic_gaussian_t4d_gan_reconstruct.yaml --train --gpu 0 data.dataroot="dataset/enerf_actor1_dynamic" name=dynamic_gaussian_gan_reconstruct tag=enerf_actor1_dynamic data.camera_layout="default" system.prompt_processor.prompt="Elon Musk wearing white shirt" system.geometry.geometry_convert_from=/root/autodl-tmp/threestudio/dataset/enerf/actor1/background.ply data.eval_interpolation=[0,7,50] data.eval_time_interpolation=[0.0,1.0] system.renderer.near=1.0 system.geometry.gaussian.min_opac_prune=0.005 data.online_load_image=True system.geometry.dynamic_flow_config.bbox_min=[-5,-5,-5] system.geometry.dynamic_flow_config.bbox_max=[5,5,5] data.train_downsample_resolution=1 data.eval_downsample_resolution=1
+# enerf dynamic recon
+python launch.py --config configs/dynamic_gaussian_t4d_reconstruct.yaml --train --gpu 1 data.dataroot="dataset/enerf_actor1_dynamic2" name=dynamic_gaussian_reconstruct tag=enerf_actor1_dynamic2 data.camera_layout="default" system.prompt_processor.prompt="Elon Musk wearing white shirt" system.geometry.geometry_convert_from=/root/autodl-tmp/threestudio/outputs/dynamic_gaussian_reconstruct/enerf_actor1_dynamic2@20231110-062013/save/it23000-0.ply data.eval_interpolation=[0,7,50] data.eval_time_interpolation=[0.0,1.0] system.renderer.near=1.0 system.geometry.gaussian.min_opac_prune=0.005 data.online_load_image=True system.geometry.dynamic_flow_config.bbox_min=[-5,-5,-5] system.geometry.dynamic_flow_config.bbox_max=[5,5,5] data.train_downsample_resolution=1 data.eval_downsample_resolution=1 trainer.max_steps=100000 system.geometry.gaussian.densify_until_iter=100000
+python launch.py --config configs/dynamic_gaussian_t4d_reconstruct.yaml --train --gpu 1 data.dataroot="dataset/enerf_actor1_dynamic2" name=dynamic_gaussian_reconstruct tag=enerf_actor1_dynamic2 data.camera_layout="default" system.prompt_processor.prompt="Elon Musk wearing white shirt" resume=/root/autodl-tmp/threestudio/outputs/dynamic_gaussian_reconstruct/enerf_actor1_dynamic2@20231110-060717/ckpts/last.ckpt data.eval_interpolation=[0,7,30] data.eval_time_interpolation=[0.0,1.0] system.renderer.near=1.0 system.geometry.gaussian.min_opac_prune=0.005 data.online_load_image=True system.geometry.dynamic_flow_config.bbox_min=[-5,-5,-5] system.geometry.dynamic_flow_config.bbox_max=[5,5,5] data.train_downsample_resolution=1 data.eval_downsample_resolution=1 trainer.max_steps=100000 system.geometry.gaussian.densify_until_iter=100000
+
+# edit du enerf
+python launch.py --config configs/dynamic_gaussian_t4d_du.yaml --train --gpu 1 data.dataroot="dataset/enerf_actor1_dynamic_editing2" name=dynamic_gaussian_du tag=enerf_dynamic_pose_editing data.camera_layout="default" system.prompt_processor.prompt="Elon Musk wearing white shirt" system.geometry.geometry_convert_from=//root/autodl-tmp/threestudio/outputs/dynamic_gaussian_reconstruct/enerf_actor1_dynamic2@20231110-215249_reg/ckpts/last.ckpt data.eval_interpolation=[0,79,50] data.eval_time_interpolation=[0.0,1.0] system.renderer.near=1.0 data.online_load_image=True system.geometry.dynamic_flow_config.bbox_min=[-5,-5,-5] system.geometry.dynamic_flow_config.bbox_max=[5,5,5] system.get_patch_size=1024 data.train_downsample_resolution=1 data.eval_downsample_resolution=1 system.renderer.need_refine=False
+python launch.py --config configs/dynamic_gaussian_t4d_du.yaml --train --gpu 1 data.dataroot="dataset/enerf_actor1_dynamic_editing2" name=dynamic_gaussian_du tag=enerf_dynamic_pose_editing data.camera_layout="default" system.prompt_processor.prompt="Elon Musk wearing white shirt" resume=/root/autodl-tmp/threestudio/outputs/dynamic_gaussian_du/enerf_dynamic_pose_editing@20231110-224730/ckpts/last.ckpt data.eval_interpolation=[0,79,50] data.eval_time_interpolation=[0.0,1.0] system.renderer.near=1.0 data.online_load_image=True system.geometry.dynamic_flow_config.bbox_min=[-5,-5,-5] system.geometry.dynamic_flow_config.bbox_max=[5,5,5] system.get_patch_size=1024 data.train_downsample_resolution=1 data.eval_downsample_resolution=1 system.renderer.need_refine=False