1 GPU exp

Author: theEricMa

configs/multi-prompt_benchmark/asd_mv_triplane_transformer_10k_1GPU.yaml

@@ -0,0 +1,136 @@
+name: "asd_mv_triplane_100k_1GPU"
+tag: "${rmspace:${system.prompt_processor.prompt_library},_}"
+exp_root_dir: "outputs"
+seed: 0
+
+data_type: "multiprompt-multiview-camera-datamodule"
+data:
+  batch_size: 4
+  n_view: 4
+  # 0-4999: 64x64, >=5000: 512x512
+  # this drastically reduces VRAM usage as empty space is pruned in early training
+  width: 64
+  height: 64
+  camera_distance_range: [0.8, 1.0] # relative
+  fovy_range: [15, 60]
+  elevation_range: [0, 30]
+  camera_perturb: 0.
+  center_perturb: 0.
+  up_perturb: 0.
+  eval_camera_distance: 3.0
+  eval_fovy_deg: 40.
+  n_val_views: 40
+  # generation related
+  prompt_library: ${system.prompt_processor.prompt_library}
+  dim_gaussian: 1 # not used
+
+system_type: "multiprompt-radience-field-generator-system"
+system:
+  stage: "coarse"
+  initialize_shape: false # #true #
+  visualize_samples: false 
+  validation_via_video: true
+
+  geometry_type: "Triplane-transformer-sdf"
+  geometry:
+    radius: 2.0
+    normal_type: "finite_difference"
+    finite_difference_normal_eps: 0.01
+
+    sdf_bias: sphere
+    sdf_bias_params: 0.8 # easier to converge than 0.5
+
+    space_generator_config: # adopt from OpenLRM
+      inner_dim: 768
+      condition_dim: 1024
+      triplane_low_res: 32
+      triplane_high_res: 64
+      triplane_dim: 32
+      num_layers: 12
+      num_heads: 16
+      mlp_ratio: 4
+      local_text: true
+
+  material_type: no-material
+  material:
+    n_output_dims: 3
+    color_activation: sigmoid-mipnerf # follow OpenLRM
+    requires_normal: true
+
+  background_type: "neural-environment-map-background"
+  background:
+    color_activation: sigmoid-mipnerf # follow OpenLRM
+    random_aug: false
+
+  renderer_type: "generative-space-volsdf-volume-renderer"
+  renderer:
+    radius: ${system.geometry.radius}
+    use_volsdf: true
+    trainable_variance: false # important!
+    learned_variance_init: 0.340119 # 0.340119 = log(30) / 10, 30 is the most common variance across the prompts
+
+    randomized: false
+
+    estimator: importance
+    num_samples_per_ray: 64
+    num_samples_per_ray_importance: 128
+    near_plane: 0.1
+    far_plane: 4.0
+    train_chunk_size: 0 #100000
+
+  prompt_processor_type: "stable-diffusion-multi-prompt-processor"
+  prompt_processor:
+    pretrained_model_name_or_path: "pretrained/stable-diffusion-2-1-base"
+    use_local_text_embeddings: ${system.geometry.space_generator_config.local_text}
+    prompt_library: ???
+    negative_prompt: "ugly, bad anatomy, blurry, pixelated obscure, unnatural colors, poor lighting, dull, and unclear, cropped, lowres, low quality, artifacts, duplicate, morbid, mutilated, poorly drawn face, deformed, dehydrated, bad proportions"
+
+  guidance_type: "mvdream-asynchronous-score-distillation-guidance"
+  guidance:
+    model_name: "sd-v2.1-base-4view"
+    ckpt_path: "pretrained/sd-v2.1-base-4view.pt" # path to a pre-downloaded checkpoint file (null for loading from URL)
+    guidance_scale: 7.5
+    plus_ratio: 0.1
+    plus_random: true
+    min_step_percent: [0, 0.5, 0.02, 50000] # follow MVDream
+    max_step_percent: [0, 0.98, 0.5, 50000] # same as vsd
+  loggers:
+    wandb:
+      enable: false
+      project: "threestudio"
+      name: None
+
+  loss:
+    lambda_asd: 1.
+    lambda_orient: 0.
+    lambda_sparsity: 20
+    lambda_opaque: [40000, 0, 1., 50000] # final 1/5 iterations
+    lambda_z_variance: 0.
+    lambda_eikonal: 0.01 #[1, 10., 1., 10000] # fisrt 1/10 iterations
+
+  optimizer:
+    name: Adan # for fast convergence
+    args:
+      betas: [0.98, 0.92, 0.99]
+      eps: 1.e-15
+    params:
+      geometry:
+        lr: 0.0002
+      background:
+        lr: 0.0002
+
+
+trainer:
+  max_steps: 50000
+  log_every_n_steps: 1
+  num_sanity_val_steps: 0
+  val_check_interval: 10000
+  enable_progress_bar: true
+  precision: 32
+  strategy: "ddp" # "deepspeed_stage_2"
+  accumulate_grad_batches: 8
+
+checkpoint:
+  save_last: true
+  save_top_k: -1
+  every_n_train_steps: ${trainer.val_check_interval}