Add convolutional reconstruction model

.gitignore

@@ -152,4 +152,6 @@ dmypy.json
 # Cython debug symbols
 cython_debug/
 
-out/
+out/
+data/
+exp-*/

README.md

@@ -40,7 +40,12 @@ Install nvdiffrast according to the official [doc](https://nvlabs.github.io/nvdi
 pip install git+https://github.com/NVlabs/nvdiffrast
 ```
 
+### Step 3 - Pytorch3d
+Install nvdiffrast according to the official [doc](https://github.com/facebookresearch/pytorch3d/blob/main/INSTALL.md), e.g.
 
+```bash
+pip install git+https://github.com/facebookresearch/pytorch3d.git
+```
 
 ## Inference
 
@@ -61,7 +66,32 @@ It will output the preprocessed image, generated 6-view images and CCMs and a 3D
 **Tips:** (1) If the result is unsatisfatory, please check whether the input image is correctly pre-processed into a grey background. Otherwise the results will be unpredictable.
 (2) Different from the [Huggingface Demo](https://huggingface.co/spaces/Zhengyi/CRM), this official implementation uses UV texture instead of vertex color. It has better texture than the online demo but longer generating time owing to the UV texturing.
 
-## Train
+
+## Train CRM
+### Step 1. Generate Chunks
+Multiprocess multi-threads chunks generation
+```bash
+python make_chunk.py -c specs_xxxx.json -p 4 # change to proper number of process for your machine
+```
+log info will be recorded in the log.log in the root directory.
+
+### Step 2. Train
+Use the same config json file in **step.1** for training
+#### Single GPU 
+```bash
+python train_crm.py -c specs_xxx.json 
+```
+#### Multi GPU
+```bash
+torchrun --standalone --nproc_per_node=8 train_crm.py -c specs_xxx.json
+
+# Training break to resume training
+torchrun --standalone --nproc_per_node=8 train_crm.py --resume exp-xxx/mm_dd-hh_mm_ss
+
+```
+
+
+## Train Diffusion
 We provide training script for multivew generation and their data requirements.
 To launch a simple one instance overfit training of multivew gen:
 ```shell

configs/specs_objaverse_lvis.json

@@ -0,0 +1,71 @@
+{
+  "Model": "TrDGen",
+
+  "Input": {
+    "mesh_dir": "/home/yudajiang/datasets/Objaverse/objaverse_lvis_mesh",
+    "render_img_dir": "/home/yudajiang/datasets/Objaverse/objaverse_lvis",
+    "blender_transform": "/home/yudajiang/datasets/Objaverse/normalize_anno/meta_data_new.json",
+    "triview_color_dir": "/home/yudajiang/datasets/Objaverse/objaverse_lvis_newdata",
+    "triview_xyz_dir": "/home/yudajiang/datasets/Objaverse/objaverse_lvis_newxyz",
+    "prompt_path": "/home/yudajiang/datasets/Objaverse/Objaverse_anno.csv",
+    "dataset_name": "Objaverse",
+    "img_num": 16,
+    "split_dir": "data_split/objaverse_lvis",
+    "class": "lvis",
+    "camera_angle_num": 8,
+    "tet_grid_size": 90,
+    "validate_num": 16,
+    "dataset_type": "filesystem",
+    "chunk_path": "data/Objaverse",
+    "chunk_size": 256,
+    "scale": 0.95,
+    "radius": 3,
+    "resolution": [256, 256],
+    "resolution_img": [256, 256],
+    "resolution_triview": [256, 256]
+  },
+
+  "Train": {
+    "mode": "rnd",
+    "num_epochs": 500,
+    "warm_up": 0,
+    "decay": 0.000,
+    "learning_rate": {
+      "init": 1e-4
+    },
+    "sample_points_num": 20000,
+    "batch_size": 1,
+    "eva_iter": 100,
+    "save_chunk": 50,
+    "lambda_smooth": 1,
+    "lambda_lp": 0.2,
+    "lambda_color": 2,
+    "radius": 0.5
+  },
+
+  "EncoderSpecs": {
+    "latent_size": 32,
+    "hidden_dim": 64,
+    "color_output": 64,
+    "unet_kwargs": {
+      "depth": 6,
+      "merge_mode": "concat",
+      "start_filts": 64
+    },
+    "plane_resolution": 256
+  },
+
+  "DecoderSpecs": {
+    "c_dim": 32,
+    "unet_kwargs": {
+      "depth": 6,
+      "merge_mode": "concat",
+      "start_filts": 64
+    },
+    "plane_resolution": 256
+  },
+
+  "Output": {
+    "exp_name": "exp-objaverse"
+  }
+}

data_split/objaverse_lvis/exclude.txt

@@ -0,0 +1,3 @@
+1d06ba8ef476420094ec09e818bbebb7
+4d7414c670fc4818868efd717c8556c0
+f4bdcfe475e842d6a95972211d68694c