evaluation code

YuliangXiu · Jun 14, 2022 · 55a74e0 · 55a74e0
1 parent f757b98
commit 55a74e0
Show file tree

Hide file tree

Showing 7 changed files with 136 additions and 47 deletions.
diff --git a/README.md b/README.md
@@ -69,6 +69,9 @@
     <a href="#training">Training</a>
     </li>
     <li>
+    <a href="#evaluation">Evaluation</a>
+    </li>
+    <li>
     <a href="#add-on">Add-on</a>
     </li>
     <li>
@@ -130,11 +133,11 @@
 - [x] dataset processing
   - [x] RGB/Normal Rendering
   - [x] Visibility computing
-- [ ] training codes
+- [ ] training code
   - [x] ICON (w/ & w/o global encoder, w/ SMPL-X)
   - [x] PIFu\* (RGB image + predicted normal map as input)
   - [ ] PaMIR\* (RGB image + predicted normal map as input, w/ voxelized SMPL)
-- [ ] evaluation on CAPE (Hard, Easy)
+- [x] evaluation code
 
 ## Installation
 
@@ -148,6 +151,10 @@ Please follow the [Dataset Instruction](docs/dataset.md) to generate the train/v
 
 Please follow the [Training Instruction](docs/training.md) to train your own model using THuman2.0.
 
+## Evaluation
+
+Please follow the [Evaluation Instruction](docs/evaluation.md) to benchmark models on THuman2.0.
+
 ## Add-on
 1. [Garment Extraction from Fashion Images](docs/garment-extraction.md), supported by ETH Zürich students as 3DV course project.
 

diff --git a/apps/ICON.py b/apps/ICON.py
@@ -60,9 +60,9 @@ def __init__(self, cfg):
         self.evaluator = Evaluator(
             device=torch.device(f"cuda:{self.cfg.gpus[0]}"))
 
-        # # set test engine
-        # if self.cfg.test_mode:
-        #     self.evaluator.init_gl()
+        # set test engine
+        if self.cfg.test_mode:
+            self.evaluator.init_gl()
 
         self.resolutions = (
             np.logspace(
@@ -657,9 +657,7 @@ def test_epoch_end(self, outputs):
             outputs,
             rot_num=3,
             split={
-                "renderpeople": (0, 50),
-                "cape-easy": (50, 100),
-                "cape-hard": (100, 200),
+                "thuman2": (0, 5),
             },
         )
 

diff --git a/apps/train.py b/apps/train.py
@@ -103,11 +103,9 @@ def save_checkpoint(self, filepath, weights_only=False):
             "test_mode",
             True,
             "dataset.types",
-            ["renderpeople", "cape"],
-            "dataset.rp_type",
-            "pifu450",
+            ["thuman2"],
             "dataset.scales",
-            [1.0, 100.0],
+            [100.0],
         ]
         cfg.merge_from_list(cfg_test_mode)
 

diff --git a/configs/train/pamir.yaml b/configs/train/pamir.yaml
@@ -0,0 +1,77 @@
+name: pamir
+ckpt_dir: "./data/ckpt/"
+resume_path: "./data/ckpt/pamir.ckpt"
+normal_path: "./data/ckpt/normal.ckpt"
+results_path: "./results"
+
+dataset:
+  root: "./data/"
+  rotation_num: 36
+  num_sample_geo: 8000
+  num_sample_color: 0
+  num_sample_seg: 0
+  num_sample_knn: 0
+  sigma_geo: 5.0
+  sigma_seg: 0.00
+  sigma_color: 0.00
+  train_bsize: 1.0
+  val_bsize: 1.0
+  test_bsize: 1.0
+  ray_sample_num: 1
+  zray_type: False
+  semantic_p: False
+  remove_outlier: False
+
+  noise_type: ['pose', 'beta']
+  noise_scale: [0.05, 0.5]
+
+  types: ["thuman2"]
+  scales: [100.0]
+
+net:
+  mlp_dim: [256, 512, 256, 128, 1]
+  res_layers: [2,3,4]
+  num_stack: 2
+  prior_type: "pamir"
+  use_filter: True
+  in_geo: (('image',3), ('normal_F',3), ('normal_B',3))
+  in_nml: (('image',3), ('T_normal_F',3), ('T_normal_B',3))
+  gtype: 'HGPIFuNet'
+  ctype: 'resnet34'
+  norm_mlp: 'batch'
+  N_freqs: 10
+  geo_w: 0.1
+  norm_w: 0.001
+  dc_w: 1.0
+  hourglass_dim: 6
+  voxel_dim: 7
+
+lr_G: 1e-4
+weight_decay: 0.0 
+momentum: 0.0
+batch_size: 8
+num_threads: 16
+gpus: [0]
+test_gpus: [0]
+
+sdf: False
+sdf_clip: 15.0
+
+fast_dev: 0
+resume: False
+test_mode: False
+mcube_res: 512
+clean_mesh: True
+num_sanity_val_steps: 1
+
+momentum: 0.0 
+optim: RMSprop
+
+# training (batch=4, set=agora, rot-6)
+overfit: False
+num_epoch: 10
+freq_show_train: 0.1
+freq_show_val: 0.3
+freq_plot: 0.01
+freq_eval: 0.2
+schedule: [3, 8]
diff --git a/docs/evaluation.md b/docs/evaluation.md
@@ -0,0 +1,38 @@
+## Evaluation
+
+## Prerequirement
+
+Make sure you have already generated all the required synthetic data (refer to [Dataset Instruction](dataset.md)) under `./data/thuman2_{num_views}views`, which includes the rendered RGB (`render/`), normal images(`normal_B/`, `normal_F/`, `T_normal_B/`, `T_normal_F/`), corresponding calibration matrix (`calib/`) and pre-computed visibility arrays (`vis/`).
+
+
+:warning: Don't support headless mode currently, `unset PYOPENGL_PLATFORM` before training. (will fix it later...)
+
+## Command
+
+```bash
+conda activate icon
+
+# ICON w/ filter
+CUDA_VISIBLE_DEVICES=0 python -m apps.train -cfg ./configs/train/icon-filter.yaml -test
+
+# ICON w/o filter
+CUDA_VISIBLE_DEVICES=0 python -m apps.train -cfg ./configs/train/icon-nofilter.yaml -test
+
+# PIFu
+CUDA_VISIBLE_DEVICES=0 python -m apps.train -cfg ./configs/train/pifu.yaml -test
+
+# PaMIR
+CUDA_VISIBLE_DEVICES=0 python -m apps.train -cfg ./configs/train/pamir.yaml -test
+```
+
+## Intermediate Results
+
+All the intermediate results are exported at `./results/ckpt_perfect_smpl`
+
+## Benchmark on THuman2.0 (testset: 0500-0504, 3 views)
+
+|Metrics|ICON w/ filter|ICON w/o filter|PIFu|PaMIR|
+|---|---|---|---|---|
+|Chamfer|1.068|1.218|1.711|-|
+|P2S|1.068|1.210|1.159|-|
+|NC|0.061|0.073|0.075|-|
diff --git a/lib/common/train_util.py b/lib/common/train_util.py
@@ -455,32 +455,13 @@ def bar_log_convert(log_dict, name=None, rot=None):
     return new_log_dict
 
 
-# def accumulate(outputs):
-#     acc_log_dict = {}
-#     hparam_log_dict = {}
-#     for id in range(len(outputs)):
-#         output = outputs[id]
-#         for key in output.keys():
-#             if key not in acc_log_dict.keys():
-#                 acc_log_dict[key] = output[key]
-#                 hparam_log_dict["hparam/"+key] = output[key]
-#             else:
-#                 acc_log_dict[key] += output[key]
-#                 hparam_log_dict["hparam/"+key] += output[key]
-
-#     for key in hparam_log_dict.keys():
-#         hparam_log_dict[key] /= len(outputs)
-
-#     return hparam_log_dict
-
-
 def accumulate(outputs, rot_num, split):
 
     hparam_log_dict = {}
 
     metrics = outputs[0].keys()
     datasets = split.keys()
-
+    
     for dataset in datasets:
         for metric in metrics:
             keyword = f"hparam/{dataset}-{metric}"
@@ -492,19 +473,7 @@ def accumulate(outputs, rot_num, split):
             hparam_log_dict[keyword] /= (split[dataset][1] -
                                          split[dataset][0]) * rot_num
 
-    out_lst = []
-    for dataset in ['renderpeople', 'cape-easy', 'cape-hard']:
-        # for dataset in ['cape-easy', 'cape-hard']:
-        # for dataset in ['renderpeople']:
-        for metric in ['chamfer', 'p2s', 'NC']:
-            out_lst.append(hparam_log_dict[f'hparam/{dataset}-{metric}'])
-
-    for i in range(3):
-        out_lst.append((out_lst[i + 3] + out_lst[i + 3 * 2] * 2.0) / 3.0)
-
-    out_str1 = " & ".join([f"{item:.3f}" for item in out_lst])
-
-    print(colored(out_str1, "green"))
+    print(colored(hparam_log_dict, "green"))
 
     return hparam_log_dict
 

diff --git a/lib/dataset/PIFuDataset.py b/lib/dataset/PIFuDataset.py
@@ -205,10 +205,12 @@ def __getitem__(self, index):
             data_dict, is_valid=self.split == "val", is_sdf=self.use_sdf))
         data_dict.update(self.load_smpl(data_dict, self.vis))
 
-        if not self.vis:
-            del data_dict['mesh']
+        if (not self.vis) and (self.split != 'test'):
+
             del data_dict['verts']
             del data_dict['faces']
+
+        del data_dict['mesh']
 
         path_keys = [
             key for key in data_dict.keys() if '_path' in key or '_dir' in key