From 55a74e001e8fb4bca1492eb54ac621cf5be9f218 Mon Sep 17 00:00:00 2001 From: Yuliang Xiu Date: Tue, 14 Jun 2022 15:35:09 +0200 Subject: [PATCH] evaluation code --- README.md | 11 +++++- apps/ICON.py | 10 ++--- apps/train.py | 6 +-- configs/train/pamir.yaml | 77 ++++++++++++++++++++++++++++++++++++++ docs/evaluation.md | 38 +++++++++++++++++++ lib/common/train_util.py | 35 +---------------- lib/dataset/PIFuDataset.py | 6 ++- 7 files changed, 136 insertions(+), 47 deletions(-) create mode 100644 configs/train/pamir.yaml create mode 100644 docs/evaluation.md diff --git a/README.md b/README.md index 8acdc38..b2de1e3 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,9 @@ Training
  • + Evaluation +
    • +
  • Add-on
  • @@ -130,11 +133,11 @@ - [x] dataset processing - [x] RGB/Normal Rendering - [x] Visibility computing -- [ ] training codes +- [ ] training code - [x] ICON (w/ & w/o global encoder, w/ SMPL-X) - [x] PIFu\* (RGB image + predicted normal map as input) - [ ] PaMIR\* (RGB image + predicted normal map as input, w/ voxelized SMPL) -- [ ] evaluation on CAPE (Hard, Easy) +- [x] evaluation code ## Installation @@ -148,6 +151,10 @@ Please follow the [Dataset Instruction](docs/dataset.md) to generate the train/v Please follow the [Training Instruction](docs/training.md) to train your own model using THuman2.0. +## Evaluation + +Please follow the [Evaluation Instruction](docs/evaluation.md) to benchmark models on THuman2.0. + ## Add-on 1. [Garment Extraction from Fashion Images](docs/garment-extraction.md), supported by ETH Zürich students as 3DV course project. diff --git a/apps/ICON.py b/apps/ICON.py index b9a4cc6..e79b90c 100644 --- a/apps/ICON.py +++ b/apps/ICON.py @@ -60,9 +60,9 @@ def __init__(self, cfg): self.evaluator = Evaluator( device=torch.device(f"cuda:{self.cfg.gpus[0]}")) - # # set test engine - # if self.cfg.test_mode: - # self.evaluator.init_gl() + # set test engine + if self.cfg.test_mode: + self.evaluator.init_gl() self.resolutions = ( np.logspace( @@ -657,9 +657,7 @@ def test_epoch_end(self, outputs): outputs, rot_num=3, split={ - "renderpeople": (0, 50), - "cape-easy": (50, 100), - "cape-hard": (100, 200), + "thuman2": (0, 5), }, ) diff --git a/apps/train.py b/apps/train.py index 10dce66..25d5b93 100644 --- a/apps/train.py +++ b/apps/train.py @@ -103,11 +103,9 @@ def save_checkpoint(self, filepath, weights_only=False): "test_mode", True, "dataset.types", - ["renderpeople", "cape"], - "dataset.rp_type", - "pifu450", + ["thuman2"], "dataset.scales", - [1.0, 100.0], + [100.0], ] cfg.merge_from_list(cfg_test_mode) diff --git a/configs/train/pamir.yaml b/configs/train/pamir.yaml new file mode 100644 index 0000000..990ecf1 --- /dev/null +++ b/configs/train/pamir.yaml @@ -0,0 +1,77 @@ +name: pamir +ckpt_dir: "./data/ckpt/" +resume_path: "./data/ckpt/pamir.ckpt" +normal_path: "./data/ckpt/normal.ckpt" +results_path: "./results" + +dataset: + root: "./data/" + rotation_num: 36 + num_sample_geo: 8000 + num_sample_color: 0 + num_sample_seg: 0 + num_sample_knn: 0 + sigma_geo: 5.0 + sigma_seg: 0.00 + sigma_color: 0.00 + train_bsize: 1.0 + val_bsize: 1.0 + test_bsize: 1.0 + ray_sample_num: 1 + zray_type: False + semantic_p: False + remove_outlier: False + + noise_type: ['pose', 'beta'] + noise_scale: [0.05, 0.5] + + types: ["thuman2"] + scales: [100.0] + +net: + mlp_dim: [256, 512, 256, 128, 1] + res_layers: [2,3,4] + num_stack: 2 + prior_type: "pamir" + use_filter: True + in_geo: (('image',3), ('normal_F',3), ('normal_B',3)) + in_nml: (('image',3), ('T_normal_F',3), ('T_normal_B',3)) + gtype: 'HGPIFuNet' + ctype: 'resnet34' + norm_mlp: 'batch' + N_freqs: 10 + geo_w: 0.1 + norm_w: 0.001 + dc_w: 1.0 + hourglass_dim: 6 + voxel_dim: 7 + +lr_G: 1e-4 +weight_decay: 0.0 +momentum: 0.0 +batch_size: 8 +num_threads: 16 +gpus: [0] +test_gpus: [0] + +sdf: False +sdf_clip: 15.0 + +fast_dev: 0 +resume: False +test_mode: False +mcube_res: 512 +clean_mesh: True +num_sanity_val_steps: 1 + +momentum: 0.0 +optim: RMSprop + +# training (batch=4, set=agora, rot-6) +overfit: False +num_epoch: 10 +freq_show_train: 0.1 +freq_show_val: 0.3 +freq_plot: 0.01 +freq_eval: 0.2 +schedule: [3, 8] diff --git a/docs/evaluation.md b/docs/evaluation.md new file mode 100644 index 0000000..7dc5da8 --- /dev/null +++ b/docs/evaluation.md @@ -0,0 +1,38 @@ +## Evaluation + +## Prerequirement + +Make sure you have already generated all the required synthetic data (refer to [Dataset Instruction](dataset.md)) under `./data/thuman2_{num_views}views`, which includes the rendered RGB (`render/`), normal images(`normal_B/`, `normal_F/`, `T_normal_B/`, `T_normal_F/`), corresponding calibration matrix (`calib/`) and pre-computed visibility arrays (`vis/`). + + +:warning: Don't support headless mode currently, `unset PYOPENGL_PLATFORM` before training. (will fix it later...) + +## Command + +```bash +conda activate icon + +# ICON w/ filter +CUDA_VISIBLE_DEVICES=0 python -m apps.train -cfg ./configs/train/icon-filter.yaml -test + +# ICON w/o filter +CUDA_VISIBLE_DEVICES=0 python -m apps.train -cfg ./configs/train/icon-nofilter.yaml -test + +# PIFu +CUDA_VISIBLE_DEVICES=0 python -m apps.train -cfg ./configs/train/pifu.yaml -test + +# PaMIR +CUDA_VISIBLE_DEVICES=0 python -m apps.train -cfg ./configs/train/pamir.yaml -test +``` + +## Intermediate Results + +All the intermediate results are exported at `./results/ckpt_perfect_smpl` + +## Benchmark on THuman2.0 (testset: 0500-0504, 3 views) + +|Metrics|ICON w/ filter|ICON w/o filter|PIFu|PaMIR| +|---|---|---|---|---| +|Chamfer|1.068|1.218|1.711|-| +|P2S|1.068|1.210|1.159|-| +|NC|0.061|0.073|0.075|-| \ No newline at end of file diff --git a/lib/common/train_util.py b/lib/common/train_util.py index 739aed0..7775c63 100644 --- a/lib/common/train_util.py +++ b/lib/common/train_util.py @@ -455,32 +455,13 @@ def bar_log_convert(log_dict, name=None, rot=None): return new_log_dict -# def accumulate(outputs): -# acc_log_dict = {} -# hparam_log_dict = {} -# for id in range(len(outputs)): -# output = outputs[id] -# for key in output.keys(): -# if key not in acc_log_dict.keys(): -# acc_log_dict[key] = output[key] -# hparam_log_dict["hparam/"+key] = output[key] -# else: -# acc_log_dict[key] += output[key] -# hparam_log_dict["hparam/"+key] += output[key] - -# for key in hparam_log_dict.keys(): -# hparam_log_dict[key] /= len(outputs) - -# return hparam_log_dict - - def accumulate(outputs, rot_num, split): hparam_log_dict = {} metrics = outputs[0].keys() datasets = split.keys() - + for dataset in datasets: for metric in metrics: keyword = f"hparam/{dataset}-{metric}" @@ -492,19 +473,7 @@ def accumulate(outputs, rot_num, split): hparam_log_dict[keyword] /= (split[dataset][1] - split[dataset][0]) * rot_num - out_lst = [] - for dataset in ['renderpeople', 'cape-easy', 'cape-hard']: - # for dataset in ['cape-easy', 'cape-hard']: - # for dataset in ['renderpeople']: - for metric in ['chamfer', 'p2s', 'NC']: - out_lst.append(hparam_log_dict[f'hparam/{dataset}-{metric}']) - - for i in range(3): - out_lst.append((out_lst[i + 3] + out_lst[i + 3 * 2] * 2.0) / 3.0) - - out_str1 = " & ".join([f"{item:.3f}" for item in out_lst]) - - print(colored(out_str1, "green")) + print(colored(hparam_log_dict, "green")) return hparam_log_dict diff --git a/lib/dataset/PIFuDataset.py b/lib/dataset/PIFuDataset.py index 613f33d..d54cc2f 100644 --- a/lib/dataset/PIFuDataset.py +++ b/lib/dataset/PIFuDataset.py @@ -205,10 +205,12 @@ def __getitem__(self, index): data_dict, is_valid=self.split == "val", is_sdf=self.use_sdf)) data_dict.update(self.load_smpl(data_dict, self.vis)) - if not self.vis: - del data_dict['mesh'] + if (not self.vis) and (self.split != 'test'): + del data_dict['verts'] del data_dict['faces'] + + del data_dict['mesh'] path_keys = [ key for key in data_dict.keys() if '_path' in key or '_dir' in key