From 677977e44c17f5126ebacc5651731a0d5a448c86 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Tue, 2 Feb 2021 22:22:51 +0000 Subject: [PATCH 01/24] added path to learning statistics in demo README. --- demo/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/demo/README.md b/demo/README.md index 7c9db57..104e420 100644 --- a/demo/README.md +++ b/demo/README.md @@ -7,7 +7,7 @@ You can run the DGP pipeline on the example dataset provided in this repo by run python demo/run_dgp_demo.py --dlcpath data/Reaching-Mackenzie-2018-08-30 ``` -The output of the pipeline, including the labeled videos and the h5/csv files with predicted trajectories will be stored in `{DGP_DIR}/data/Reaching-Mackenzie-2018-08-30/videos_pred`. +The output of the pipeline, including the labeled videos and the h5/csv files with predicted trajectories will be stored in `{DGP_DIR}/data/Reaching-Mackenzie-2018-08-30/videos_pred`. You can see information about training statistics in the file `{DGP_DIR}/data/dlc-models/iteration-0/ReachingAug30-trainset95shuffle1/train/learning_stats.csv`. You can run the DGP pipeline on your own dataset as long as it exists in a DLC file directory structure, for example @@ -28,4 +28,4 @@ In particular, you can use the DLC GUI to create a DLC project, label videos, an `python {DGP_DIR}/demo/run_dgp_demo.py --dlcpath {PROJ_DIR}/task-scorer-date/ --shuffle 'the shuffle to run' --dlcsnapshot 'specify the DLC snapshot if you've already run DLC with location refinement'` -If you have not yet run DLC you can simply remove the `--dlcsnapshot` argument and DLC will automatically be fit as part of the pipeline. \ No newline at end of file +If you have not yet run DLC you can simply remove the `--dlcsnapshot` argument and DLC will automatically be fit as part of the pipeline. From 92a755f498b93d7ff50f3ed26ddcc2c216a96372 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Tue, 9 Feb 2021 14:13:14 +0000 Subject: [PATCH 02/24] added catch for if config file has already been updated. --- demo/run_dgp_demo.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/demo/run_dgp_demo.py b/demo/run_dgp_demo.py index 0090ac0..f8c3d91 100644 --- a/demo/run_dgp_demo.py +++ b/demo/run_dgp_demo.py @@ -36,7 +36,11 @@ def update_config_files(dlcpath): yaml_cfg = yaml.load(f, Loader=yaml.SafeLoader) yaml_cfg['project_path'] = os.path.join(base_path, dlcpath) video_loc = os.path.join(base_path, dlcpath, 'videos', 'reachingvideo1.avi') - yaml_cfg['video_sets'][video_loc] = yaml_cfg['video_sets'].pop('videos/reachingvideo1.avi') + try: + yaml_cfg['video_sets'][video_loc] = yaml_cfg['video_sets'].pop('videos/reachingvideo1.avi') + except KeyError: + ## check if update has already been done. + assert yaml_cfg["video_sets"].get(video_loc,False), "Can't find original or updated video path in config file." with open(proj_cfg_path, 'w') as f: yaml.dump(yaml_cfg, f) From 0a052a0087b679338bce31400fe0acfcf9403f50 Mon Sep 17 00:00:00 2001 From: cellistigs Date: Sat, 20 Feb 2021 14:51:27 -0500 Subject: [PATCH 03/24] added option to create_labels_md to set seed. --- src/deepgraphpose/preprocess/get_morig_labeled_data.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/deepgraphpose/preprocess/get_morig_labeled_data.py b/src/deepgraphpose/preprocess/get_morig_labeled_data.py index 50aa7d3..c3cd07c 100644 --- a/src/deepgraphpose/preprocess/get_morig_labeled_data.py +++ b/src/deepgraphpose/preprocess/get_morig_labeled_data.py @@ -178,7 +178,7 @@ def create_labels(task, date, overwrite_flag=False, check_labels=False, verbose= #%% -def create_labels_md(config_path, video_path, scorer, overwrite_flag=False, check_labels=False, verbose=False): +def create_labels_md(config_path, video_path, scorer, overwrite_flag=False, check_labels=False, verbose=False,seed= None): from deepgraphpose.utils_data import local_extract_frames_md from deeplabcut.utils import auxiliaryfunctions @@ -202,10 +202,12 @@ def create_labels_md(config_path, video_path, scorer, overwrite_flag=False, chec #% if cfg["numframes2pick"] is not None: + np.random.seed(seed) ## option to set seed. assert len(np.unique(frames_index_keep)) >= cfg["numframes2pick"] frames2pick = np.sort( np.random.choice(frames_index_keep, cfg["numframes2pick"], replace=False) ) + np.random.seed() ## make sure we return to randomness else: frames2pick = frames_index_keep numframes2pick += [len(frames2pick)] @@ -304,4 +306,4 @@ def create_labels_md(config_path, video_path, scorer, overwrite_flag=False, chec # tf.app.run(main=create_labels, argv=[sys.argv[0]] + unparsed) input_params = parser.parse_args() create_labels(input_params.task, input_params.date, input_params.overwrite_flag) - #%% \ No newline at end of file + #%% From beacdb0788740018eca1c3fd444bc7f79a70d11e Mon Sep 17 00:00:00 2001 From: cellistigs Date: Wed, 12 May 2021 12:20:08 -0400 Subject: [PATCH 04/24] added prediction only script. Assumes that the snapshot is indexed as 'step2' --- predict_dgp_demo.py | 225 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 225 insertions(+) create mode 100644 predict_dgp_demo.py diff --git a/predict_dgp_demo.py b/predict_dgp_demo.py new file mode 100644 index 0000000..7f02968 --- /dev/null +++ b/predict_dgp_demo.py @@ -0,0 +1,225 @@ +# If you have collected labels using DLC's GUI you can run DGP with the following +"""Main fitting function for DGP. + step 0: run DLC + step 1: run DGP with labeled frames only + step 2: run DGP with spatial clique + step 3: do prediction on all videos +""" +import argparse +import os +from os import listdir +from os.path import isfile, join +from pathlib import Path +import sys +import yaml + +if sys.platform == 'darwin': + import wx + if int(wx.__version__[0]) > 3: + wx.Thread_IsMain = wx.IsMainThread + +os.environ["DLClight"] = "True" +os.environ["Colab"] = "True" +from deeplabcut.utils import auxiliaryfunctions + +from deepgraphpose.models.fitdgp import fit_dlc, fit_dgp, fit_dgp_labeledonly +from deepgraphpose.models.fitdgp_util import get_snapshot_path +from deepgraphpose.models.eval import plot_dgp + + +def update_config_files(dlcpath): + base_path = os.getcwd() + + # project config + proj_cfg_path = os.path.join(base_path, dlcpath, 'config.yaml') + with open(proj_cfg_path, 'r') as f: + yaml_cfg = yaml.load(f, Loader=yaml.SafeLoader) + yaml_cfg['project_path'] = os.path.join(base_path, dlcpath) + video_loc = os.path.join(base_path, dlcpath, 'videos', 'reachingvideo1.avi') + try: + yaml_cfg['video_sets'][video_loc] = yaml_cfg['video_sets'].pop('videos/reachingvideo1.avi') + except KeyError: + ## check if update has already been done. + assert yaml_cfg["video_sets"].get(video_loc,False), "Can't find original or updated video path in config file." + with open(proj_cfg_path, 'w') as f: + yaml.dump(yaml_cfg, f) + + # train model config + model_cfg_path = get_model_cfg_path(base_path, 'train') + with open(model_cfg_path, 'r') as f: + yaml_cfg = yaml.load(f, Loader=yaml.SafeLoader) + yaml_cfg['init_weights'] = get_init_weights_path(base_path) + yaml_cfg['project_path'] = os.path.join(base_path, dlcpath) + with open(model_cfg_path, 'w') as f: + yaml.dump(yaml_cfg, f) + + # download resnet weights if necessary + if not os.path.exists(yaml_cfg['init_weights']): + raise FileNotFoundError('Must download resnet-50 weights; see README for instructions') + + # test model config + model_cfg_path = get_model_cfg_path(base_path, 'test') + with open(model_cfg_path, 'r') as f: + yaml_cfg = yaml.load(f, Loader=yaml.SafeLoader) + yaml_cfg['init_weights'] = get_init_weights_path(base_path) + with open(model_cfg_path, 'w') as f: + yaml.dump(yaml_cfg, f) + + return os.path.join(base_path, dlcpath) + + +def return_configs(): + base_path = os.getcwd() + dlcpath = 'data/Reaching-Mackenzie-2018-08-30' + + # project config + proj_cfg_path = os.path.join(base_path, dlcpath, 'config.yaml') + with open(proj_cfg_path, 'r') as f: + yaml_cfg = yaml.load(f, Loader=yaml.SafeLoader) + yaml_cfg['project_path'] = dlcpath + video_loc = os.path.join(base_path, dlcpath, 'videos', 'reachingvideo1.avi') + yaml_cfg['video_sets']['videos/reachingvideo1.avi'] = yaml_cfg['video_sets'].pop(video_loc) + with open(proj_cfg_path, 'w') as f: + yaml.dump(yaml_cfg, f) + + # train model config + model_cfg_path = get_model_cfg_path(base_path, 'train') + with open(model_cfg_path, 'r') as f: + yaml_cfg = yaml.load(f, Loader=yaml.SafeLoader) + yaml_cfg['init_weights'] = 'resnet_v1_50.ckpt' + yaml_cfg['project_path'] = dlcpath + with open(model_cfg_path, 'w') as f: + yaml.dump(yaml_cfg, f) + + # test model config + model_cfg_path = get_model_cfg_path(base_path, 'test') + with open(model_cfg_path, 'r') as f: + yaml_cfg = yaml.load(f, Loader=yaml.SafeLoader) + yaml_cfg['init_weights'] = 'resnet_v1_50.ckpt' + with open(model_cfg_path, 'w') as f: + yaml.dump(yaml_cfg, f) + + +def get_model_cfg_path(base_path, dtype): + return os.path.join( + base_path, dlcpath, 'dlc-models', 'iteration-0', 'ReachingAug30-trainset95shuffle1', + dtype, 'pose_cfg.yaml') + + +def get_init_weights_path(base_path): + return os.path.join( + base_path, 'src', 'DeepLabCut', 'deeplabcut', 'pose_estimation_tensorflow', + 'models', 'pretrained', 'resnet_v1_50.ckpt') + + +if __name__ == '__main__': + + # %% set up dlcpath for DLC project and hyperparameters + parser = argparse.ArgumentParser() + parser.add_argument( + "--dlcpath", + type=str, + default=None, + help="the absolute path of the DLC project", + ) + + parser.add_argument( + "--dlcsnapshot", + type=str, + default=None, + help="use the DLC snapshot to initialize DGP", + ) + + parser.add_argument("--shuffle", type=int, default=1, help="Project shuffle") + parser.add_argument( + "--batch_size", + type=int, + default=10, + help="size of the batch, if there are memory issues, decrease it value") + parser.add_argument("--test", action='store_true', default=False) + + input_params = parser.parse_known_args()[0] + print(input_params) + + dlcpath = input_params.dlcpath + shuffle = input_params.shuffle + dlcsnapshot = input_params.dlcsnapshot + batch_size = input_params.batch_size + test = input_params.test + + update_configs = False + if dlcpath == 'data/Reaching-Mackenzie-2018-08-30': + # update config files + dlcpath = update_config_files(dlcpath) + update_configs = True + + ## Specifying snapshot manually at the moment assuming training. + step = 2 + snapshot = 'snapshot-step{}-final--0'.format(step) + + try: + + # -------------------------------------------------------------------------------- + # Test DGP model + # -------------------------------------------------------------------------------- + + # %% step 3 predict on all videos in videos_dgp folder + print( + ''' + ========================== + | | + | | + | Predict with DGP | + | | + | | + ========================== + ''' + , flush=True) + + snapshot_path, cfg_yaml = get_snapshot_path(snapshot, dlcpath, shuffle=shuffle) + cfg = auxiliaryfunctions.read_config(cfg_yaml) + + video_path = str(Path(dlcpath) / 'videos_dgp') + if not (os.path.exists(video_path)): + print(video_path + " does not exist!") + video_sets = list(cfg['video_sets']) + else: + video_sets = [ + video_path + '/' + f for f in listdir(video_path) + if isfile(join(video_path, f)) and ( + f.find('avi') > 0 or f.find('mp4') > 0 or f.find('mov') > 0 or f.find( + 'mkv') > 0) + ] + + video_pred_path = str(Path(dlcpath) / 'videos_pred') + if not os.path.exists(video_pred_path): + os.makedirs(video_pred_path) + + print('video_sets', video_sets, flush=True) + + if test: + for video_file in [video_sets[0]]: + from moviepy.editor import VideoFileClip + clip =VideoFileClip(str(video_file)) + if clip.duration > 10: + clip = clip.subclip(10) + video_file_name = video_file.rsplit('/', 1)[-1].rsplit('.',1)[0] + '.mp4' + print('\nwriting {}'.format(video_file_name)) + clip.write_videofile(video_file_name) + output_dir = os.getcwd() + '/' + plot_dgp(video_file=str(video_file_name), + output_dir=output_dir, + proj_cfg_file=str(cfg_yaml), + dgp_model_file=str(snapshot_path), + shuffle=shuffle) + else: + for video_file in video_sets: + plot_dgp(str(video_file), + str(video_pred_path), + proj_cfg_file=str(cfg_yaml), + dgp_model_file=str(snapshot_path), + shuffle=shuffle) + finally: + + if update_configs: + return_configs() From e931bc8ac750368cf1c38883310fa437762545dc Mon Sep 17 00:00:00 2001 From: cellistigs Date: Wed, 12 May 2021 14:39:59 -0400 Subject: [PATCH 05/24] added predict file. --- predict_dgp_demo.py => demo/predict_dgp_demo.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename predict_dgp_demo.py => demo/predict_dgp_demo.py (100%) diff --git a/predict_dgp_demo.py b/demo/predict_dgp_demo.py similarity index 100% rename from predict_dgp_demo.py rename to demo/predict_dgp_demo.py From 06c2f96d67450e0fade7d3a019fdcb93c7f5be7b Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Wed, 12 May 2021 19:45:23 +0000 Subject: [PATCH 06/24] don't restore configs. --- demo/predict_dgp_demo.py | 12 +++++------- demo/run_dgp_demo.py | 13 ++++++------- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/demo/predict_dgp_demo.py b/demo/predict_dgp_demo.py index 7f02968..c01d7fc 100644 --- a/demo/predict_dgp_demo.py +++ b/demo/predict_dgp_demo.py @@ -147,11 +147,9 @@ def get_init_weights_path(base_path): batch_size = input_params.batch_size test = input_params.test - update_configs = False - if dlcpath == 'data/Reaching-Mackenzie-2018-08-30': - # update config files - dlcpath = update_config_files(dlcpath) - update_configs = True + # update config files + dlcpath = update_config_files(dlcpath) + update_configs = True ## Specifying snapshot manually at the moment assuming training. step = 2 @@ -221,5 +219,5 @@ def get_init_weights_path(base_path): shuffle=shuffle) finally: - if update_configs: - return_configs() + #if update_configs: + # return_configs() diff --git a/demo/run_dgp_demo.py b/demo/run_dgp_demo.py index f8c3d91..fa85de9 100644 --- a/demo/run_dgp_demo.py +++ b/demo/run_dgp_demo.py @@ -147,11 +147,9 @@ def get_init_weights_path(base_path): batch_size = input_params.batch_size test = input_params.test - update_configs = False - if dlcpath == 'data/Reaching-Mackenzie-2018-08-30': - # update config files - dlcpath = update_config_files(dlcpath) - update_configs = True + # update config files + dlcpath = update_config_files(dlcpath) + update_configs = True # ------------------------------------------------------------------------------------ # Train models @@ -308,6 +306,7 @@ def get_init_weights_path(base_path): dgp_model_file=str(snapshot_path), shuffle=shuffle) finally: + pass - if update_configs: - return_configs() + #if update_configs: + # return_configs() From f1845ad462d96da6e456ad9cb05002b58282e4b4 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Wed, 12 May 2021 19:58:10 +0000 Subject: [PATCH 07/24] changed test to write into model folder properly, but postpend test. --- demo/run_dgp_demo.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/demo/run_dgp_demo.py b/demo/run_dgp_demo.py index fa85de9..3a06be1 100644 --- a/demo/run_dgp_demo.py +++ b/demo/run_dgp_demo.py @@ -289,12 +289,12 @@ def get_init_weights_path(base_path): clip =VideoFileClip(str(video_file)) if clip.duration > 10: clip = clip.subclip(10) - video_file_name = video_file.rsplit('/', 1)[-1].rsplit('.',1)[0] + '.mp4' + #video_file_name = video_file.rsplit('/', 1)[-1].rsplit('.',1)[0] + '.mp4' + video_file_name = video_file.splitext("avi")[0] +"test"+ ".mp4" print('\nwriting {}'.format(video_file_name)) clip.write_videofile(video_file_name) - output_dir = os.getcwd() + '/' plot_dgp(video_file=str(video_file_name), - output_dir=output_dir, + str(video_pred_path), proj_cfg_file=str(cfg_yaml), dgp_model_file=str(snapshot_path), shuffle=shuffle) From b9faa3bac00c06dbfc2cbaf9444026e21938c344 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Wed, 12 May 2021 20:30:44 +0000 Subject: [PATCH 08/24] adding updated config handling. --- demo/predict_dgp_demo.py | 3 ++- demo/run_dgp_demo.py | 48 +++++++++++++++++++++++++++++++++++++--- 2 files changed, 47 insertions(+), 4 deletions(-) diff --git a/demo/predict_dgp_demo.py b/demo/predict_dgp_demo.py index c01d7fc..2a406f7 100644 --- a/demo/predict_dgp_demo.py +++ b/demo/predict_dgp_demo.py @@ -25,6 +25,7 @@ from deepgraphpose.models.fitdgp import fit_dlc, fit_dgp, fit_dgp_labeledonly from deepgraphpose.models.fitdgp_util import get_snapshot_path from deepgraphpose.models.eval import plot_dgp +from run_dgp_demo import update_config_files_general def update_config_files(dlcpath): @@ -148,7 +149,7 @@ def get_init_weights_path(base_path): test = input_params.test # update config files - dlcpath = update_config_files(dlcpath) + dlcpath = update_config_files_general(dlcpath) update_configs = True ## Specifying snapshot manually at the moment assuming training. diff --git a/demo/run_dgp_demo.py b/demo/run_dgp_demo.py index 3a06be1..c455d69 100644 --- a/demo/run_dgp_demo.py +++ b/demo/run_dgp_demo.py @@ -27,6 +27,48 @@ from deepgraphpose.models.eval import plot_dgp +def update_config_files_general(dlcpath): + """General purpose version of the function below that applies to all models, not just the default reachingvideo. + + """ + base_path = os.getcwd() + + # project config + proj_cfg_path = os.path.join(base_path, dlcpath, 'config.yaml') + with open(proj_cfg_path, 'r') as f: + yaml_cfg = yaml.load(f, Loader=yaml.SafeLoader) + yaml_cfg['project_path'] = os.path.join(base_path, dlcpath) + # video_loc = os.path.join(base_path, dlcpath, 'videos', 'reachingvideo1.avi') + # try: + # yaml_cfg['video_sets'][video_loc] = yaml_cfg['video_sets'].pop('videos/reachingvideo1.avi') + # except KeyError: + # ## check if update has already been done. + # assert yaml_cfg["video_sets"].get(video_loc,False), "Can't find original or updated video path in config file." + with open(proj_cfg_path, 'w') as f: + yaml.dump(yaml_cfg, f) + + # train model config + model_cfg_path = get_model_cfg_path(base_path, 'train') + with open(model_cfg_path, 'r') as f: + yaml_cfg = yaml.load(f, Loader=yaml.SafeLoader) + yaml_cfg['init_weights'] = get_init_weights_path(base_path) + yaml_cfg['project_path'] = os.path.join(base_path, dlcpath) + with open(model_cfg_path, 'w') as f: + yaml.dump(yaml_cfg, f) + + # download resnet weights if necessary + if not os.path.exists(yaml_cfg['init_weights']): + raise FileNotFoundError('Must download resnet-50 weights; see README for instructions') + + # test model config + model_cfg_path = get_model_cfg_path(base_path, 'test') + with open(model_cfg_path, 'r') as f: + yaml_cfg = yaml.load(f, Loader=yaml.SafeLoader) + yaml_cfg['init_weights'] = get_init_weights_path(base_path) + with open(model_cfg_path, 'w') as f: + yaml.dump(yaml_cfg, f) + + return os.path.join(base_path, dlcpath) def update_config_files(dlcpath): base_path = os.getcwd() @@ -148,7 +190,7 @@ def get_init_weights_path(base_path): test = input_params.test # update config files - dlcpath = update_config_files(dlcpath) + dlcpath = update_config_files_general(dlcpath) update_configs = True # ------------------------------------------------------------------------------------ @@ -290,10 +332,10 @@ def get_init_weights_path(base_path): if clip.duration > 10: clip = clip.subclip(10) #video_file_name = video_file.rsplit('/', 1)[-1].rsplit('.',1)[0] + '.mp4' - video_file_name = video_file.splitext("avi")[0] +"test"+ ".mp4" + video_file_name = os.path.splitext(video_file)[0] +"test"+ ".mp4" print('\nwriting {}'.format(video_file_name)) clip.write_videofile(video_file_name) - plot_dgp(video_file=str(video_file_name), + plot_dgp(str(video_file_name), str(video_pred_path), proj_cfg_file=str(cfg_yaml), dgp_model_file=str(snapshot_path), From 062d2d1443a3abe1a00f7c36f15a57186b747b96 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Wed, 12 May 2021 20:35:30 +0000 Subject: [PATCH 09/24] committing current state of deepgraphpose fork. needs work on the config file handling. --- demo/run_dgp_demo.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/demo/run_dgp_demo.py b/demo/run_dgp_demo.py index c455d69..5a58428 100644 --- a/demo/run_dgp_demo.py +++ b/demo/run_dgp_demo.py @@ -69,6 +69,7 @@ def update_config_files_general(dlcpath): yaml.dump(yaml_cfg, f) return os.path.join(base_path, dlcpath) + def update_config_files(dlcpath): base_path = os.getcwd() @@ -147,6 +148,11 @@ def get_model_cfg_path(base_path, dtype): base_path, dlcpath, 'dlc-models', 'iteration-0', 'ReachingAug30-trainset95shuffle1', dtype, 'pose_cfg.yaml') +def get_model_cfg_path_general(base_path, dtype): + return os.path.join( + base_path, dlcpath, 'dlc-models', 'iteration-0', 'ReachingAug30-trainset95shuffle1', + dtype, 'pose_cfg.yaml') + def get_init_weights_path(base_path): return os.path.join( From f2ab7c6651766a6c3407c10addc3f33fdffba55a Mon Sep 17 00:00:00 2001 From: cellistigs Date: Wed, 12 May 2021 16:48:23 -0400 Subject: [PATCH 10/24] general purpose handling of taskname added. --- demo/predict_dgp_demo.py | 2 +- demo/run_dgp_demo.py | 20 ++++++++++++++------ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/demo/predict_dgp_demo.py b/demo/predict_dgp_demo.py index 2a406f7..3ebf60b 100644 --- a/demo/predict_dgp_demo.py +++ b/demo/predict_dgp_demo.py @@ -149,7 +149,7 @@ def get_init_weights_path(base_path): test = input_params.test # update config files - dlcpath = update_config_files_general(dlcpath) + dlcpath = update_config_files_general(dlcpath,shuffle) update_configs = True ## Specifying snapshot manually at the moment assuming training. diff --git a/demo/run_dgp_demo.py b/demo/run_dgp_demo.py index 5a58428..0c5d4c8 100644 --- a/demo/run_dgp_demo.py +++ b/demo/run_dgp_demo.py @@ -27,8 +27,9 @@ from deepgraphpose.models.eval import plot_dgp -def update_config_files_general(dlcpath): +def update_config_files_general(dlcpath,shuffle): """General purpose version of the function below that applies to all models, not just the default reachingvideo. + Requires in addition to the dlc path parameters from the project config file: """ base_path = os.getcwd() @@ -38,6 +39,9 @@ def update_config_files_general(dlcpath): with open(proj_cfg_path, 'r') as f: yaml_cfg = yaml.load(f, Loader=yaml.SafeLoader) yaml_cfg['project_path'] = os.path.join(base_path, dlcpath) + task = yaml_cfg["Task"] + TrainingFraction = yaml_cfg["TrainingFraction"] + date = yaml_cfg["date"] # video_loc = os.path.join(base_path, dlcpath, 'videos', 'reachingvideo1.avi') # try: # yaml_cfg['video_sets'][video_loc] = yaml_cfg['video_sets'].pop('videos/reachingvideo1.avi') @@ -48,7 +52,8 @@ def update_config_files_general(dlcpath): yaml.dump(yaml_cfg, f) # train model config - model_cfg_path = get_model_cfg_path(base_path, 'train') + projectname = "{t}{d}-trainset{tf}shuffle{s}".format(t=Task,d=date,tf = int(TrainingFraction*100),s = shuffle) + model_cfg_path = get_model_cfg_path_general(base_path, 'train', projectname) with open(model_cfg_path, 'r') as f: yaml_cfg = yaml.load(f, Loader=yaml.SafeLoader) yaml_cfg['init_weights'] = get_init_weights_path(base_path) @@ -61,7 +66,7 @@ def update_config_files_general(dlcpath): raise FileNotFoundError('Must download resnet-50 weights; see README for instructions') # test model config - model_cfg_path = get_model_cfg_path(base_path, 'test') + model_cfg_path = get_model_cfg_path_general(base_path, 'test', projectname) with open(model_cfg_path, 'r') as f: yaml_cfg = yaml.load(f, Loader=yaml.SafeLoader) yaml_cfg['init_weights'] = get_init_weights_path(base_path) @@ -148,9 +153,12 @@ def get_model_cfg_path(base_path, dtype): base_path, dlcpath, 'dlc-models', 'iteration-0', 'ReachingAug30-trainset95shuffle1', dtype, 'pose_cfg.yaml') -def get_model_cfg_path_general(base_path, dtype): +def get_model_cfg_path_general(base_path, dtype, projectname): + """General purpose version of get_model_cfg_path that can work with non-demo projects. + + """ return os.path.join( - base_path, dlcpath, 'dlc-models', 'iteration-0', 'ReachingAug30-trainset95shuffle1', + base_path, dlcpath, 'dlc-models', 'iteration-0', projectname, dtype, 'pose_cfg.yaml') @@ -196,7 +204,7 @@ def get_init_weights_path(base_path): test = input_params.test # update config files - dlcpath = update_config_files_general(dlcpath) + dlcpath = update_config_files_general(dlcpath,shuffle) update_configs = True # ------------------------------------------------------------------------------------ From 302d982fcf39d8353a4572ddefd8acec066b1c5d Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Wed, 12 May 2021 21:14:40 +0000 Subject: [PATCH 11/24] updated dgp to accept video sets. --- demo/run_dgp_demo.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/demo/run_dgp_demo.py b/demo/run_dgp_demo.py index 0c5d4c8..fdac89b 100644 --- a/demo/run_dgp_demo.py +++ b/demo/run_dgp_demo.py @@ -40,8 +40,15 @@ def update_config_files_general(dlcpath,shuffle): yaml_cfg = yaml.load(f, Loader=yaml.SafeLoader) yaml_cfg['project_path'] = os.path.join(base_path, dlcpath) task = yaml_cfg["Task"] - TrainingFraction = yaml_cfg["TrainingFraction"] + TrainingFraction = yaml_cfg["TrainingFraction"][0] ## Train with the first training fraction. date = yaml_cfg["date"] + try: + video_locs = yaml_cfg["video_sets"] + full_sets = [os.path.join(base_path,dlcpath,vl) for vl in video_locs] + yaml_cfg["video_sets"] = full_sets + except KeyError: + print("no videos given.") + # video_loc = os.path.join(base_path, dlcpath, 'videos', 'reachingvideo1.avi') # try: # yaml_cfg['video_sets'][video_loc] = yaml_cfg['video_sets'].pop('videos/reachingvideo1.avi') @@ -52,7 +59,7 @@ def update_config_files_general(dlcpath,shuffle): yaml.dump(yaml_cfg, f) # train model config - projectname = "{t}{d}-trainset{tf}shuffle{s}".format(t=Task,d=date,tf = int(TrainingFraction*100),s = shuffle) + projectname = "{t}{d}-trainset{tf}shuffle{s}".format(t=task,d=date,tf = int(TrainingFraction*100),s = shuffle) model_cfg_path = get_model_cfg_path_general(base_path, 'train', projectname) with open(model_cfg_path, 'r') as f: yaml_cfg = yaml.load(f, Loader=yaml.SafeLoader) From c7feae4ec250958725af0751a19aba563f8292ca Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Wed, 12 May 2021 21:17:46 +0000 Subject: [PATCH 12/24] demo code now works on a non-mackenzie dataset. --- demo/run_dgp_demo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demo/run_dgp_demo.py b/demo/run_dgp_demo.py index fdac89b..d24a5dc 100644 --- a/demo/run_dgp_demo.py +++ b/demo/run_dgp_demo.py @@ -44,7 +44,7 @@ def update_config_files_general(dlcpath,shuffle): date = yaml_cfg["date"] try: video_locs = yaml_cfg["video_sets"] - full_sets = [os.path.join(base_path,dlcpath,vl) for vl in video_locs] + full_sets = {os.path.join(base_path,dlcpath,vl):cropdata for vl,cropdata in video_locs.items()} yaml_cfg["video_sets"] = full_sets except KeyError: print("no videos given.") From 1abe8f2d367ffb61a1995dffb538384bc132c783 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Wed, 12 May 2021 21:46:57 +0000 Subject: [PATCH 13/24] fixed writepaths everywhere. --- demo/predict_dgp_demo.py | 94 ++-------------------------------------- 1 file changed, 4 insertions(+), 90 deletions(-) diff --git a/demo/predict_dgp_demo.py b/demo/predict_dgp_demo.py index 3ebf60b..1acfcfe 100644 --- a/demo/predict_dgp_demo.py +++ b/demo/predict_dgp_demo.py @@ -27,92 +27,6 @@ from deepgraphpose.models.eval import plot_dgp from run_dgp_demo import update_config_files_general - -def update_config_files(dlcpath): - base_path = os.getcwd() - - # project config - proj_cfg_path = os.path.join(base_path, dlcpath, 'config.yaml') - with open(proj_cfg_path, 'r') as f: - yaml_cfg = yaml.load(f, Loader=yaml.SafeLoader) - yaml_cfg['project_path'] = os.path.join(base_path, dlcpath) - video_loc = os.path.join(base_path, dlcpath, 'videos', 'reachingvideo1.avi') - try: - yaml_cfg['video_sets'][video_loc] = yaml_cfg['video_sets'].pop('videos/reachingvideo1.avi') - except KeyError: - ## check if update has already been done. - assert yaml_cfg["video_sets"].get(video_loc,False), "Can't find original or updated video path in config file." - with open(proj_cfg_path, 'w') as f: - yaml.dump(yaml_cfg, f) - - # train model config - model_cfg_path = get_model_cfg_path(base_path, 'train') - with open(model_cfg_path, 'r') as f: - yaml_cfg = yaml.load(f, Loader=yaml.SafeLoader) - yaml_cfg['init_weights'] = get_init_weights_path(base_path) - yaml_cfg['project_path'] = os.path.join(base_path, dlcpath) - with open(model_cfg_path, 'w') as f: - yaml.dump(yaml_cfg, f) - - # download resnet weights if necessary - if not os.path.exists(yaml_cfg['init_weights']): - raise FileNotFoundError('Must download resnet-50 weights; see README for instructions') - - # test model config - model_cfg_path = get_model_cfg_path(base_path, 'test') - with open(model_cfg_path, 'r') as f: - yaml_cfg = yaml.load(f, Loader=yaml.SafeLoader) - yaml_cfg['init_weights'] = get_init_weights_path(base_path) - with open(model_cfg_path, 'w') as f: - yaml.dump(yaml_cfg, f) - - return os.path.join(base_path, dlcpath) - - -def return_configs(): - base_path = os.getcwd() - dlcpath = 'data/Reaching-Mackenzie-2018-08-30' - - # project config - proj_cfg_path = os.path.join(base_path, dlcpath, 'config.yaml') - with open(proj_cfg_path, 'r') as f: - yaml_cfg = yaml.load(f, Loader=yaml.SafeLoader) - yaml_cfg['project_path'] = dlcpath - video_loc = os.path.join(base_path, dlcpath, 'videos', 'reachingvideo1.avi') - yaml_cfg['video_sets']['videos/reachingvideo1.avi'] = yaml_cfg['video_sets'].pop(video_loc) - with open(proj_cfg_path, 'w') as f: - yaml.dump(yaml_cfg, f) - - # train model config - model_cfg_path = get_model_cfg_path(base_path, 'train') - with open(model_cfg_path, 'r') as f: - yaml_cfg = yaml.load(f, Loader=yaml.SafeLoader) - yaml_cfg['init_weights'] = 'resnet_v1_50.ckpt' - yaml_cfg['project_path'] = dlcpath - with open(model_cfg_path, 'w') as f: - yaml.dump(yaml_cfg, f) - - # test model config - model_cfg_path = get_model_cfg_path(base_path, 'test') - with open(model_cfg_path, 'r') as f: - yaml_cfg = yaml.load(f, Loader=yaml.SafeLoader) - yaml_cfg['init_weights'] = 'resnet_v1_50.ckpt' - with open(model_cfg_path, 'w') as f: - yaml.dump(yaml_cfg, f) - - -def get_model_cfg_path(base_path, dtype): - return os.path.join( - base_path, dlcpath, 'dlc-models', 'iteration-0', 'ReachingAug30-trainset95shuffle1', - dtype, 'pose_cfg.yaml') - - -def get_init_weights_path(base_path): - return os.path.join( - base_path, 'src', 'DeepLabCut', 'deeplabcut', 'pose_estimation_tensorflow', - 'models', 'pretrained', 'resnet_v1_50.ckpt') - - if __name__ == '__main__': # %% set up dlcpath for DLC project and hyperparameters @@ -202,12 +116,11 @@ def get_init_weights_path(base_path): clip =VideoFileClip(str(video_file)) if clip.duration > 10: clip = clip.subclip(10) - video_file_name = video_file.rsplit('/', 1)[-1].rsplit('.',1)[0] + '.mp4' + video_file_name = os.path.splitext(video_file)[0] +"test"+ ".mp4" print('\nwriting {}'.format(video_file_name)) clip.write_videofile(video_file_name) - output_dir = os.getcwd() + '/' - plot_dgp(video_file=str(video_file_name), - output_dir=output_dir, + plot_dgp(str(video_file_name), + str(video_pred_path), proj_cfg_file=str(cfg_yaml), dgp_model_file=str(snapshot_path), shuffle=shuffle) @@ -219,6 +132,7 @@ def get_init_weights_path(base_path): dgp_model_file=str(snapshot_path), shuffle=shuffle) finally: + pass #if update_configs: # return_configs() From f8fe65b1b6f4d0ac164103cd977e140285b4be79 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Wed, 12 May 2021 22:54:29 +0000 Subject: [PATCH 14/24] made demo work for general purpose model folders. --- demo/predict_dgp_demo.py | 0 demo/run_dgp_demo.py | 6 +++--- 2 files changed, 3 insertions(+), 3 deletions(-) mode change 100644 => 100755 demo/predict_dgp_demo.py mode change 100644 => 100755 demo/run_dgp_demo.py diff --git a/demo/predict_dgp_demo.py b/demo/predict_dgp_demo.py old mode 100644 new mode 100755 diff --git a/demo/run_dgp_demo.py b/demo/run_dgp_demo.py old mode 100644 new mode 100755 index d24a5dc..5f2d1aa --- a/demo/run_dgp_demo.py +++ b/demo/run_dgp_demo.py @@ -60,7 +60,7 @@ def update_config_files_general(dlcpath,shuffle): # train model config projectname = "{t}{d}-trainset{tf}shuffle{s}".format(t=task,d=date,tf = int(TrainingFraction*100),s = shuffle) - model_cfg_path = get_model_cfg_path_general(base_path, 'train', projectname) + model_cfg_path = get_model_cfg_path_general(base_path, dlcpath, 'train', projectname) with open(model_cfg_path, 'r') as f: yaml_cfg = yaml.load(f, Loader=yaml.SafeLoader) yaml_cfg['init_weights'] = get_init_weights_path(base_path) @@ -73,7 +73,7 @@ def update_config_files_general(dlcpath,shuffle): raise FileNotFoundError('Must download resnet-50 weights; see README for instructions') # test model config - model_cfg_path = get_model_cfg_path_general(base_path, 'test', projectname) + model_cfg_path = get_model_cfg_path_general(base_path, dlcpath, 'test', projectname) with open(model_cfg_path, 'r') as f: yaml_cfg = yaml.load(f, Loader=yaml.SafeLoader) yaml_cfg['init_weights'] = get_init_weights_path(base_path) @@ -160,7 +160,7 @@ def get_model_cfg_path(base_path, dtype): base_path, dlcpath, 'dlc-models', 'iteration-0', 'ReachingAug30-trainset95shuffle1', dtype, 'pose_cfg.yaml') -def get_model_cfg_path_general(base_path, dtype, projectname): +def get_model_cfg_path_general(base_path, dlcpath, dtype, projectname): """General purpose version of get_model_cfg_path that can work with non-demo projects. """ From 850d4ee113a6125408f1fef0c22f5f985607720d Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Mon, 24 May 2021 13:34:41 +0000 Subject: [PATCH 15/24] running prediction on remote instance. --- demo/predict_dgp_ensemble.py | 140 +++++++++++++++++++++++++++++++++++ 1 file changed, 140 insertions(+) create mode 100755 demo/predict_dgp_ensemble.py diff --git a/demo/predict_dgp_ensemble.py b/demo/predict_dgp_ensemble.py new file mode 100755 index 0000000..4271d8b --- /dev/null +++ b/demo/predict_dgp_ensemble.py @@ -0,0 +1,140 @@ +# If you have collected labels using DLC's GUI you can run DGP with the following +"""Ensemble fitting function for DGP. + step 0: run DLC + step 1: run DGP with labeled frames only + step 2: run DGP with spatial clique + step 3: do prediction on all videos +""" +import argparse +import os +from os import listdir +from os.path import isfile, join +from pathlib import Path +import sys +import yaml + +if sys.platform == 'darwin': + import wx + if int(wx.__version__[0]) > 3: + wx.Thread_IsMain = wx.IsMainThread + +os.environ["DLClight"] = "True" +os.environ["Colab"] = "True" +from deeplabcut.utils import auxiliaryfunctions + +from deepgraphpose.models.fitdgp import fit_dlc, fit_dgp, fit_dgp_labeledonly +from deepgraphpose.models.fitdgp_util import get_snapshot_path +from deepgraphpose.models.eval import plot_dgp +from run_dgp_demo import update_config_files_general + +from dgp_ensembletools.models import Ensemble + +if __name__ == '__main__': + + # %% set up dlcpath for DLC project and hyperparameters + parser = argparse.ArgumentParser() + parser.add_argument( + "--dlcpath", + type=str, + default=None, + help="the absolute path of the DLC project", + ) + + parser.add_argument( + "--dlcsnapshot", + type=str, + default=None, + help="use the DLC snapshot to initialize DGP", + ) + + parser.add_argument("--shuffle", type=int, default=1, help="Project shuffle") + parser.add_argument( + "--batch_size", + type=int, + default=10, + help="size of the batch, if there are memory issues, decrease it value") + parser.add_argument("--test", action='store_true', default=False) + + input_params = parser.parse_known_args()[0] + print(input_params) + + dlcpath = input_params.dlcpath + shuffle = input_params.shuffle + dlcsnapshot = input_params.dlcsnapshot + batch_size = input_params.batch_size + test = input_params.test + + # update config files + dlcpath = update_config_files_general(dlcpath,shuffle) + update_configs = True + + ## Specifying snapshot manually at the moment assuming training. + step = 2 + snapshot = 'snapshot-step{}-final--0'.format(step) + + try: + + # -------------------------------------------------------------------------------- + # Test DGP model + # -------------------------------------------------------------------------------- + + # %% step 3 predict on all videos in videos_dgp folder + print( + ''' + ========================== + | | + | | + | Predict with DGP | + | | + | | + ========================== + ''' + , flush=True) + + snapshot_path, cfg_yaml = get_snapshot_path(snapshot, dlcpath, shuffle=shuffle) + cfg = auxiliaryfunctions.read_config(cfg_yaml) + + video_path = str(Path(dlcpath) / 'videos_dgp') + if not (os.path.exists(video_path)): + print(video_path + " does not exist!") + video_sets = list(cfg['video_sets']) + else: + video_sets = [ + video_path + '/' + f for f in listdir(video_path) + if isfile(join(video_path, f)) and ( + f.find('avi') > 0 or f.find('mp4') > 0 or f.find('mov') > 0 or f.find( + 'mkv') > 0) + ] + + video_pred_path = str(Path(dlcpath) / 'videos_pred') + if not os.path.exists(video_pred_path): + os.makedirs(video_pred_path) + + print('video_sets', video_sets, flush=True) + + if test: + for video_file in [video_sets[0]]: + from moviepy.editor import VideoFileClip + clip =VideoFileClip(str(video_file)) + if clip.duration > 10: + clip = clip.subclip(10) + video_file_name = os.path.splitext(video_file)[0] +"test"+ ".mp4" + print('\nwriting {}'.format(video_file_name)) + clip.write_videofile(video_file_name) + plot_dgp(str(video_file_name), + str(video_pred_path), + proj_cfg_file=str(cfg_yaml), + dgp_model_file=str(snapshot_path), + shuffle=shuffle) + else: + for video_file in video_sets: + plot_dgp(str(video_file), + str(video_pred_path), + proj_cfg_file=str(cfg_yaml), + dgp_model_file=str(snapshot_path), + shuffle=shuffle) + finally: + pass + + #if update_configs: + # return_configs() From 3552a0445f7055419eb4e8fbe47582c3593e0d62 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Mon, 24 May 2021 17:36:51 +0000 Subject: [PATCH 16/24] changed run dgp general parsing file to assume video paths are given relative to this dir. --- demo/run_dgp_demo.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/demo/run_dgp_demo.py b/demo/run_dgp_demo.py index 5f2d1aa..b6321b1 100755 --- a/demo/run_dgp_demo.py +++ b/demo/run_dgp_demo.py @@ -44,7 +44,9 @@ def update_config_files_general(dlcpath,shuffle): date = yaml_cfg["date"] try: video_locs = yaml_cfg["video_sets"] - full_sets = {os.path.join(base_path,dlcpath,vl):cropdata for vl,cropdata in video_locs.items()} + #full_sets = {os.path.join(base_path,dlcpath,vl):cropdata for vl,cropdata in video_locs.items()} + ## TODO Test this bottom line: assume videos are correctly located in the directory's videos subdir. + full_sets = {os.path.join(base_path,dlcpath,"videos",os.path.basename(vl)):cropdata for vl,cropdata in video_locs.items()} yaml_cfg["video_sets"] = full_sets except KeyError: print("no videos given.") From 8a0fd86bb4b9107d5a4634ebe695ee84ef91dbbd Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Mon, 24 May 2021 19:52:05 +0000 Subject: [PATCH 17/24] dgp prediction code for ensemble works now. --- demo/predict_dgp_ensemble.py | 140 +++++++++++++++++++++-------------- 1 file changed, 86 insertions(+), 54 deletions(-) diff --git a/demo/predict_dgp_ensemble.py b/demo/predict_dgp_ensemble.py index 4271d8b..20e4506 100755 --- a/demo/predict_dgp_ensemble.py +++ b/demo/predict_dgp_ensemble.py @@ -10,8 +10,11 @@ from os import listdir from os.path import isfile, join from pathlib import Path +from joblib import Memory import sys import yaml +import pandas as pd +import numpy as np if sys.platform == 'darwin': import wx @@ -20,6 +23,9 @@ os.environ["DLClight"] = "True" os.environ["Colab"] = "True" + +from moviepy.editor import VideoFileClip + from deeplabcut.utils import auxiliaryfunctions from deepgraphpose.models.fitdgp import fit_dlc, fit_dgp, fit_dgp_labeledonly @@ -34,10 +40,11 @@ # %% set up dlcpath for DLC project and hyperparameters parser = argparse.ArgumentParser() parser.add_argument( - "--dlcpath", + "--modelpaths", + nargs="+", type=str, default=None, - help="the absolute path of the DLC project", + help="the absolute path of the DLC projects you want to ensemble", ) parser.add_argument( @@ -47,6 +54,13 @@ help="use the DLC snapshot to initialize DGP", ) + parser.add_argument( + "--videopath", + type=str, + default=None, + help="path to video", + ) + parser.add_argument("--shuffle", type=int, default=1, help="Project shuffle") parser.add_argument( "--batch_size", @@ -58,15 +72,17 @@ input_params = parser.parse_known_args()[0] print(input_params) - dlcpath = input_params.dlcpath + modelpaths = input_params.modelpaths shuffle = input_params.shuffle + videopath = input_params.videopath dlcsnapshot = input_params.dlcsnapshot batch_size = input_params.batch_size test = input_params.test # update config files - dlcpath = update_config_files_general(dlcpath,shuffle) - update_configs = True + for modelpath in modelpaths: + dlcpath = update_config_files_general(modelpath,shuffle) + update_configs = True ## Specifying snapshot manually at the moment assuming training. step = 2 @@ -81,58 +97,74 @@ # %% step 3 predict on all videos in videos_dgp folder print( ''' - ========================== - | | - | | - | Predict with DGP | - | | - | | - ========================== + ================================== + | | + | | + | Predict with DGP Ensemble | + | | + | | + ================================== ''' , flush=True) - snapshot_path, cfg_yaml = get_snapshot_path(snapshot, dlcpath, shuffle=shuffle) - cfg = auxiliaryfunctions.read_config(cfg_yaml) - - video_path = str(Path(dlcpath) / 'videos_dgp') - if not (os.path.exists(video_path)): - print(video_path + " does not exist!") - video_sets = list(cfg['video_sets']) - else: - video_sets = [ - video_path + '/' + f for f in listdir(video_path) - if isfile(join(video_path, f)) and ( - f.find('avi') > 0 or f.find('mp4') > 0 or f.find('mov') > 0 or f.find( - 'mkv') > 0) - ] - - video_pred_path = str(Path(dlcpath) / 'videos_pred') - if not os.path.exists(video_pred_path): - os.makedirs(video_pred_path) - - print('video_sets', video_sets, flush=True) - - if test: - for video_file in [video_sets[0]]: - from moviepy.editor import VideoFileClip - clip =VideoFileClip(str(video_file)) - if clip.duration > 10: - clip = clip.subclip(10) - video_file_name = os.path.splitext(video_file)[0] +"test"+ ".mp4" - print('\nwriting {}'.format(video_file_name)) - clip.write_videofile(video_file_name) - plot_dgp(str(video_file_name), - str(video_pred_path), - proj_cfg_file=str(cfg_yaml), - dgp_model_file=str(snapshot_path), - shuffle=shuffle) - else: - for video_file in video_sets: - plot_dgp(str(video_file), - str(video_pred_path), - proj_cfg_file=str(cfg_yaml), - dgp_model_file=str(snapshot_path), - shuffle=shuffle) + ## get ensembleparameters + topdir = os.path.dirname(modelpaths[0]) ## they're all loaded into the same anyway. + modeldirs = [os.path.basename(m) for m in modelpaths] + videoext = os.path.splitext(videopath)[-1].split(".",1)[-1] ## remove the dot as well. + video = VideoFileClip(videopath) + ## Write results to: + resultpath = os.path.join(os.path.dirname(os.path.dirname(videopath)),"results") + + + + framelength = int(video.duration*video.fps) + ## can do some processing based on length here. + framerange = range(0,framelength) + + remoteensemble = Ensemble(topdir,modeldirs,videoext,memory = Memory(os.path.dirname(videopath))) + [model.predict(videopath) for model in remoteensemble.models.values()] + predict_videoname = "_labeled".join(os.path.splitext(os.path.basename(videopath))) + predict_h5name = "_labeled".join([os.path.splitext(os.path.basename(videopath))[0],".h5"]) + consensus_videoname = "_labeled_consensus".join(os.path.splitext(os.path.basename(videopath))) + consensus_csvname = "_labeled_consensus".join([os.path.splitext(os.path.basename(videopath))[0],".csv"]) + consensus_h5name = "_labeled_consensus".join([os.path.splitext(os.path.basename(videopath))[0],".h5"]) + ## outputs pose of shape xy, time, body part + meanx,meany = remoteensemble.get_mean_pose(predict_videoname,framerange,snapshot = snapshot, shuffle = shuffle) + + ## reshape and save in shape of existing: + likearray = np.empty(meanx.shape) ## don't get likelihoods right now. + likearray[:] = np.NaN + stacked = np.stack((meanx,meany,likearray),axis = -1) + dfshaped = stacked.reshape(stacked.shape[0],stacked.shape[1]*stacked.shape[2]) + ## get sample dataframe: + sampledf = pd.read_hdf(os.path.join(modelpaths[0],"videos_pred",predict_h5name)) + sampledf.iloc[:len(dfshaped),:] = dfshaped + sampledf.drop([i for i in range(len(dfshaped),len(sampledf))],inplace = True) + + sampledf.to_csv(os.path.join(resultpath,consensus_csvname)) + sampledf.to_hdf(os.path.join(resultpath,consensus_h5name),key="consensus") + + ### Not writing video of consensus for now: + #if test: + # for video_file in [video_sets[0]]: + # clip =VideoFileClip(str(video_file)) + # if clip.duration > 10: + # clip = clip.subclip(10) + # video_file_name = os.path.splitext(video_file)[0] +"test"+ ".mp4" + # print('\nwriting {}'.format(video_file_name)) + # clip.write_videofile(video_file_name) + # plot_dgp(str(video_file_name), + # str(video_pred_path), + # proj_cfg_file=str(cfg_yaml), + # dgp_model_file=str(snapshot_path), + # shuffle=shuffle) + #else: + # for video_file in video_sets: + # plot_dgp(str(video_file), + # str(video_pred_path), + # proj_cfg_file=str(cfg_yaml), + # dgp_model_file=str(snapshot_path), + # shuffle=shuffle) finally: pass From 11b518f0b93aa1a0be9740973f5872c29192f390 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Mon, 24 May 2021 19:57:00 +0000 Subject: [PATCH 18/24] make result dir if does not exist. --- demo/predict_dgp_ensemble.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/demo/predict_dgp_ensemble.py b/demo/predict_dgp_ensemble.py index 20e4506..c1d4f52 100755 --- a/demo/predict_dgp_ensemble.py +++ b/demo/predict_dgp_ensemble.py @@ -141,6 +141,9 @@ sampledf.iloc[:len(dfshaped),:] = dfshaped sampledf.drop([i for i in range(len(dfshaped),len(sampledf))],inplace = True) + if not os.path.isdir(resultpath): + os.makedirs(resultpath) + sampledf.to_csv(os.path.join(resultpath,consensus_csvname)) sampledf.to_hdf(os.path.join(resultpath,consensus_h5name),key="consensus") From df9bd6687f5747e530e452950d21cc16c5e7668b Mon Sep 17 00:00:00 2001 From: cellistigs Date: Tue, 31 Aug 2021 13:11:23 -0400 Subject: [PATCH 19/24] stubbed out segment_videos function. --- src/deepgraphpose/contrib/segment_videos.py | 38 +++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 src/deepgraphpose/contrib/segment_videos.py diff --git a/src/deepgraphpose/contrib/segment_videos.py b/src/deepgraphpose/contrib/segment_videos.py new file mode 100644 index 0000000..833b862 --- /dev/null +++ b/src/deepgraphpose/contrib/segment_videos.py @@ -0,0 +1,38 @@ +# Given a DLC project, check all videos and clip those that are longer than some tolerance into shorter disjoint clips. + +def split_video_and_trainframes(config_path,tol=1000,suffix): + """Splits videos and trainframes in a model config that are larger than some tolerance in frames. + + :param config_path: parameter to config file. + :param tol: tolerance in number of frames. + """ + vids = check_videos(config_path,tol) + splitlength = tol + for v in vids: + split_video(v,splitlength,suffix) + format_frames(v,splitlength,suffix) + +def check_videos(config_path,tol): + """Checks all videos given in the model directory and checks if any are longer than the given length. + + :param config_path: parameter to config file. + :param tol: tolerance in number of frames. + """ + +def split_video(vidpath,splitlength,suffix = None): + """splits a given video into subclips. + + :param vidpath: path to video + :param splitlength: length to chunk into + :param suffix: custom suffix to add to subclips + """ + +def format_frames(vidpath,splitlength,suffix = None): + """reformats training frames into format that matches sublclips + + :param vidpath: path to video + :param splitlength: length to chunk into + :param suffix: custom suffix to add to subclips + """ + + From d741792aba4bf53a266c316dc946fd2fa22fdb50 Mon Sep 17 00:00:00 2001 From: cellistigs Date: Tue, 14 Sep 2021 12:54:17 -0400 Subject: [PATCH 20/24] added raw video split code. --- src/deepgraphpose/contrib/segment_videos.py | 61 ++++++++++++++++++--- tests/test_segment_videos.py | 38 +++++++++++++ 2 files changed, 92 insertions(+), 7 deletions(-) create mode 100644 tests/test_segment_videos.py diff --git a/src/deepgraphpose/contrib/segment_videos.py b/src/deepgraphpose/contrib/segment_videos.py index 833b862..3856610 100644 --- a/src/deepgraphpose/contrib/segment_videos.py +++ b/src/deepgraphpose/contrib/segment_videos.py @@ -1,31 +1,78 @@ +import os +from moviepy.editor import VideoFileClip # Given a DLC project, check all videos and clip those that are longer than some tolerance into shorter disjoint clips. -def split_video_and_trainframes(config_path,tol=1000,suffix): +# Take the project, and look within for all videos that will be trained on (these come from the config file) and analyzed (these come from the folder videos_dgp). +# Those videos that have labels need to be split on the training labels as well. + +def split_video_and_trainframes(config_path,tol=5000,suffix=None): """Splits videos and trainframes in a model config that are larger than some tolerance in frames. :param config_path: parameter to config file. :param tol: tolerance in number of frames. + :param suffix: video suffix. """ - vids = check_videos(config_path,tol) + trainvids = check_videos(config_path,tol) + analyzevids = check_analysis_videos(folder,tol) splitlength = tol + vids = trainvids + analyzevids for v in vids: split_video(v,splitlength,suffix) - format_frames(v,splitlength,suffix) + if v in trainvids: + format_frames(v,splitlength,suffix) def check_videos(config_path,tol): - """Checks all videos given in the model directory and checks if any are longer than the given length. + """Checks all videos given in the model cfg file and checks if any are longer than the given length. + + :param config_path: parameter to config file. + :param tol: tolerance in number of frames. + """ + +def check_analysis_videos(folder_path,tol): + """Checks all videos given in the videos_dgp directory and checks if any are longer than the given length. :param config_path: parameter to config file. :param tol: tolerance in number of frames. """ -def split_video(vidpath,splitlength,suffix = None): - """splits a given video into subclips. +def split_video(vidpath,splitlength,suffix = "",outputloc = None): + """splits a given video into subclips of type mp4. Note: will work best (even frames per subclip) if you pass a splitlength that is divisible by your frame rate. :param vidpath: path to video - :param splitlength: length to chunk into + :param splitlength: length to chunk into in frames :param suffix: custom suffix to add to subclips + :param outputloc: directory to write outputs to. Default is same directory. + :returns: list of paths to new video files. """ + try: + clip = VideoFileClip(vidpath) + except FileNotFoundError: + print("file not found.") + + duration = clip.duration + splitlength_secs = splitlength/clip.fps + viddir,vidname = os.path.dirname(vidpath),os.path.basename(vidpath) + base,ext = os.path.splitext(vidname) + subname = base+suffix+"{n}"+".mp4" + if outputloc is None: + subpath = os.path.join(viddir,subname) + else: + subpath = os.path.join(outputloc,subname) + + clipnames = [] + clipstart = 0 + clipind = 0 + while clipstart < duration: + subname = subpath.format(n=clipind) + subclip = clip.subclip(clipstart,min(duration,clipstart+splitlength_secs)) + subclip.write_videofile(subname,codec = "mpeg4") + clipnames.append(subname) + clipstart += splitlength_secs + clipind+=1 + return clipnames + + + def format_frames(vidpath,splitlength,suffix = None): """reformats training frames into format that matches sublclips diff --git a/tests/test_segment_videos.py b/tests/test_segment_videos.py new file mode 100644 index 0000000..2afb479 --- /dev/null +++ b/tests/test_segment_videos.py @@ -0,0 +1,38 @@ +import pytest +from deepgraphpose.contrib.segment_videos import split_video +from moviepy.editor import VideoFileClip +import math +import os + +here = os.path.abspath(os.path.dirname(__file__)) +testmodel_path = os.path.join(here,"testmodel") + +def test_split_video(tmp_path): + output = tmp_path/"subclips" + output.mkdir() + frame_duration = 30 + vidpath = os.path.join(testmodel_path,"videos","reachingvideo1.avi") + video_locs = split_video(vidpath,frame_duration,suffix = "test",outputloc = str(output)) + + origclip = VideoFileClip(vidpath) + duration = origclip.duration*origclip.fps + assert len(video_locs) == math.ceil(duration/frame_duration) + vid_inds = [] + for vi in video_locs: + prefix = os.path.splitext(os.path.basename(vidpath))[0]+"test" + assert os.path.splitext(os.path.basename(vi))[0].startswith(prefix) + vid_inds.append(int(vi.split(prefix)[-1].split(".mp4")[0])) + sub = VideoFileClip(vi) + assert sub.duration*sub.fps - frame_duration < 1e-1 + assert set(vid_inds) == set(range(len(video_locs))) + + + + + + + + + + + From a5a4bec3fa5b4aae7377d5d03768f7dc86e302c6 Mon Sep 17 00:00:00 2001 From: cellistigs Date: Tue, 14 Sep 2021 12:59:39 -0400 Subject: [PATCH 21/24] added option to split videos. --- demo/run_dgp_demo.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/demo/run_dgp_demo.py b/demo/run_dgp_demo.py index 875a6bd..1c41e42 100644 --- a/demo/run_dgp_demo.py +++ b/demo/run_dgp_demo.py @@ -21,7 +21,7 @@ os.environ["DLClight"] = "True" os.environ["Colab"] = "True" from deeplabcut.utils import auxiliaryfunctions - +from deepgraphpose.contrib.segment_videos import split_video from deepgraphpose.models.fitdgp import fit_dlc, fit_dgp, fit_dgp_labeledonly from deepgraphpose.models.fitdgp_util import get_snapshot_path from deepgraphpose.models.eval import plot_dgp @@ -136,6 +136,8 @@ def get_init_weights_path(base_path): default=10, help="size of the batch, if there are memory issues, decrease it value") parser.add_argument("--test", action='store_true', default=False) + parser.add_argument("--split", action = "store_true",help = "whether or not we should run inference on chopped up videos") + parser.add_argument("--splitlength", default = 6000, help= "number of frames in block if splitting videos. ") input_params = parser.parse_known_args()[0] print(input_params) @@ -146,6 +148,8 @@ def get_init_weights_path(base_path): batch_size = input_params.batch_size test = input_params.test + split,splitlength = input_params.split,input_params.splitlength + update_configs = False if dlcpath == join('data','Reaching-Mackenzie-2018-08-30'): # update config files @@ -283,6 +287,14 @@ def get_init_weights_path(base_path): os.makedirs(video_pred_path) print('video_sets', video_sets, flush=True) + if split: + video_cut_path = str(Path(dlcpath) / 'videos_cut') + if not os.path.exists(video_cut_path): + os.makedirs(video_cut_path) + clip_sets = [] + for v in video_sets: + clip_sets.extend(split_video(v,splitlength,suffix = "demo",video_cut_path)) + video_sets = clip_sets ## replace video_sets with clipped versions. if test: for video_file in [video_sets[0]]: From 938be7b28184323601e6c09a8dd64bfbaf6c776a Mon Sep 17 00:00:00 2001 From: cellistigs Date: Tue, 14 Sep 2021 16:53:35 -0400 Subject: [PATCH 22/24] uncommented workflow. --- demo/run_dgp_demo.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/demo/run_dgp_demo.py b/demo/run_dgp_demo.py index 1c41e42..178e730 100644 --- a/demo/run_dgp_demo.py +++ b/demo/run_dgp_demo.py @@ -148,7 +148,7 @@ def get_init_weights_path(base_path): batch_size = input_params.batch_size test = input_params.test - split,splitlength = input_params.split,input_params.splitlength + splitflag,splitlength = input_params.split,input_params.splitlength update_configs = False if dlcpath == join('data','Reaching-Mackenzie-2018-08-30'): @@ -287,13 +287,13 @@ def get_init_weights_path(base_path): os.makedirs(video_pred_path) print('video_sets', video_sets, flush=True) - if split: + if splitflag: video_cut_path = str(Path(dlcpath) / 'videos_cut') if not os.path.exists(video_cut_path): os.makedirs(video_cut_path) clip_sets = [] for v in video_sets: - clip_sets.extend(split_video(v,splitlength,suffix = "demo",video_cut_path)) + clip_sets.extend(split_video(v,int(splitlength),suffix = "demo",outputloc = video_cut_path)) video_sets = clip_sets ## replace video_sets with clipped versions. if test: From 3bf04663200f95aea0176e81fed1ca7fca45ae6b Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 16 Sep 2021 21:17:09 +0000 Subject: [PATCH 23/24] changed predict. --- demo/predict_dgp_demo.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/demo/predict_dgp_demo.py b/demo/predict_dgp_demo.py index 1acfcfe..131b13b 100755 --- a/demo/predict_dgp_demo.py +++ b/demo/predict_dgp_demo.py @@ -53,6 +53,9 @@ help="size of the batch, if there are memory issues, decrease it value") parser.add_argument("--test", action='store_true', default=False) + parser.add_argument("--split", action = "store_true",help = "whether or not we should run inference on chopped up videos") + parser.add_argument("--splitlength", default = 6000, help= "number of frames in block if splitting videos. ") + input_params = parser.parse_known_args()[0] print(input_params) @@ -61,6 +64,8 @@ dlcsnapshot = input_params.dlcsnapshot batch_size = input_params.batch_size test = input_params.test + splitflag,splitlength = input_params.split,input_params.splitlength + # update config files dlcpath = update_config_files_general(dlcpath,shuffle) @@ -109,6 +114,14 @@ os.makedirs(video_pred_path) print('video_sets', video_sets, flush=True) + if splitflag: + video_cut_path = str(Path(dlcpath) / 'videos_cut') + if not os.path.exists(video_cut_path): + os.makedirs(video_cut_path) + clip_sets = [] + for v in video_sets: + clip_sets.extend(split_video(v,int(splitlength),suffix = "demo",outputloc = video_cut_path)) + video_sets = clip_sets ## replace video_sets with clipped versions. if test: for video_file in [video_sets[0]]: From 25d55757921b4aa01c914d252b74fbb25705793e Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Mon, 27 Sep 2021 15:26:09 +0000 Subject: [PATCH 24/24] changed dlcsnapshot flag to snapshot flag for overwrite. changed split to not overwrite if videos_cut exists. --- demo/predict_dgp_demo.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/demo/predict_dgp_demo.py b/demo/predict_dgp_demo.py index 131b13b..0f9a039 100755 --- a/demo/predict_dgp_demo.py +++ b/demo/predict_dgp_demo.py @@ -25,6 +25,7 @@ from deepgraphpose.models.fitdgp import fit_dlc, fit_dgp, fit_dgp_labeledonly from deepgraphpose.models.fitdgp_util import get_snapshot_path from deepgraphpose.models.eval import plot_dgp +from deepgraphpose.contrib.segment_videos import split_video from run_dgp_demo import update_config_files_general if __name__ == '__main__': @@ -39,10 +40,10 @@ ) parser.add_argument( - "--dlcsnapshot", + "--snapshot", type=str, default=None, - help="use the DLC snapshot to initialize DGP", + help="use snapshot for prediction. If not given, assumes `snapshot-step2-final--0`", ) parser.add_argument("--shuffle", type=int, default=1, help="Project shuffle") @@ -61,7 +62,13 @@ dlcpath = input_params.dlcpath shuffle = input_params.shuffle - dlcsnapshot = input_params.dlcsnapshot + if input_params.snapshot is not None: + snapshot = input_params.snapshot + else: + ## Specifying snapshot as the end product of training. + step = 2 + snapshot = 'snapshot-step{}-final--0'.format(step) + batch_size = input_params.batch_size test = input_params.test splitflag,splitlength = input_params.split,input_params.splitlength @@ -71,9 +78,6 @@ dlcpath = update_config_files_general(dlcpath,shuffle) update_configs = True - ## Specifying snapshot manually at the moment assuming training. - step = 2 - snapshot = 'snapshot-step{}-final--0'.format(step) try: @@ -113,15 +117,17 @@ if not os.path.exists(video_pred_path): os.makedirs(video_pred_path) - print('video_sets', video_sets, flush=True) if splitflag: video_cut_path = str(Path(dlcpath) / 'videos_cut') if not os.path.exists(video_cut_path): os.makedirs(video_cut_path) - clip_sets = [] - for v in video_sets: - clip_sets.extend(split_video(v,int(splitlength),suffix = "demo",outputloc = video_cut_path)) + clip_sets = [] + for v in video_sets: + clip_sets.extend(split_video(v,int(splitlength),suffix = "demo",outputloc = video_cut_path)) + else: + clip_sets = [os.path.join(video_cut_path,v) for v in os.listdir(video_cut_path)] video_sets = clip_sets ## replace video_sets with clipped versions. + print('video_sets', video_sets, flush=True) if test: for video_file in [video_sets[0]]: