reds data preparation

basicsr/utils/create_lmdb.py

@@ -28,15 +28,15 @@ def prepare_keys(folder_path, suffix='png'):
     return img_path_list, keys
 
 def create_lmdb_for_reds():
-    folder_path = './datasets/REDS/val/sharp_300'
-    lmdb_path = './datasets/REDS/val/sharp_300.lmdb'
-    img_path_list, keys = prepare_keys(folder_path, 'png')
-    make_lmdb_from_imgs(folder_path, lmdb_path, img_path_list, keys)
+    # folder_path = './datasets/REDS/val/sharp_300'
+    # lmdb_path = './datasets/REDS/val/sharp_300.lmdb'
+    # img_path_list, keys = prepare_keys(folder_path, 'png')
+    # make_lmdb_from_imgs(folder_path, lmdb_path, img_path_list, keys)
     #
-    folder_path = './datasets/REDS/val/blur_300'
-    lmdb_path = './datasets/REDS/val/blur_300.lmdb'
-    img_path_list, keys = prepare_keys(folder_path, 'jpg')
-    make_lmdb_from_imgs(folder_path, lmdb_path, img_path_list, keys)
+    # folder_path = './datasets/REDS/val/blur_300'
+    # lmdb_path = './datasets/REDS/val/blur_300.lmdb'
+    # img_path_list, keys = prepare_keys(folder_path, 'jpg')
+    # make_lmdb_from_imgs(folder_path, lmdb_path, img_path_list, keys)
 
     folder_path = './datasets/REDS/train/train_sharp'
     lmdb_path = './datasets/REDS/train/train_sharp.lmdb'
@@ -62,17 +62,17 @@ def create_lmdb_for_gopro():
     img_path_list, keys = prepare_keys(folder_path, 'png')
     make_lmdb_from_imgs(folder_path, lmdb_path, img_path_list, keys)
 
-    folder_path = './datasets/GoPro/test/target'
-    lmdb_path = './datasets/GoPro/test/target.lmdb'
+    # folder_path = './datasets/GoPro/test/target'
+    # lmdb_path = './datasets/GoPro/test/target.lmdb'
 
-    img_path_list, keys = prepare_keys(folder_path, 'png')
-    make_lmdb_from_imgs(folder_path, lmdb_path, img_path_list, keys)
+    # img_path_list, keys = prepare_keys(folder_path, 'png')
+    # make_lmdb_from_imgs(folder_path, lmdb_path, img_path_list, keys)
 
-    folder_path = './datasets/GoPro/test/input'
-    lmdb_path = './datasets/GoPro/test/input.lmdb'
+    # folder_path = './datasets/GoPro/test/input'
+    # lmdb_path = './datasets/GoPro/test/input.lmdb'
 
-    img_path_list, keys = prepare_keys(folder_path, 'png')
-    make_lmdb_from_imgs(folder_path, lmdb_path, img_path_list, keys)
+    # img_path_list, keys = prepare_keys(folder_path, 'png')
+    # make_lmdb_from_imgs(folder_path, lmdb_path, img_path_list, keys)
 
 def create_lmdb_for_rain13k():
     folder_path = './datasets/Rain13k/train/input'
@@ -101,6 +101,8 @@ def create_lmdb_for_SIDD():
     make_lmdb_from_imgs(folder_path, lmdb_path, img_path_list, keys)
 
     #for val
+    '''
+    
     folder_path = './datasets/SIDD/val/input_crops'
     lmdb_path = './datasets/SIDD/val/input_crops.lmdb'
     mat_path = './datasets/SIDD/ValidationNoisyBlocksSrgb.mat'
@@ -128,3 +130,4 @@ def create_lmdb_for_SIDD():
         cv2.imwrite(osp.join(folder_path, 'ValidationBlocksSrgb_{}.png'.format(i)), cv2.cvtColor(data[i,...], cv2.COLOR_RGB2BGR)) 
     img_path_list, keys = prepare_keys(folder_path, 'png')
     make_lmdb_from_imgs(folder_path, lmdb_path, img_path_list, keys)
+    '''

scripts/data_preparation/reds.py

@@ -0,0 +1,63 @@
+# ------------------------------------------------------------------------
+# Copyright (c) 2021 megvii-model. All Rights Reserved.
+# ------------------------------------------------------------------------
+# Modified from BasicSR (https://github.com/xinntao/BasicSR)
+# Copyright 2018-2020 BasicSR Authors
+# ------------------------------------------------------------------------
+'''
+for val set, extract the subset val-300
+
+'''
+import os
+import time
+from basicsr.utils.create_lmdb import create_lmdb_for_reds
+
+def make_val_300(folder, dst):
+    if not os.path.exists(dst):
+        os.mkdir(dst)
+    templates = '*9.*'
+    cp_command = 'cp {} {}'.format(os.path.join(folder, templates), dst)
+    os.system(cp_command)
+
+
+def flatten_folders(folder):
+    for vid in range(300):
+        vidfolder_path = '{:03}'.format(vid)
+
+        if not os.path.exists(os.path.join(folder, vidfolder_path)):
+            continue
+
+        print('working on .. {} .. {}'.format(folder, vid))
+        for fid in range(100):
+            src_filename = '{:08}'.format(fid)
+
+            suffixes = ['.jpg', '.png']
+            suffix = None
+
+            for suf in suffixes:
+                # print(os.path.join(folder, vidfolder_path, src_filename+suf))
+                if os.path.exists(os.path.join(folder, vidfolder_path, src_filename+suf)):
+                    suffix = suf
+                    break
+            assert suffix is not None
+
+
+            src_filepath = os.path.join(folder, vidfolder_path, src_filename+suffix)
+            dst_filepath = os.path.join(folder, '{}_{}{}'.format(vidfolder_path, src_filename, suffix))
+            os.system('mv {} {}'.format(src_filepath, dst_filepath))
+            time.sleep(0.001)
+        os.system('rm -r {}'.format(os.path.join(folder, vidfolder_path)))
+
+
+if __name__ == '__main__':
+    flatten_folders('./datasets/REDS/train/train_blur_jpeg')
+    flatten_folders('./datasets/REDS/train/train_sharp')
+
+    # flatten_folders('./datasets/REDS/val/val_blur_jpeg')
+    # flatten_folders('./datasets/REDS/val/val_sharp')
+    # make_val_300('./datasets/REDS/val/val_blur_jpeg', './datasets/REDS/val/blur_300')
+    # make_val_300('./datasets/REDS/val/val_sharp', './datasets/REDS/val/sharp_300')
+
+    create_lmdb_for_reds()
+
+