Create folders return split by datatype (#382)

* Output folders split by datatype, update test.

* Update docs.

* Fix tests, if datatype passed without ses names then coerce to proper input and display message.

* fix 'sub_path' -> 'ses_path'!!

Co-authored-by: Niko Sirmpilatze <[email protected]>

* Rename 'folder_path_list' to 'folder_path_dict' in getting_started.md

* 'created_folder_list' -> 'created_folder_dict' in tests.

* Add 'Returns' section to 'create_folders()', fix incorrect type hints.

* Remove unused tests.

* Add test to cover split sub / ses behaviour.

* Search and replace 'created_folders_list' -> 'created_folders_dict'.

* Fix rebase conflicts still in the docstring.

---------

Co-authored-by: Niko Sirmpilatze <[email protected]>

datashuttle/datashuttle.py

@@ -149,7 +149,7 @@ def create_folders(
         datatype: Union[str, List[str]] = "",
         bypass_validation: bool = False,
         log: bool = True,
-    ) -> List[Path]:
+    ) -> Dict[str, List[Path]]:
         """
         Create a subject / session folder tree in the project
         folder. The passed subject / session names are
@@ -191,6 +191,16 @@ def create_folders(
         log : bool
             If `True`, details of folder creation will be logged.
 
+        Returns
+        -------
+        created_paths :
+            A dictionary of the full filepaths made during folder creation,
+            where the keys are the type of folder made and the values are a
+            list of created folder paths (Path objects). If datatype were
+            created, the dict keys will separate created folders by datatype
+            name. Similarly, if only subject or session level folders were
+            created, these are separated by "sub" and "ses" keys.
+
         Notes
         -----
 
@@ -223,6 +233,13 @@ def create_folders(
 
         self._check_top_level_folder(top_level_folder)
 
+        if ses_names is None and datatype != "":
+            datatype = ""
+            utils.log_and_message(
+                "`datatype` passed without `ses_names`, no datatype "
+                "folders will be created."
+            )
+
         utils.log("\nFormatting Names...")
         ds_logger.log_names(["sub_names", "ses_names"], [sub_names, ses_names])
 

datashuttle/utils/folders.py

@@ -36,7 +36,7 @@ def create_folder_trees(
     ses_names: Union[str, list],
     datatype: Union[List[str], str],
     log: bool = True,
-) -> List[Path]:
+) -> Dict[str, List[Path]]:
     """
     Entry method to make a full folder tree. It will
     iterate through all passed subjects, then sessions, then
@@ -64,7 +64,12 @@ def create_folder_trees(
         if is_invalid:
             utils.log_and_raise_error(message, NeuroBlueprintError)
 
-    all_paths = []
+        all_paths: Dict = {}
+    else:
+        all_paths = {
+            "sub": [],
+            "ses": [],
+        }
 
     for sub in sub_names:
         sub_path = cfg.build_project_path(
@@ -76,7 +81,7 @@ def create_folder_trees(
         create_folders(sub_path, log)
 
         if not any(ses_names):
-            all_paths.append(sub_path)
+            all_paths["sub"].append(sub_path)
             continue
 
         for ses in ses_names:
@@ -89,12 +94,16 @@ def create_folder_trees(
             create_folders(ses_path, log)
 
             if datatype_passed:
-                datatype_paths = make_datatype_folders(
-                    cfg, datatype, ses_path, "ses", log=log
+                make_datatype_folders(
+                    cfg,
+                    datatype,
+                    ses_path,
+                    "ses",
+                    save_paths=all_paths,
+                    log=log,
                 )
-                all_paths.extend(datatype_paths)
             else:
-                all_paths.append(ses_path)
+                all_paths["ses"].append(ses_path)
 
     return all_paths
 
@@ -104,15 +113,18 @@ def make_datatype_folders(
     datatype: Union[list, str],
     sub_or_ses_level_path: Path,
     level: str,
+    save_paths: Dict,
     log: bool = True,
-) -> List[Path]:
+):
     """
     Make datatype folder (e.g. behav) at the sub or ses
     level. Checks folder_class.Folders attributes,
     whether the datatype is used and at the current level.
 
     Parameters
     ----------
+    cfg : ConfigsClass
+
     datatype : datatype (e.g. "behav", "all") to use. Use
         empty string ("") for none.
 
@@ -123,21 +135,28 @@ def make_datatype_folders(
     level : The folder level that the
         folder will be made at, "sub" or "ses"
 
+    save_paths : A dictionary, which will be filled
+        with created paths split by datatype name.
+
     log : whether to log on or not (if True, logging must
         already be initialised).
     """
     datatype_items = cfg.get_datatype_as_dict_items(datatype)
 
-    all_datatype_paths = []
-
     for datatype_key, datatype_folder in datatype_items:  # type: ignore
         if datatype_folder.level == level:
-            datatype_path = sub_or_ses_level_path / datatype_folder.name
+
+            datatype_name = datatype_folder.name
+
+            datatype_path = sub_or_ses_level_path / datatype_name
 
             create_folders(datatype_path, log)
-            all_datatype_paths.append(datatype_path)
 
-    return all_datatype_paths
+            # Use the custom datatype names for the output.
+            if datatype_name in save_paths:
+                save_paths[datatype_name].append(datatype_path)
+            else:
+                save_paths[datatype_name] = [datatype_path]
 
 
 # Create Folders Helpers --------------------------------------------------------

docs/source/pages/how_tos/create-folders.md

@@ -166,8 +166,8 @@ created_folders = project.create_folders(
 )
 ```
 
-The method outputs `created_folders`, which contains a list of all
-`Path`s to all created datatype folders. See the below section for
+The method outputs `created_folders`, which contains the
+`Path`s to created datatype folders. See the below section for
 details on the `@DATE@` and other convenience tags.
 
 By default, an error will be raised if the folder names break

docs/source/pages/tutorials/getting_started.md

@@ -442,15 +442,15 @@ Finally, hover the mouse over the `Directory Tree` and press `CTRL+R` to refresh
 These can be used in acquisition scripts to save data to these folders:
 
 ```python
-folder_path_list = project.create_folders(
+folder_path_dict = project.create_folders(
     top_level_folder="rawdata",
     sub_names=["sub-001"],
     ses_names=["ses-001_@DATE@"],
     datatype=["behav", "ephys"]
 
 )
 
-print([path_ for path_ in folder_path_list if path_.name == "behav"])
+print([path_ for path_ in folder_path_dict["behav"]])
 # ["C:\Users\Joe\data\local\my_first_project\sub-001\ses-001_16052024\behav"]
 ```
 

tests/test_utils.py

@@ -220,7 +220,7 @@ def get_all_folders_used(value=True):
 
 
 def check_folder_tree_is_correct(
-    base_folder, subs, sessions, folder_used, created_folder_list=None
+    base_folder, subs, sessions, folder_used, created_folder_dict=None
 ):
     """
     Automated test that folders are made based
@@ -235,6 +235,9 @@ def check_folder_tree_is_correct(
     rely on project settings itself,
     as this doesn't explicitly test this.
     """
+    if created_folder_dict is None:
+        created_folder_dict = {}
+
     for sub in subs:
         path_to_sub_folder = join(base_folder, sub)
         check_and_cd_folder(path_to_sub_folder)
@@ -262,12 +265,22 @@ def check_folder_tree_is_correct(
 
                 if folder_used[key]:
                     check_and_cd_folder(datatype_path)
-                    if created_folder_list:
-                        assert Path(datatype_path) in created_folder_list
+
+                    # Check the created path is found only in the expected
+                    # dict entry.
+                    for (
+                        datatype_name,
+                        all_datatype_paths,
+                    ) in created_folder_dict.items():
+                        if datatype_name == key:
+                            assert Path(datatype_path) in all_datatype_paths
+                        else:
+                            assert (
+                                Path(datatype_path) not in all_datatype_paths
+                            )
                 else:
                     assert not os.path.isdir(datatype_path)
-                    if created_folder_list:
-                        assert Path(datatype_path) not in created_folder_list
+                    assert key not in created_folder_dict
 
 
 def check_and_cd_folder(path_):
@@ -344,7 +357,7 @@ def make_and_check_local_project_folders(
 
 
 def make_local_folders_with_files_in(
-    project, top_level_folder, subs, sessions=None, datatype="all"
+    project, top_level_folder, subs, sessions=None, datatype=""
 ):
     project.create_folders(top_level_folder, subs, sessions, datatype)
     for root, dirs, _ in os.walk(project.cfg["local_path"]):

tests/tests_integration/test_create_folders.py

@@ -95,7 +95,7 @@ def test_every_datatype_passed(self, project, behav, ephys, funcimg, anat):
         subs = ["sub-001", "sub-002"]
         sessions = ["ses-001", "ses-002"]
 
-        created_folder_list = project.create_folders(
+        created_folder_dict = project.create_folders(
             "rawdata", subs, sessions, datatypes_to_make
         )
 
@@ -110,7 +110,7 @@ def test_every_datatype_passed(self, project, behav, ephys, funcimg, anat):
                 "funcimg": funcimg,
                 "anat": anat,
             },
-            created_folder_list=created_folder_list,
+            created_folder_dict=created_folder_dict,
         )
 
     def test_custom_folder_names(self, project, monkeypatch):
@@ -252,6 +252,33 @@ def test_datetime_flag_in_session(self, project):
         assert all([re.search(datetime_regexp, name) for name in ses_names])
         assert all([tags("time") not in name for name in ses_names])
 
+    def test_created_paths_dict_sub_or_ses_only(self, project):
+        """
+        Test that the `created_folders` dictionary returned by
+        `create_folders` correctly splits paths when only
+        subject or session is passed. The `datatype` case is
+        tested in `test_utils.check_folder_tree_is_correct()`.
+        """
+        created_path_sub = project.create_folders("rawdata", "sub-001")
+
+        assert len(created_path_sub) == 2
+        assert created_path_sub["ses"] == []
+        assert (
+            created_path_sub["sub"][0]
+            == project.get_local_path() / "rawdata" / "sub-001"
+        )
+
+        created_path_ses = project.create_folders(
+            "rawdata", "sub-002", "ses-001"
+        )
+
+        assert len(created_path_ses) == 2
+        assert created_path_ses["sub"] == []
+        assert (
+            created_path_ses["ses"][0]
+            == project.get_local_path() / "rawdata" / "sub-002" / "ses-001"
+        )
+
     # ----------------------------------------------------------------------------------------------------------
     # Test Make Folders in Different Top Level Folders
     # ----------------------------------------------------------------------------------------------------------