Added ability to provide git options to the GitPython clone/update me…

…thod to support things like shallow clone (e.g., --depth=1) (#130)

Fixes: #103
Fixes: #11

Release version 1.2.6

CHANGELOG.md

@@ -2,6 +2,15 @@
 
 <!--next-version-placeholder-->
 
+## [1.2.6] - 02/07/2024
+### Added
+- Added ability to provide git options to the GitPython clone/update method to support things like shallow clone (e.g., --depth=1)
+### Changed
+### Deprecated
+### Removed
+### Fixed
+### Security
+
 ## [1.2.5] - 02/07/2024
 ### Added
 - Added ability to clone a user's personal projects

README.rst

@@ -89,7 +89,7 @@ Usage
 .. code-block:: bash
 
     usage: gitlabber [-h] [-t token] [-T] [-u url] [--verbose] [-p] [--print-format {json,yaml,tree}] [-n {name,path}] [-m {ssh,http}]
-                    [-a {include,exclude,only}] [-i csv] [-x csv] [-r] [-F] [-d] [-s] [-g term] [-U] [--version]
+                    [-a {include,exclude,only}] [-i csv] [-x csv] [-r] [-F] [-d] [-s] [-g term] [-U] [-o options] [--version]
                     [dest]
 
     Gitlabber - clones or pulls entire groups/projects tree from gitlab
@@ -124,6 +124,8 @@ Usage
     -g term, --group-search term
                             only include groups matching the search term, filtering done at the API level (useful for large projects, see: https://docs.gitlab.com/ee/api/groups.html#search-for-group works with partial names of path or name)
     -U, --user-projects   fetch only user personal projects (skips the group tree altogether, group related parameters are ignored). Clones personal projects to '{gitlab-username}-personal-projects'
+    -o options, --git-options options
+                            provide additional options as csv for the git command (e.g., --depth=1). See: clone/multi_options https://gitpython.readthedocs.io/en/stable/reference.html#
     --version             print the version
 
     examples:
@@ -146,6 +148,12 @@ Usage
         clone projects that start with a case insensitive 'w' using a regular expression:
         gitlabber -i '/{[w].*}' .
 
+        clone a user's personal projects to username-personal-projects
+        gitlabber -U .
+
+        perform a shallow clone of the git repositories
+        gitlabber -o "\-\-depth=1," .
+
 
 
 Debugging

gitlabber/__init__.py

@@ -1 +1 @@
-__version__ = '1.2.5.1'
+__version__ = '1.2.6'

gitlabber/cli.py

@@ -43,7 +43,8 @@ def main():
 
     tree = GitlabTree(args.url, args.token, args.method, args.naming, args.archived.api_value, includes,
                       excludes, args.file, args.concurrency, args.recursive, args.verbose,
-                      args.include_shared, args.use_fetch, args.hide_token, args.user_projects, group_search=args.group_search)
+                      args.include_shared, args.use_fetch, args.hide_token, args.user_projects, 
+                      group_search=args.group_search, git_options=args.git_options)
     tree.load_tree()
 
     if tree.is_empty():
@@ -91,6 +92,12 @@ def parse_args(argv=None):
 
     clone projects that start with a case insensitive 'w' using a regular expression:
     gitlabber -i '/{[w].*}' .
+    
+    clone a user's personal projects to username-personal-projects
+    gitlabber -U .
+    
+    perform a shallow clone of the git repositories
+    gitlabber -o "\-\-depth=1," .
     '''
 
     parser = ArgumentParser(
@@ -216,6 +223,12 @@ def parse_args(argv=None):
         action='store_true',
         default=False,
         help='fetch only user personal projects (skips the group tree altogether, group related parameters are ignored). Clones personal projects to \'{gitlab-username}-personal-projects\'')
+    parser.add_argument(
+        '-o',
+        '--git-options',
+        nargs=1,
+        metavar=('options'),
+        help='provide additional options as csv for the git command (e.g., --depth=1). See: clone/multi_options https://gitpython.readthedocs.io/en/stable/reference.html#')
     parser.add_argument(
         '--version',
         action='store_true',

gitlabber/git.py

@@ -12,14 +12,15 @@
 
 
 class GitAction:
-    def __init__(self, node, path, recursive=False, use_fetch=False, hide_token=False):
+    def __init__(self, node, path, recursive=False, use_fetch=False, hide_token=False, git_options=None):
         self.node = node
         self.path = path
         self.recursive = recursive
         self.use_fetch = use_fetch
         self.hide_token = hide_token
+        self.git_options = git_options
 
-def sync_tree(root, dest, concurrency=1, disable_progress=False, recursive=False, use_fetch=False, hide_token=False):
+def sync_tree(root, dest, concurrency=1, disable_progress=False, recursive=False, use_fetch=False, hide_token=False, git_options=None):
     if not disable_progress:
         progress.init_progress(len(root.leaves))
     actions = get_git_actions(root, dest, recursive, use_fetch, hide_token)
@@ -71,7 +72,7 @@ def clone_or_pull_project(action):
             log.fatal("User interrupted")
             sys.exit(0)
         except Exception as e:
-            log.debug("Error pulling project %s", action.path, exc_info=True)
+            log.error("Error pulling project %s", action.path, exc_info=True)
     else:
         '''
         Clone new project
@@ -86,6 +87,8 @@ def clone_or_pull_project(action):
             multi_options.append('--recursive')
         if(action.use_fetch):
             multi_options.append('--mirror')
+        if(action.git_options):
+            multi_options += action.git_options.split(',')
         try:
             git.Repo.clone_from(action.node.url, action.path, multi_options=multi_options)
 

gitlabber/gitlab_tree.py

@@ -18,7 +18,7 @@
 
 class GitlabTree:
     def __init__(self, url, token, method, naming=None, archived=None, includes=[], excludes=[], in_file=None, concurrency=1, recursive=False, disable_progress=False,
-                include_shared=True, use_fetch=False, hide_token=False, user_projects=False, group_search=None):
+                include_shared=True, use_fetch=False, hide_token=False, user_projects=False, group_search=None, git_options=None):
         self.includes = includes
         self.excludes = excludes
         self.url = url
@@ -39,6 +39,7 @@ def __init__(self, url, token, method, naming=None, archived=None, includes=[],
         self.hide_token = hide_token
         self.user_projects = user_projects
         self.group_search = group_search
+        self.git_options = git_options
 
     @staticmethod
     def get_ca_path():

tests/gitlab_test_utils.py

@@ -2,6 +2,7 @@
 import json
 from unittest import mock
 from gitlabber import gitlab_tree
+from gitlabber.method import CloneMethod
 
 URL = "http://gitlab.my.com/"
 TOKEN = "MOCK_TOKEN"
@@ -12,6 +13,7 @@
 SUBGROUP_NAME = "subgroup"
 
 PROJECT_URL = "http://gitlab.my.com/group/subgroup/project/project.git"
+PROJECT_URL_WITH_TOKEN = "http://gitlab-token:[email protected]/group/subgroup/project/project.git"
 PROJECT_NAME = "project"
 
 YAML_TEST_INPUT_FILE = "tests/test-input.yaml"
@@ -21,7 +23,7 @@
 
 
 class MockNode:
-    def __init__(self, type, id, name, url, subgroups=mock.MagicMock(), projects=mock.MagicMock(), parent_id=None, archived=0, shared=False, group_search=None):
+    def __init__(self, type, id, name, url, subgroups=mock.MagicMock(), projects=mock.MagicMock(), parent_id=None, archived=0, shared=False, group_search=None, git_options=None):
         self.type = type
         self.id = id
         self.name = name
@@ -117,10 +119,10 @@ def validate_tree(root):
     validate_subgroup(root.children[0].children[0])
     validate_project(root.children[0].children[0].children[0])
 
-def create_test_gitlab(monkeypatch, includes=None, excludes=None, in_file=None):
+def create_test_gitlab(monkeypatch, includes=None, excludes=None, in_file=None, hide_token=True, method=CloneMethod.SSH):
     gl = gitlab_tree.GitlabTree(
-        URL, TOKEN, "ssh", "name", includes=includes, excludes=excludes, in_file=in_file)
-    projects = Listable(MockNode("project", 2, PROJECT_NAME, PROJECT_URL))
+        URL, TOKEN, "ssh", "name", includes=includes, excludes=excludes, in_file=in_file, hide_token=hide_token)
+    projects = Listable(MockNode("project", 2, PROJECT_NAME, PROJECT_URL if hide_token else PROJECT_URL_WITH_TOKEN))
     groups = Listable(
             MockNode("group", 2, GROUP_NAME, GROUP_URL, subgroups=Listable(
             MockNode("subgroup", 3, SUBGROUP_NAME, SUBGROUP_URL, projects=projects)

tests/test_cli.py

@@ -36,7 +36,7 @@ def test_args_version():
 def test_args_logging(mock_tree, mock_log, mock_os, mock_sys, mock_logging):
     args_mock = mock.Mock()
     args_mock.return_value = Node(
-        type="test", name="test", version=None, verbose=True, include="", exclude="", url="test_url", token="test_token", method=CloneMethod.SSH, naming=FolderNaming.PATH, archived=ArchivedResults.INCLUDE, file=None, concurrency=1, recursive=False, disble_progress=True, print=None, dest=".", include_shared=True, use_fetch=None, hide_token=None, user_projects=None, group_search=None)
+        type="test", name="test", version=None, verbose=True, include="", exclude="", url="test_url", token="test_token", method=CloneMethod.SSH, naming=FolderNaming.PATH, archived=ArchivedResults.INCLUDE, file=None, concurrency=1, recursive=False, disble_progress=True, print=None, dest=".", include_shared=True, use_fetch=None, hide_token=None, user_projects=None, group_search=None, git_options=None)
     cli.parse_args = args_mock
 
     mock_streamhandler = mock.Mock()
@@ -57,7 +57,7 @@ def test_args_include(mock_tree):
     exc_groups = "/exc**,/exc**"
     args_mock = mock.Mock()
     args_mock.return_value = Node(
-        type="test", name="test", version=None, debug=None, include=inc_groups, exclude=exc_groups, url="test_url", token="test_token", method=CloneMethod.SSH, naming=FolderNaming.NAME, archived=ArchivedResults.INCLUDE, file=None, concurrency=1, recursive=False, disble_progress=True, print=None, dest=".", use_fetch=None, hide_token=None, user_projects=None, group_search=None)
+        type="test", name="test", version=None, debug=None, include=inc_groups, exclude=exc_groups, url="test_url", token="test_token", method=CloneMethod.SSH, naming=FolderNaming.NAME, archived=ArchivedResults.INCLUDE, file=None, concurrency=1, recursive=False, disble_progress=True, print=None, dest=".", use_fetch=None, hide_token=None, user_projects=None, group_search=None, git_options=None)
     cli.parse_args = args_mock
 
     split_mock = mock.Mock()
@@ -73,7 +73,7 @@ def test_args_include(mock_tree):
 def test_args_include(mock_tree):
     args_mock = mock.Mock()
     args_mock.return_value = Node(
-        type="test", name="test", version=None, verbose=None, include="", exclude="", url="test_url", token="test_token", method=CloneMethod.SSH, naming=FolderNaming.NAME, archived=ArchivedResults.INCLUDE, file=None, concurrency=1, recursive=False, disble_progress=True, print=True, dest=".", print_format=PrintFormat.YAML, include_shared=True, use_fetch=None, hide_token=None, user_projects=None, group_search=None)
+        type="test", name="test", version=None, verbose=None, include="", exclude="", url="test_url", token="test_token", method=CloneMethod.SSH, naming=FolderNaming.NAME, archived=ArchivedResults.INCLUDE, file=None, concurrency=1, recursive=False, disble_progress=True, print=True, dest=".", print_format=PrintFormat.YAML, include_shared=True, use_fetch=None, hide_token=None, user_projects=None, group_search=None, git_options=None)
     cli.parse_args = args_mock
 
     print_tree_mock = mock.Mock()
@@ -116,7 +116,7 @@ def test_missing_url(mock_tree):
 def test_empty_tree(mock_tree):
     args_mock = mock.Mock()
     args_mock.return_value = Node(
-        type="test", name="test", version=None, verbose=None, include="", exclude="", url="test_url", token="test_token", method=CloneMethod.SSH, naming=FolderNaming.NAME, archived=ArchivedResults.INCLUDE, file=None, concurrency=1, recursive=False, disble_progress=True, print=True, dest=".", include_shared=True, use_fetch=None, hide_token=None, user_projects=None, group_search=None)
+        type="test", name="test", version=None, verbose=None, include="", exclude="", url="test_url", token="test_token", method=CloneMethod.SSH, naming=FolderNaming.NAME, archived=ArchivedResults.INCLUDE, file=None, concurrency=1, recursive=False, disble_progress=True, print=True, dest=".", include_shared=True, use_fetch=None, hide_token=None, user_projects=None, group_search=None, git_options=None)
     cli.parse_args = args_mock
 
     with pytest.raises(SystemExit):
@@ -127,7 +127,7 @@ def test_empty_tree(mock_tree):
 def test_missing_dest(mock_tree, capsys):
     args_mock = mock.Mock()
     args_mock.return_value = Node(
-        type="test", name="test", version=None, verbose=None, include="", exclude="", url="test_url", token="test_token", method=CloneMethod.SSH, naming=FolderNaming.NAME, archived=ArchivedResults.INCLUDE, file=None, concurrency=1, recursive=False, disble_progress=True, print=False, dest=None, group_search=None)
+        type="test", name="test", version=None, verbose=None, include="", exclude="", url="test_url", token="test_token", method=CloneMethod.SSH, naming=FolderNaming.NAME, archived=ArchivedResults.INCLUDE, file=None, concurrency=1, recursive=False, disble_progress=True, print=False, dest=None, group_search=None, git_options=None)
     cli.parse_args = args_mock
     mock_tree.return_value.is_empty = mock.Mock(return_value=False)
 

tests/test_git.py

@@ -115,6 +115,20 @@ def test_pull_repo_exception(mock_git):
 
     mock_git.Repo.assert_called_once_with("dummy_dir")
     repo_instance.remotes.origin.pull.assert_called_once()
+
+@mock.patch('gitlabber.git.git')
+def test_clone_repo_exception(mock_git):
+    mock_repo = mock.Mock()
+    mock_git.Repo = mock_repo
+    git.is_git_repo = mock.MagicMock(return_value=False)
+
+    repo_instance = mock_git.Repo.return_value
+    repo_instance.clone_from.side_effect=Exception('clone test exception')
+
+    git.clone_or_pull_project(
+        GitAction(Node(type="project", name="dummy_url", url="dummy_url"), "dummy_dir"))
+    mock_git.Repo.clone_from.assert_called_once_with('dummy_url', 'dummy_dir', multi_options=[])
+    mock_git.Repo.clone_from.assert_called_once()
 
 @mock.patch('gitlabber.git.git')
 def test_pull_repo_interrupt(mock_git):
@@ -144,3 +158,27 @@ def test_clone_repo_interrupt(mock_git):
             Node(type="project", name="dummy_url", url="dummy_url"), "dummy_dir"))
 
     mock_git.Repo.clone_from.assert_called_once_with("dummy_url", "dummy_dir", multi_options=[])
+
+
+@mock.patch('gitlabber.git.git')
+def test_clone_repo_options_many_options(mock_git):
+    mock_repo = mock.Mock()
+    mock_git.Repo = mock_repo
+    git.is_git_repo = mock.MagicMock(return_value=False)
+
+    git.clone_or_pull_project(
+        GitAction(Node(type="project", name="dummy_url", url="dummy_url"), "dummy_dir", git_options="--opt1=1,--opt2=2"))
+
+    mock_git.Repo.clone_from.assert_called_once_with("dummy_url", "dummy_dir", multi_options=['--opt1=1','--opt2=2'])
+
+
+@mock.patch('gitlabber.git.git')
+def test_clone_repo_options_with_recursive(mock_git):
+    mock_repo = mock.Mock()
+    mock_git.Repo = mock_repo
+    git.is_git_repo = mock.MagicMock(return_value=False)
+
+    git.clone_or_pull_project(
+        GitAction(Node(type="project", name="dummy_url", url="dummy_url"), "dummy_dir", recursive=True, git_options="--opt1=1,--opt2=2"))
+
+    mock_git.Repo.clone_from.assert_called_once_with("dummy_url", "dummy_dir", multi_options=['--recursive','--opt1=1','--opt2=2'])

tests/test_gitlab_tree.py

@@ -1,4 +1,5 @@
 
+from gitlabber.method import CloneMethod
 import tests.gitlab_test_utils as gitlab_util
 import tests.io_test_util as output_util
 from gitlabber.archive import ArchivedResults
@@ -164,4 +165,11 @@ def test_shared_excluded(monkeypatch):
     assert len(gl.root.children) == 1
     assert len(gl.root.children[0].children) == 1
 
-    assert "project" in gl.root.children[0].children[0].name
+    assert "project" in gl.root.children[0].children[0].name
+
+
+def test_hide_token_from_project_url(monkeypatch):
+    gl = gitlab_util.create_test_gitlab(monkeypatch, hide_token=True, method=CloneMethod.HTTP)
+    gl.load_tree()
+    gl.print_tree()
+    assert 'gitlab-token:xxx@' not in gl.root.children[0].children[0].children[0].url