diff --git a/CHANGELOG.md b/CHANGELOG.md index ff593e3..fec632b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ +## [1.2.6] - 02/07/2024 +### Added +- Added ability to provide git options to the GitPython clone/update method to support things like shallow clone (e.g., --depth=1) +### Changed +### Deprecated +### Removed +### Fixed +### Security + ## [1.2.5] - 02/07/2024 ### Added - Added ability to clone a user's personal projects diff --git a/README.rst b/README.rst index 7fd86c6..1accd0e 100644 --- a/README.rst +++ b/README.rst @@ -89,7 +89,7 @@ Usage .. code-block:: bash usage: gitlabber [-h] [-t token] [-T] [-u url] [--verbose] [-p] [--print-format {json,yaml,tree}] [-n {name,path}] [-m {ssh,http}] - [-a {include,exclude,only}] [-i csv] [-x csv] [-r] [-F] [-d] [-s] [-g term] [-U] [--version] + [-a {include,exclude,only}] [-i csv] [-x csv] [-r] [-F] [-d] [-s] [-g term] [-U] [-o options] [--version] [dest] Gitlabber - clones or pulls entire groups/projects tree from gitlab @@ -124,6 +124,8 @@ Usage -g term, --group-search term only include groups matching the search term, filtering done at the API level (useful for large projects, see: https://docs.gitlab.com/ee/api/groups.html#search-for-group works with partial names of path or name) -U, --user-projects fetch only user personal projects (skips the group tree altogether, group related parameters are ignored). Clones personal projects to '{gitlab-username}-personal-projects' + -o options, --git-options options + provide additional options as csv for the git command (e.g., --depth=1). See: clone/multi_options https://gitpython.readthedocs.io/en/stable/reference.html# --version print the version examples: @@ -146,6 +148,12 @@ Usage clone projects that start with a case insensitive 'w' using a regular expression: gitlabber -i '/{[w].*}' . + clone a user's personal projects to username-personal-projects + gitlabber -U . + + perform a shallow clone of the git repositories + gitlabber -o "\-\-depth=1," . + Debugging diff --git a/gitlabber/__init__.py b/gitlabber/__init__.py index af59c3d..593fb6f 100644 --- a/gitlabber/__init__.py +++ b/gitlabber/__init__.py @@ -1 +1 @@ -__version__ = '1.2.5.1' +__version__ = '1.2.6' diff --git a/gitlabber/cli.py b/gitlabber/cli.py index 393edf5..f5e47d5 100644 --- a/gitlabber/cli.py +++ b/gitlabber/cli.py @@ -43,7 +43,8 @@ def main(): tree = GitlabTree(args.url, args.token, args.method, args.naming, args.archived.api_value, includes, excludes, args.file, args.concurrency, args.recursive, args.verbose, - args.include_shared, args.use_fetch, args.hide_token, args.user_projects, group_search=args.group_search) + args.include_shared, args.use_fetch, args.hide_token, args.user_projects, + group_search=args.group_search, git_options=args.git_options) tree.load_tree() if tree.is_empty(): @@ -91,6 +92,12 @@ def parse_args(argv=None): clone projects that start with a case insensitive 'w' using a regular expression: gitlabber -i '/{[w].*}' . + + clone a user's personal projects to username-personal-projects + gitlabber -U . + + perform a shallow clone of the git repositories + gitlabber -o "\-\-depth=1," . ''' parser = ArgumentParser( @@ -216,6 +223,12 @@ def parse_args(argv=None): action='store_true', default=False, help='fetch only user personal projects (skips the group tree altogether, group related parameters are ignored). Clones personal projects to \'{gitlab-username}-personal-projects\'') + parser.add_argument( + '-o', + '--git-options', + nargs=1, + metavar=('options'), + help='provide additional options as csv for the git command (e.g., --depth=1). See: clone/multi_options https://gitpython.readthedocs.io/en/stable/reference.html#') parser.add_argument( '--version', action='store_true', diff --git a/gitlabber/git.py b/gitlabber/git.py index edc4330..44c9c6c 100644 --- a/gitlabber/git.py +++ b/gitlabber/git.py @@ -12,14 +12,15 @@ class GitAction: - def __init__(self, node, path, recursive=False, use_fetch=False, hide_token=False): + def __init__(self, node, path, recursive=False, use_fetch=False, hide_token=False, git_options=None): self.node = node self.path = path self.recursive = recursive self.use_fetch = use_fetch self.hide_token = hide_token + self.git_options = git_options -def sync_tree(root, dest, concurrency=1, disable_progress=False, recursive=False, use_fetch=False, hide_token=False): +def sync_tree(root, dest, concurrency=1, disable_progress=False, recursive=False, use_fetch=False, hide_token=False, git_options=None): if not disable_progress: progress.init_progress(len(root.leaves)) actions = get_git_actions(root, dest, recursive, use_fetch, hide_token) @@ -71,7 +72,7 @@ def clone_or_pull_project(action): log.fatal("User interrupted") sys.exit(0) except Exception as e: - log.debug("Error pulling project %s", action.path, exc_info=True) + log.error("Error pulling project %s", action.path, exc_info=True) else: ''' Clone new project @@ -86,6 +87,8 @@ def clone_or_pull_project(action): multi_options.append('--recursive') if(action.use_fetch): multi_options.append('--mirror') + if(action.git_options): + multi_options += action.git_options.split(',') try: git.Repo.clone_from(action.node.url, action.path, multi_options=multi_options) diff --git a/gitlabber/gitlab_tree.py b/gitlabber/gitlab_tree.py index 356bfd6..e570371 100644 --- a/gitlabber/gitlab_tree.py +++ b/gitlabber/gitlab_tree.py @@ -18,7 +18,7 @@ class GitlabTree: def __init__(self, url, token, method, naming=None, archived=None, includes=[], excludes=[], in_file=None, concurrency=1, recursive=False, disable_progress=False, - include_shared=True, use_fetch=False, hide_token=False, user_projects=False, group_search=None): + include_shared=True, use_fetch=False, hide_token=False, user_projects=False, group_search=None, git_options=None): self.includes = includes self.excludes = excludes self.url = url @@ -39,6 +39,7 @@ def __init__(self, url, token, method, naming=None, archived=None, includes=[], self.hide_token = hide_token self.user_projects = user_projects self.group_search = group_search + self.git_options = git_options @staticmethod def get_ca_path(): diff --git a/tests/gitlab_test_utils.py b/tests/gitlab_test_utils.py index 6754792..142015e 100644 --- a/tests/gitlab_test_utils.py +++ b/tests/gitlab_test_utils.py @@ -2,6 +2,7 @@ import json from unittest import mock from gitlabber import gitlab_tree +from gitlabber.method import CloneMethod URL = "http://gitlab.my.com/" TOKEN = "MOCK_TOKEN" @@ -12,6 +13,7 @@ SUBGROUP_NAME = "subgroup" PROJECT_URL = "http://gitlab.my.com/group/subgroup/project/project.git" +PROJECT_URL_WITH_TOKEN = "http://gitlab-token:xxx@gitlab.my.com/group/subgroup/project/project.git" PROJECT_NAME = "project" YAML_TEST_INPUT_FILE = "tests/test-input.yaml" @@ -21,7 +23,7 @@ class MockNode: - def __init__(self, type, id, name, url, subgroups=mock.MagicMock(), projects=mock.MagicMock(), parent_id=None, archived=0, shared=False, group_search=None): + def __init__(self, type, id, name, url, subgroups=mock.MagicMock(), projects=mock.MagicMock(), parent_id=None, archived=0, shared=False, group_search=None, git_options=None): self.type = type self.id = id self.name = name @@ -117,10 +119,10 @@ def validate_tree(root): validate_subgroup(root.children[0].children[0]) validate_project(root.children[0].children[0].children[0]) -def create_test_gitlab(monkeypatch, includes=None, excludes=None, in_file=None): +def create_test_gitlab(monkeypatch, includes=None, excludes=None, in_file=None, hide_token=True, method=CloneMethod.SSH): gl = gitlab_tree.GitlabTree( - URL, TOKEN, "ssh", "name", includes=includes, excludes=excludes, in_file=in_file) - projects = Listable(MockNode("project", 2, PROJECT_NAME, PROJECT_URL)) + URL, TOKEN, "ssh", "name", includes=includes, excludes=excludes, in_file=in_file, hide_token=hide_token) + projects = Listable(MockNode("project", 2, PROJECT_NAME, PROJECT_URL if hide_token else PROJECT_URL_WITH_TOKEN)) groups = Listable( MockNode("group", 2, GROUP_NAME, GROUP_URL, subgroups=Listable( MockNode("subgroup", 3, SUBGROUP_NAME, SUBGROUP_URL, projects=projects) diff --git a/tests/test_cli.py b/tests/test_cli.py index 554e5b9..d32c2d7 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -36,7 +36,7 @@ def test_args_version(): def test_args_logging(mock_tree, mock_log, mock_os, mock_sys, mock_logging): args_mock = mock.Mock() args_mock.return_value = Node( - type="test", name="test", version=None, verbose=True, include="", exclude="", url="test_url", token="test_token", method=CloneMethod.SSH, naming=FolderNaming.PATH, archived=ArchivedResults.INCLUDE, file=None, concurrency=1, recursive=False, disble_progress=True, print=None, dest=".", include_shared=True, use_fetch=None, hide_token=None, user_projects=None, group_search=None) + type="test", name="test", version=None, verbose=True, include="", exclude="", url="test_url", token="test_token", method=CloneMethod.SSH, naming=FolderNaming.PATH, archived=ArchivedResults.INCLUDE, file=None, concurrency=1, recursive=False, disble_progress=True, print=None, dest=".", include_shared=True, use_fetch=None, hide_token=None, user_projects=None, group_search=None, git_options=None) cli.parse_args = args_mock mock_streamhandler = mock.Mock() @@ -57,7 +57,7 @@ def test_args_include(mock_tree): exc_groups = "/exc**,/exc**" args_mock = mock.Mock() args_mock.return_value = Node( - type="test", name="test", version=None, debug=None, include=inc_groups, exclude=exc_groups, url="test_url", token="test_token", method=CloneMethod.SSH, naming=FolderNaming.NAME, archived=ArchivedResults.INCLUDE, file=None, concurrency=1, recursive=False, disble_progress=True, print=None, dest=".", use_fetch=None, hide_token=None, user_projects=None, group_search=None) + type="test", name="test", version=None, debug=None, include=inc_groups, exclude=exc_groups, url="test_url", token="test_token", method=CloneMethod.SSH, naming=FolderNaming.NAME, archived=ArchivedResults.INCLUDE, file=None, concurrency=1, recursive=False, disble_progress=True, print=None, dest=".", use_fetch=None, hide_token=None, user_projects=None, group_search=None, git_options=None) cli.parse_args = args_mock split_mock = mock.Mock() @@ -73,7 +73,7 @@ def test_args_include(mock_tree): def test_args_include(mock_tree): args_mock = mock.Mock() args_mock.return_value = Node( - type="test", name="test", version=None, verbose=None, include="", exclude="", url="test_url", token="test_token", method=CloneMethod.SSH, naming=FolderNaming.NAME, archived=ArchivedResults.INCLUDE, file=None, concurrency=1, recursive=False, disble_progress=True, print=True, dest=".", print_format=PrintFormat.YAML, include_shared=True, use_fetch=None, hide_token=None, user_projects=None, group_search=None) + type="test", name="test", version=None, verbose=None, include="", exclude="", url="test_url", token="test_token", method=CloneMethod.SSH, naming=FolderNaming.NAME, archived=ArchivedResults.INCLUDE, file=None, concurrency=1, recursive=False, disble_progress=True, print=True, dest=".", print_format=PrintFormat.YAML, include_shared=True, use_fetch=None, hide_token=None, user_projects=None, group_search=None, git_options=None) cli.parse_args = args_mock print_tree_mock = mock.Mock() @@ -116,7 +116,7 @@ def test_missing_url(mock_tree): def test_empty_tree(mock_tree): args_mock = mock.Mock() args_mock.return_value = Node( - type="test", name="test", version=None, verbose=None, include="", exclude="", url="test_url", token="test_token", method=CloneMethod.SSH, naming=FolderNaming.NAME, archived=ArchivedResults.INCLUDE, file=None, concurrency=1, recursive=False, disble_progress=True, print=True, dest=".", include_shared=True, use_fetch=None, hide_token=None, user_projects=None, group_search=None) + type="test", name="test", version=None, verbose=None, include="", exclude="", url="test_url", token="test_token", method=CloneMethod.SSH, naming=FolderNaming.NAME, archived=ArchivedResults.INCLUDE, file=None, concurrency=1, recursive=False, disble_progress=True, print=True, dest=".", include_shared=True, use_fetch=None, hide_token=None, user_projects=None, group_search=None, git_options=None) cli.parse_args = args_mock with pytest.raises(SystemExit): @@ -127,7 +127,7 @@ def test_empty_tree(mock_tree): def test_missing_dest(mock_tree, capsys): args_mock = mock.Mock() args_mock.return_value = Node( - type="test", name="test", version=None, verbose=None, include="", exclude="", url="test_url", token="test_token", method=CloneMethod.SSH, naming=FolderNaming.NAME, archived=ArchivedResults.INCLUDE, file=None, concurrency=1, recursive=False, disble_progress=True, print=False, dest=None, group_search=None) + type="test", name="test", version=None, verbose=None, include="", exclude="", url="test_url", token="test_token", method=CloneMethod.SSH, naming=FolderNaming.NAME, archived=ArchivedResults.INCLUDE, file=None, concurrency=1, recursive=False, disble_progress=True, print=False, dest=None, group_search=None, git_options=None) cli.parse_args = args_mock mock_tree.return_value.is_empty = mock.Mock(return_value=False) diff --git a/tests/test_git.py b/tests/test_git.py index 41adce7..e97d66d 100644 --- a/tests/test_git.py +++ b/tests/test_git.py @@ -115,6 +115,20 @@ def test_pull_repo_exception(mock_git): mock_git.Repo.assert_called_once_with("dummy_dir") repo_instance.remotes.origin.pull.assert_called_once() + +@mock.patch('gitlabber.git.git') +def test_clone_repo_exception(mock_git): + mock_repo = mock.Mock() + mock_git.Repo = mock_repo + git.is_git_repo = mock.MagicMock(return_value=False) + + repo_instance = mock_git.Repo.return_value + repo_instance.clone_from.side_effect=Exception('clone test exception') + + git.clone_or_pull_project( + GitAction(Node(type="project", name="dummy_url", url="dummy_url"), "dummy_dir")) + mock_git.Repo.clone_from.assert_called_once_with('dummy_url', 'dummy_dir', multi_options=[]) + mock_git.Repo.clone_from.assert_called_once() @mock.patch('gitlabber.git.git') def test_pull_repo_interrupt(mock_git): @@ -144,3 +158,27 @@ def test_clone_repo_interrupt(mock_git): Node(type="project", name="dummy_url", url="dummy_url"), "dummy_dir")) mock_git.Repo.clone_from.assert_called_once_with("dummy_url", "dummy_dir", multi_options=[]) + + +@mock.patch('gitlabber.git.git') +def test_clone_repo_options_many_options(mock_git): + mock_repo = mock.Mock() + mock_git.Repo = mock_repo + git.is_git_repo = mock.MagicMock(return_value=False) + + git.clone_or_pull_project( + GitAction(Node(type="project", name="dummy_url", url="dummy_url"), "dummy_dir", git_options="--opt1=1,--opt2=2")) + + mock_git.Repo.clone_from.assert_called_once_with("dummy_url", "dummy_dir", multi_options=['--opt1=1','--opt2=2']) + + +@mock.patch('gitlabber.git.git') +def test_clone_repo_options_with_recursive(mock_git): + mock_repo = mock.Mock() + mock_git.Repo = mock_repo + git.is_git_repo = mock.MagicMock(return_value=False) + + git.clone_or_pull_project( + GitAction(Node(type="project", name="dummy_url", url="dummy_url"), "dummy_dir", recursive=True, git_options="--opt1=1,--opt2=2")) + + mock_git.Repo.clone_from.assert_called_once_with("dummy_url", "dummy_dir", multi_options=['--recursive','--opt1=1','--opt2=2']) \ No newline at end of file diff --git a/tests/test_gitlab_tree.py b/tests/test_gitlab_tree.py index 6162043..513123d 100644 --- a/tests/test_gitlab_tree.py +++ b/tests/test_gitlab_tree.py @@ -1,4 +1,5 @@ +from gitlabber.method import CloneMethod import tests.gitlab_test_utils as gitlab_util import tests.io_test_util as output_util from gitlabber.archive import ArchivedResults @@ -164,4 +165,11 @@ def test_shared_excluded(monkeypatch): assert len(gl.root.children) == 1 assert len(gl.root.children[0].children) == 1 - assert "project" in gl.root.children[0].children[0].name \ No newline at end of file + assert "project" in gl.root.children[0].children[0].name + + +def test_hide_token_from_project_url(monkeypatch): + gl = gitlab_util.create_test_gitlab(monkeypatch, hide_token=True, method=CloneMethod.HTTP) + gl.load_tree() + gl.print_tree() + assert 'gitlab-token:xxx@' not in gl.root.children[0].children[0].children[0].url