Feat: New method add_maven_config for JCasC config

This method allows for adding settings files for new projects via
JCasC, rather than the old method of using a Groovy script to manually
add the file to Jenkins.

This also removes the __del__ function. The function was not entirely
necessary, and was causing issues with unit testing. All work is done
in /tmp, so these files will be automatically cleaned by the system.

Issue: RELENG-3893
Change-Id: I03511b9c3e1c9f9c9f79054bd6f8991cd274e8f5
Signed-off-by: Eric Ball <[email protected]>

docs/commands/gerrit.rst

@@ -81,3 +81,23 @@ addinfojob
      username = lfid2
      password = password2
      signed_off_by = Your Name <[email protected]>
+
+addmavenconfig
+--------------
+.. program-output:: lftools gerrit addmavenconfig --help
+
+
+An example of the lftools.ini entry for a Gerrit server making use of a full
+configuration:
+
+.. code-block:: none
+
+     [gerrit.example.org]
+     username = lfid
+     password = password
+     signed_off_by = Your Name <[email protected]>
+     endpoint = https://gerrit.example.org/
+     default_servers = releases,snapshots,staging,site
+     nexus3 = nexus3.example.org
+     nexus3_ports = 10001,10002,10003,10004
+     additional_credentials = {"docker.io": "dockerhub-cred", "nexus-iq": "nexus-iq-cred"}

lftools/cli/gerrit.py

@@ -184,6 +184,43 @@ def list_project_inherits_from(ctx, gerrit_fqdn, gerrit_project):
     log.info(data)
 
 
+@click.command(name="addmavenconfig")
+@click.argument("gerrit_fqdn")
+@click.argument("gerrit_project")
+@click.argument("jjbrepo")
+@click.option("--issue_id", type=str, required=False, help="For projects that enforce an issue id for changesets")
+@click.option("--nexus3", type=str, required=False, help="Specify a Nexus 3 server, e.g. nexus3.example.org")
+@click.option(
+    "--nexus3_ports",
+    type=str,
+    required=False,
+    help="Comma-separated list of ports supported by the Nexus 3 server specified",
+)
+@click.pass_context
+def addmavenconfig(ctx, gerrit_fqdn, gerrit_project, jjbrepo, issue_id, nexus3, nexus3_ports):
+    """Add maven config file for JCasC.
+
+    \b
+    The following options can be set in the gerrit server's entry in lftools.ini:
+      * default_servers: Comma-separated list of servers using the <projectname>
+        credential. Default: releases,snapshots,staging,site
+      * additional_credentials: JSON-formatted string containing
+        servername:credentialname pairings. This should be on a single line,
+        without quotes surrounding the string.
+      * nexus3: The nexus3 server url for a given project.
+      * nexus3_ports: Comma-separated list of ports used by Nexus3.
+        Default: 10001,10002,10003,10004
+
+    \f
+    The 'b' escape character above disables auto-formatting, so that the help
+    text will follow the exact formatting used here. The 'f' escape is to keep
+    this from appearing in the --help text.
+    https://click.palletsprojects.com/en/latest/documentation/
+    """
+    git = git_gerrit(fqdn=gerrit_fqdn, project=jjbrepo)
+    git.add_maven_config(gerrit_fqdn, gerrit_project, issue_id, nexus3, nexus3_ports)
+
+
 gerrit_cli.add_command(addinfojob)
 gerrit_cli.add_command(addfile)
 gerrit_cli.add_command(addgitreview)
@@ -193,3 +230,4 @@ def list_project_inherits_from(ctx, gerrit_fqdn, gerrit_project):
 gerrit_cli.add_command(create_saml_group)
 gerrit_cli.add_command(list_project_permissions)
 gerrit_cli.add_command(list_project_inherits_from)
+gerrit_cli.add_command(addmavenconfig)

lftools/git/gerrit.py

@@ -10,9 +10,10 @@
 
 """Gerrit git interface."""
 
+import configparser
+import json
 import logging
 import os
-import shutil
 import tempfile
 import urllib
 
@@ -66,12 +67,6 @@ def __init__(self, **params):
         default_ref = self.repo.git.rev_parse("origin/HEAD", abbrev_ref=True)
         self.default_branch = default_ref.split("/")[-1]
 
-    def __del__(self):
-        try:
-            shutil.rmtree(self.repo.working_tree_dir)
-        except FileNotFoundError:
-            log.info("Could not remove working directory {}".format(self.repo.working_tree_dir))
-
     def get_commit_hook(self, endpoint, working_dir):
         """Pulls in the Gerrit server's commit hook to add a changeId."""
         hook_url = urllib.parse.urljoin(endpoint, "tools/hooks/commit-msg")
@@ -92,19 +87,26 @@ def get_commit_hook(self, endpoint, working_dir):
             os.chmod(commit_msg_hook_path, 0o755)
 
     def add_file(self, filepath, content):
-        """Add a file to the current git repo.
-
-        Example:
-
-        local_path /tmp/INFO.yaml
-        file_path="somedir/example-INFO.yaml"
-        """
+        """Add a file to the current git repo."""
         if filepath.find("/") >= 0:
-            os.makedirs(os.path.split(filepath)[0])
+            try:
+                os.makedirs(os.path.split(filepath)[0])
+            except FileExistsError:
+                pass
         with open(filepath, "w") as newfile:
             newfile.write(content)
         self.repo.git.add(filepath)
 
+    def add_symlink(self, filepath, target):
+        """Add a symlink to the current git repo."""
+        try:
+            os.symlink(target, filepath)
+        except FileExistsError:
+            if not os.path.islink(filepath):
+                log.error("{} exists and is not a symlink".format(filepath))
+                return
+        self.repo.git.add(filepath)
+
     def commit(self, commit_msg, issue_id, push=False):
         """Commit staged changes.
 
@@ -190,3 +192,77 @@ def add_git_review(self, fqdn, gerrit_project, issue_id):
         self.add_file(filename, content)
         commit_msg = "Chore: Automation adds {}".format(filename)
         self.commit(commit_msg, issue_id, push=True)
+
+    def add_maven_config(self, fqdn, gerrit_project, issue_id, nexus3_url="", nexus3_ports=""):
+        """Add the four required JCasC files to create settings for a new project."""
+        project_dashed = gerrit_project.replace("/", "-")
+        params_path = "config-params.yaml"
+        creds_path = "serverCredentialMappings.yaml"
+        content_path = "content"
+        sb_creds_path = "serverCredentialMappings.sandbox.yaml"
+        nexus3_ports = nexus3_ports.split(",")
+
+        try:
+            default_servers = config.get_setting(self.fqdn, "default_servers")
+            default_servers = default_servers.split(",")
+        except configparser.NoOptionError:
+            default_servers = ["releases", "snapshots", "staging", "site"]
+
+        additional_credentials = ""
+        try:
+            credential_json = config.get_setting(self.fqdn, "additional_credentials")
+            additional_credentials = json.loads(credential_json)
+        except configparser.NoOptionError:
+            log.debug("No additional credentials found")
+        except json.decoder.JSONDecodeError:
+            log.error(
+                'Improperly formatted JSON in "additional_credentials". '
+                + "Please ensure that all credentials are on a single line, and are not quoted."
+            )
+
+        if not nexus3_url:
+            try:
+                nexus3_url = config.get_setting(self.fqdn, "nexus3")
+            except configparser.NoOptionError:
+                pass  # If no url is passed in or present in lftools.ini, skip it.
+
+        if nexus3_url and not nexus3_ports:
+            try:
+                nexus3_ports = config.get_setting(self.fqdn, "nexus3_ports")
+                nexus3_ports = nexus3_ports.split(",")
+            except configparser.NoOptionError:
+                nexus3_ports = ["10001", "10002", "10003", "10004"]
+
+        jinja_env = Environment(loader=PackageLoader("lftools.git"), autoescape=select_autoescape())
+        template = jinja_env.get_template(params_path)
+        config_params_content = template.render(project_dashed=project_dashed)
+        log.debug("config-params.yaml contents:\n{}".format(config_params_content))
+
+        template = jinja_env.get_template(creds_path)
+        server_creds_content = template.render(
+            project_dashed=project_dashed,
+            default_servers=default_servers,
+            nexus3_url=nexus3_url,
+            nexus3_ports=nexus3_ports,
+            additional_credentials=additional_credentials,
+        )
+        log.debug("config-params.yaml contents:\n{}".format(server_creds_content))
+
+        config_path = "jenkins-config/managed-config-files/mavenSettings/{}".format(project_dashed)
+        try:
+            os.makedirs(config_path)
+        except FileExistsError:
+            pass
+
+        self.add_symlink(
+            os.path.join(config_path, content_path), "../../../managed-config-templates/mavenSettings-content"
+        )
+        self.add_symlink(
+            os.path.join(config_path, sb_creds_path),
+            "../../../managed-config-templates/mavenSettings-serverCredentialMappings.sandbox.yaml",
+        )
+        self.add_file(os.path.join(config_path, params_path), config_params_content)
+        self.add_file(os.path.join(config_path, creds_path), server_creds_content)
+
+        commit_msg = "Chore: Automation adds {} config files".format(gerrit_project)
+        self.commit(commit_msg, issue_id, push=True)

lftools/git/templates/config-params.yaml

@@ -0,0 +1,3 @@
+---
+name: "{{ project_dashed }}"
+comment: "{{ project_dashed }}"

lftools/git/templates/serverCredentialMappings.yaml

@@ -0,0 +1,18 @@
+---
+serverCredentialMappings:
+{%- for server in default_servers %}
+  - serverId: "{{ server }}"
+    credentialsId: "{{ project_dashed }}"
+{%- endfor -%}
+{%- if nexus3_url and nexus3_ports -%}
+  {% for port in nexus3_ports %}
+  - serverId: "{{ nexus3_url }}:{{port}}"
+    credentialsId: "{{ project_dashed }}"
+  {%- endfor -%}
+{%- endif -%}
+{%- if additional_credentials -%}
+  {%- for server, cred in additional_credentials.items() %}
+  - serverId: "{{ server }}"
+    credentialsId: "{{ cred }}"
+  {%- endfor -%}
+{% endif %}

releasenotes/notes/jcasc-add-settings-487d8fb321a8a4a4.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    New method git.add_maven_config allows for adding settings files for new
+    projects via JCasC, rather than the old method of using a Groovy script to
+    manually add the file to Jenkins.

tests/test_git.py

@@ -9,6 +9,7 @@
 ##############################################################################
 """Test git command."""
 
+import configparser
 import os
 
 import pytest
@@ -117,3 +118,61 @@ def test_add_git_review(mock_init, datafiles, mocker):
 
     Gerrit.add_file.assert_called_once_with(filepath, content)
     Gerrit.commit.assert_called_once_with(commit_msg, issue_id, push=True)
+
+
+@pytest.mark.datafiles(os.path.join(FIXTURE_DIR, "git"))
+def test_add_maven_config(mock_init, datafiles, mocker):
+    fqdn = "gerrit.example.com"
+    gerrit_project = "project/subproject"
+    issue_id = "TEST-123"
+    commit_msg = "Chore: Automation adds project/subproject config files"
+
+    creds_path = "jenkins-config/managed-config-files/mavenSettings/project-subproject/serverCredentialMappings.yaml"
+    server_creds_content = """---
+serverCredentialMappings:
+  - serverId: "releases"
+    credentialsId: "project-subproject"
+  - serverId: "snapshots"
+    credentialsId: "project-subproject"
+  - serverId: "staging"
+    credentialsId: "project-subproject"
+  - serverId: "site"
+    credentialsId: "project-subproject\""""
+
+    get_setting_mock = mocker.patch("lftools.config.get_setting")
+    get_setting_mock.side_effect = configparser.NoOptionError(fqdn, "default_servers")
+    mocker.patch.object(Gerrit, "add_symlink")
+    mocker.patch.object(Gerrit, "add_file")
+    mocker.patch.object(Gerrit, "commit")
+
+    # Test 1 #
+    mock_init.add_maven_config(fqdn, gerrit_project, issue_id)
+    Gerrit.add_file.assert_called_with(creds_path, server_creds_content)
+    Gerrit.commit.assert_called_once_with(commit_msg, issue_id, push=True)
+
+    # Test 2 #
+    server_creds_content += """
+  - serverId: "nexus3.example.com:10001"
+    credentialsId: "project-subproject"
+  - serverId: "nexus3.example.com:10002"
+    credentialsId: "project-subproject\""""
+
+    mock_init.add_maven_config(fqdn, gerrit_project, issue_id, "nexus3.example.com", "10001,10002")
+    Gerrit.add_file.assert_called_with(creds_path, server_creds_content)
+
+    # Test 3 #
+    get_setting_mock = mocker.patch("lftools.config.get_setting")
+
+    def setting_response(*args, **kwargs):
+        if "additional_credentials" in args:
+            return '{"docker.io": "dockerhub-cred"}'
+        raise configparser.NoOptionError(fqdn, "default_servers")
+
+    get_setting_mock.side_effect = setting_response
+
+    server_creds_content += """
+  - serverId: "docker.io"
+    credentialsId: "dockerhub-cred\""""
+
+    mock_init.add_maven_config(fqdn, gerrit_project, issue_id, "nexus3.example.com", "10001,10002")
+    Gerrit.add_file.assert_called_with(creds_path, server_creds_content)