Merge pull request #75 from QualiSystemsLab/natti-offline-support

Natti Gitlab Offline Support

Author: alexazarh

.gitignore

@@ -64,4 +64,5 @@ cloudshell_config.yml
 
 # env files
 *.env
-!*.template.env
+!*.template.env
+*.zip

package/__init__.py

@@ -70,13 +70,14 @@ class ATTRIBUTE_NAMES:
     CT_INPUTS = "Custom Tags"
     APPLY_TAGS = "Apply Tags"
     REMOTE_STATE_PROVIDER = "Remote State Provider"
-    GITHUB_TERRAFORM_MODULE_URL = "Github Terraform Module URL"
+    GIT_TERRAFORM_MODULE_URL = "Git Terraform Module URL"
     TERRAFORM_VERSION = "Terraform Version"
-    GITHUB_TOKEN = "Github Token"
-    GITHUB_URL = "Github Terraform Module URL"
+    GIT_TOKEN = "Git Token"
     BRANCH = "Branch"
     CLOUD_PROVIDER = "Cloud Provider"
     UUID = "UUID"
+    GIT_PROVIDER = "Git Provider"
+    LOCAL_TERRAFORM = "Local Terraform"
 
 
 GET_BACKEND_DATA_COMMAND = "get_backend_data"

package/cloudshell/iac/terraform/constants.py

@@ -0,0 +1,19 @@
+from abc import ABC, abstractmethod
+from logging import Logger
+
+
+class GitScriptDownloaderBase(ABC):
+
+    def __init__(self, logger: Logger):
+        self.logger = logger
+
+    @abstractmethod
+    def download_repo(self, url: str, token: str, branch: str = "") -> str:
+        """
+        method should do the following:
+        1.make request
+        2. download repo
+        3. prepare working dir, add repo contents to working dir
+        4. return full path of working dir as string
+        """
+        pass

package/cloudshell/iac/terraform/downloaders/base_git_downloader.py

@@ -1,23 +1,35 @@
+import logging
+from typing import Type
+
 from cloudshell.iac.terraform.constants import ATTRIBUTE_NAMES
-from cloudshell.iac.terraform.downloaders.github_downloader import GitHubScriptDownloader
 from cloudshell.iac.terraform.downloaders.tf_exec_downloader import TfExecDownloader
 from cloudshell.iac.terraform.models.shell_helper import ShellHelperObject
+from cloudshell.iac.terraform.downloaders.base_git_downloader import GitScriptDownloaderBase
+from cloudshell.iac.terraform.downloaders.github_downloader import GitHubScriptDownloader
+from cloudshell.iac.terraform.downloaders.gitlab_downloader import GitLabScriptDownloader
 
 
 class Downloader(object):
     def __init__(self, shell_helper: ShellHelperObject):
         self._shell_helper = shell_helper
 
     def download_terraform_module(self) -> str:
-        url = self._shell_helper.attr_handler.get_attribute(ATTRIBUTE_NAMES.GITHUB_TERRAFORM_MODULE_URL)
-        token_enc = self._shell_helper.attr_handler.get_attribute(ATTRIBUTE_NAMES.GITHUB_TOKEN)
+        url = self._shell_helper.attr_handler.get_attribute(ATTRIBUTE_NAMES.GIT_TERRAFORM_MODULE_URL)
+        if not url:
+            raise ValueError(f"Must populate attribute '{ATTRIBUTE_NAMES.GIT_TERRAFORM_MODULE_URL}'")
+
+        token_enc = self._shell_helper.attr_handler.get_attribute(ATTRIBUTE_NAMES.GIT_TOKEN)
         token = self._shell_helper.api.DecryptPassword(token_enc).Value
         branch = self._shell_helper.attr_handler.get_attribute(ATTRIBUTE_NAMES.BRANCH)
 
-        self._shell_helper.sandbox_messages.write_message("downloading Terraform module from repository...")
-        self._shell_helper.logger.info("Downloading Terraform Repo from Github")
+        # get downloader mapped to git provider
+        provider = self._shell_helper.attr_handler.get_attribute(ATTRIBUTE_NAMES.GIT_PROVIDER)
+        downloader = self._downloader_factory(provider, logger=self._shell_helper.logger)
 
-        downloader = GitHubScriptDownloader(self._shell_helper.logger)
+        # download repo and return working dir
+        self._shell_helper.sandbox_messages.write_message("downloading Terraform module from repository...")
+        self._shell_helper.logger.info(f"Downloading Terraform Repo from '{provider}'")
+        self._shell_helper.logger.info(f"Download URL: '{url}'")
         return downloader.download_repo(url, token, branch)
 
     def download_terraform_executable(self, tf_workingdir: str) -> None:
@@ -32,3 +44,17 @@ def download_terraform_executable(self, tf_workingdir: str) -> None:
         except Exception as e:
             self._shell_helper.logger.error(f"Failed downloading Terraform Repo from Github {str(e)}")
             raise
+
+    def _get_downloader_class(self, git_provider: str) -> Type[GitScriptDownloaderBase]:
+        """ extend this dictionary with additional git provider downloaders """
+        git_downloader_map = {
+            "github": GitHubScriptDownloader,
+            "gitlab": GitLabScriptDownloader
+        }
+        if git_provider.lower() not in git_downloader_map:
+            raise NotImplementedError(f"Git Provider '{git_provider}' not supported")
+        return git_downloader_map[git_provider.lower()]
+
+    def _downloader_factory(self, git_provider: str, logger: logging.Logger) -> GitScriptDownloaderBase:
+        downloader_class = self._get_downloader_class(git_provider)
+        return downloader_class(logger)

package/cloudshell/iac/terraform/downloaders/downloader.py

@@ -2,7 +2,6 @@
 import json
 import os
 import re
-from logging import Logger
 from zipfile import ZipFile
 import tempfile
 import requests
@@ -11,17 +10,15 @@
 from retry import retry
 
 from cloudshell.iac.terraform.constants import GITHUB_REPO_PATTERN
+from cloudshell.iac.terraform.downloaders.base_git_downloader import GitScriptDownloaderBase
 
 GitHubFileData = collections.namedtuple(
     'GitHubFileData', 'account_id repo_id branch_id path api_zip_dl_url api_tf_dl_url'
 )
 REPO_FILE_NAME = "repo.zip"
 
 
-class GitHubScriptDownloader(object):
-
-    def __init__(self, logger: Logger):
-        self.logger = logger
+class GitHubScriptDownloader(GitScriptDownloaderBase):
 
     @retry((HTTPError, URLError), delay=1, backoff=2, tries=5)
     def download_repo(self, url: str, token: str, branch: str = "") -> str:

package/cloudshell/iac/terraform/downloaders/github_downloader.py

@@ -0,0 +1,151 @@
+import re
+from dataclasses import dataclass
+from typing import List
+from urllib.error import HTTPError, URLError
+from retry import retry
+from cloudshell.iac.terraform.downloaders.base_git_downloader import GitScriptDownloaderBase
+from cloudshell.iac.terraform.services.gitlab_api_handler import GitlabApiHandler
+from urllib.parse import unquote
+
+
+@dataclass
+class CommonGitLabUrlData:
+    protocol: str
+    domain: str
+    path: str
+    full_url: str
+    sha: str
+
+
+@dataclass
+class GitLabBrowserUrlData(CommonGitLabUrlData):
+    gitlab_user: str
+    project_name: str
+
+
+@dataclass
+class GitLabApiUrlData(CommonGitLabUrlData):
+    api_version: str
+    project_id: int
+    api_endpoint: str
+
+
+def extract_data_from_browser_url(url) -> GitLabBrowserUrlData:
+    """
+    Take api style url and extract data
+    Sample Raw Browser url: "http://192.168.85.26/quali_natti/terraformstuff/-/tree/test-branch/rds/project1"
+    'sha' can be branch or commit id
+    """
+    pattern = (r'^(?P<protocol>https?)://(?P<domain>[^/]+)/(?P<user>[^/]+)/(?P<project>[^/]+)/-/tree/'
+               r'(?P<sha>[^/]+)/(?P<path>.*)?$')
+
+    match = re.match(pattern, url)
+    if not match:
+        raise ValueError(f"No GitLab URL Data found in RAW url '{url}'")
+
+    groups = match.groupdict()
+    return GitLabBrowserUrlData(protocol=groups['protocol'],
+                                domain=groups['domain'],
+                                gitlab_user=groups['user'],
+                                project_name=groups['project'],
+                                sha=groups['sha'],
+                                path=groups['path'],
+                                full_url=url)
+
+
+def get_query_param_val(param_key: str, params_list: List[List[str]]) -> str:
+    """
+    look for target param in 2D list of key pair values
+    [[k1,v1],[k2,v2]]
+    if not found return empty string
+    """
+    target_param_search = [x for x in params_list if x[0] == param_key]
+    param_val = target_param_search[0][1] if target_param_search else ""
+    return param_val
+
+
+def extract_data_from_api_url(url) -> GitLabApiUrlData:
+    """
+    Take user style url and extract data
+    supports url-encoded style paths as well
+    Sample Api url: "http://192.168.85.26/api/v4/projects/2/repository/archive.zip?path=rds"
+    """
+    pattern = (r'^(?P<protocol>https?)://(?P<domain>[^/]+)(?P<api_version>/api/v\d+)?'
+               r'(?P<api_endpoint>/projects/(?P<project_id>\d+)/repository/archive\.zip)'
+               r'(?P<params>\?([^&]+=[^&]+&)*[^&]+=[^&]+$)')
+
+    match = re.match(pattern, url)
+    if not match:
+        raise ValueError(f"No GitLab url data found in API url '{url}'")
+
+    groups = match.groupdict()
+    query_params = groups['params']
+
+    # remove the leading '?' of the query  param string
+    query_params = query_params.split("?")[-1]
+
+    # split into 2D list [[k1,v1],[k2,v2]]
+    params_list = [x.split("=") for x in query_params.split("&")]
+
+    # search for target params
+    path = get_query_param_val("path", params_list)
+    sha = get_query_param_val("sha", params_list)
+    ref = get_query_param_val("ref", params_list)
+
+    # take sha param if passed, otherwise use the ref
+    sha = sha if sha else ref
+
+    # url encoded path not necessary
+    path = unquote(path)
+    sha = unquote(sha)
+    return GitLabApiUrlData(protocol=groups['protocol'],
+                            domain=groups['domain'],
+                            api_version=groups['api_version'],
+                            project_id=groups['project_id'],
+                            api_endpoint=groups['api_endpoint'],
+                            path=path,
+                            sha=sha,
+                            full_url=url)
+
+
+def is_gitlab_api_url(url: str) -> bool:
+    """
+    check if is api endpoint
+    Sample Api url: "http://192.168.85.26/api/v4/projects/2/repository/archive.zip?path=rds"
+    """
+    pattern = r'^(?P<protocol>https?)://(?P<domain>[^/]+)(?P<api_version>/api/v\d+)?(?P<api_endpoint>/[^\s]+)*/?$'
+    match = re.match(pattern, url)
+    if not match:
+        return False
+
+    groups = match.groupdict()
+    api_version = groups['api_version']  # "/api/v4"
+
+    if not api_version:
+        return False
+
+    return True
+
+
+class GitLabScriptDownloader(GitScriptDownloaderBase):
+
+    @retry((HTTPError, URLError), delay=1, backoff=2, tries=5)
+    def download_repo(self, url: str, token: str, branch: str = "") -> str:
+
+        # extract data from browser "raw style url" or "gitlab api" style
+        is_api_url = is_gitlab_api_url(url)
+        if is_api_url:
+            url_data = extract_data_from_api_url(url)
+        else:
+            url_data = extract_data_from_browser_url(url)
+
+        # allow service branch attr to override the url defined sha
+        sha = branch if branch else url_data.sha
+        is_https = True if url_data.protocol == "https" else False
+        api_handler = GitlabApiHandler(host=url_data.domain, token=token, is_https=is_https)
+
+        # if using raw style url, do lookup for project id from project name
+        project_id = url_data.project_id if is_api_url else api_handler.get_project_id_from_name(url_data.project_name)
+        working_dir = api_handler.download_archive_to_temp_dir(project_id=project_id, path=url_data.path, sha=sha)
+        self.logger.info(f"Temp Working Dir: {working_dir}")
+        return working_dir