Makefile

SHELL = bash
OK_MSG = \x1b[32m ✔\x1b[0m
FAIL_MSG = \x1b[31m ✖\x1b[0m
YELLOW = \x1b[33m
BLUE = \x1b[36m
RED = \x1b[31m
RESET_COLOR = \x1b[0m
PIPELINEWISE_HOME = $(shell pwd)
VENV_DIR = ${PIPELINEWISE_HOME}/.virtualenvs

python ?= "python3"

start_time:=$(shell date +%s)

PIP_ARGS="[test]"

pw_connector=

define DEFAULT_CONNECTORS
tap-jira\
tap-kafka\
tap-mysql\
tap-postgres\
tap-s3-csv\
tap-salesforce\
tap-snowflake\
tap-zendesk\
tap-mongodb\
tap-github\
tap-slack\
tap-mixpanel\
tap-twilio\
target-s3-csv\
target-snowflake\
target-redshift\
target-postgres\
target-bigquery\
transform-field
endef

define EXTRA_CONNECTORS
tap-adwords\
tap-oracle\
tap-zuora\
tap-google-analytics\
tap-shopify
endef

define print_installed_connectors
	@echo
	@echo "--------------------------------------------------------------------------"
	@echo "Installed components:"
	@echo "--------------------------------------------------------------------------"
	@echo
	@echo "Component            Version"
	@echo "-------------------- -------"
	@for i in $(shell ls $(VENV_DIR)); do\
		VERSION=`$(VENV_DIR)/$$i/bin/python3 -m pip list | grep "$$i[[:space:]]" | awk '{print $$2}'`;\
		printf "%-20s %s\n" $$i "$$VERSION";\
	done;
	@echo "-------------------- -------"
endef

define print_execute_time
	$(eval end_time:=`date +%s`)
	@echo
	@echo "--------------------------------------------------------------------------"
	@echo "$(1) installed successfully in $$(( $(end_time) - $(start_time) )) seconds"
	@echo "--------------------------------------------------------------------------"
endef

define clean_connectors
	echo -n "Cleaning previous installations in $(VENV_DIR)/$(1)..."
	rm -rf $(VENV_DIR)/$(1)
	@echo -e "$(OK_MSG)"
endef

define check_license
	@echo "Checking license..."
	@echo -e "$(YELLOW)"
	@$(VENV_DIR)/$(1)/bin/python3 -m pip install pip-licenses==3.5.3
	@echo -e "$(RESET_COLOR)"
	$(eval PKG_NAME:=`$(VENV_DIR)/$(1)/bin/pip-licenses|grep "$(1)[[:space:]]"| awk '{print $$$$1}'`)
	$(eval PKG_VERSION:=`$(VENV_DIR)/$(1)/bin/pip-licenses | grep "$(1)[[:space:]]" | awk '{print $$$$2}'`)
	$(eval PKG_LICENSE:=`$(VENV_DIR)/$(1)/bin/pip-licenses --from mixed | grep "$(1)[[:space:]]" | awk '{for (i=1; i<=NF-2; i++) $$$$i = $$$$(i+2); NF-=2; print}'`)

	$(eval MAIN_LICENSE:="Apache Software License")

	@if [[ "$(PKG_LICENSE)" != $(MAIN_LICENSE) && "$(PKG_LICENSE)" != "UNKNOWN" ]]; then\
		echo -e "$(RED)";\
		echo;\
        echo "  | $(PKG_NAME) ($(PKG_VERSION)) is licensed under $(PKG_LICENSE)";\
        echo "  |";\
        echo "  | WARNING. The license of this connector is different than the default PipelineWise license ($(MAIN_LICENSE)).";\
        if [[ "$(ACCEPT_LICENSES)" != "YES" ]]; then\
            echo "  | You need to accept the connector's license agreement to proceed.";\
            echo "  |";\
            read -r -p "  | Do you accept the [$(PKG_LICENSE)] license agreement of $(PKG_NAME) connector? [y/N] " response;\
			if [[ $$response != "y" && $$response != "Y" ]]; then\
				echo;\
				echo -e "$(RESET_COLOR)";\
				echo "EXIT. License agreement not accepted!";\
				exit 1;\
			fi;\
		else\
			echo "  | You automatically accepted this license agreement by running this script with acceptlicenses=YES option.";\
        fi;\
        echo;\
	fi
	@echo -e "$(RESET_COLOR)"
	@echo -n "License accepted..."
	@echo -e "$(OK_MSG)"
endef

define make_virtualenv
	@echo -n "Making Virtual Environment for $(1) in $(VENV_DIR)..."
	@echo -e -n "$(YELLOW)"
	@test -d $(VENV_DIR)/$(1) || $(python) -m venv $(VENV_DIR)/$(1)
	@source $(VENV_DIR)/$(1)/bin/activate
	@echo -e "$(OK_MSG)"
	@echo -e -n "$(YELLOW)"
	@$(VENV_DIR)/$(1)/bin/python3 -m pip install --upgrade pip setuptools wheel
	@echo -e "$(RESET_COLOR)"
	@echo -n "Python setup tools updated..."
	@echo -e "$(OK_MSG)"
	@echo -e -n "$(YELLOW)"
	@test ! -s $(2)pre_requirements.txt ||\
 		($(VENV_DIR)/$(1)/bin/pip install --upgrade -r $(2)pre_requirements.txt\
 		&& echo -e "$(RESET_COLOR)"\
 		&& echo -n "Pre requirements installed..."\
 		&& echo -e "$(OK_MSG)")
	@echo -e -n "$(YELLOW)"
	@test ! -s $(2)requirements.txt ||\
		($(VENV_DIR)/$(1)/bin/pip install --upgrade -r $(2)requirements.txt\
 		&& echo -e "$(RESET_COLOR)"\
 		&& echo -n "Requirements installed..."\
 		&& echo -e "$(OK_MSG)")
	@echo -e -n "$(RESET_COLOR)"
	@test ! -s $(2)setup.py ||\
		(echo "Installing the package..."\
		 && echo -e "$(YELLOW)"\
		 && $(VENV_DIR)/$(1)/bin/pip install --upgrade -e .$(PIP_ARGS)\
		 && echo -e "$(RESET_COLOR)"\
		 && echo -n "Package installation completed..."\
		 && echo -e "$(OK_MSG)")
	@echo -e "$(RESET_COLOR)"
	$(call check_license,$(1))
endef


define install_connectors
	echo
	echo "--------------------------------------------------------------------------"
	echo "Installing $1 connector..."
	echo "--------------------------------------------------------------------------"
	if [[ ! -d singer-connectors/$1 ]]; then\
		echo "ERROR: Directory not exists and does not look like a valid singer connector: singer-connectors: singer-connectors/$1";\
		exit 1;\
    fi
    $(call make_virtualenv,$1,singer-connectors/$1/)
endef

define print_list_of_connectors
	echo "   $1"
endef

help: .check_gettext .pw_logo
	@echo
	@echo "  Targets"
	@echo "  ======="
	@echo "     pipelinewise                                               Install the main PipelineWise component"
	@echo "     pipelinewise_no_test_extras                                Install the main Pipelinewise component without test extras"
	@echo
	@echo "     all_connectors                                             Install all connectors"
	@echo "     default_connectors                                         Install default connectors"
	@echo "     extra_connectors                                           Install only extra connectors"
	@echo "     connectors -e pw_connector=connector1,connector2,...       Install specific connector(s)"
	@echo
	@echo "     list_installed_components                                  Show a list of installed components"
	@echo "     list_default_connectors                                    Show a list of available default connectors"
	@echo "     list_extra_connectors                                      Show a list of available extra connectors"
	@echo
	@echo "     clean_all                                                  Clean all installed components"
	@echo "     clean -e pw_connector=connector1,connector2,...            Clean a specific connector(s)"
	@echo
	@echo "   Options"
	@echo "   ======="
	@echo "      -e pw_connector=connector1,connector2,...                 Define a list of connectors for installing or cleaning"
	@echo "      -e pw_acceptlicenses=y/Y/Yes/YES                          Forcing to accept the licenses automatically"
	@echo
	@echo "   To start CLI"
	@echo "   ============"
	@echo "      $$ source $(VENV_DIR)/pipelinewise/bin/activate"
	@echo "      $$ export PIPELINEWISE_HOME=$(PIPELINEWISE_HOME)"
	@echo "      $$ pipelinewise status"
	@echo
	@echo "--------------------------------------------------------------------------"


pipelinewise: .check_gettext .pw_logo
	$(call make_virtualenv,pipelinewise)
	$(call print_execute_time,PipelineWise)

pipelinewise_no_test_extras: .set_pip_args pipelinewise

clean_all:
	@echo -n "Cleaning previous installations in $(VENV_DIR)..."
	@rm -rf $(VENV_DIR)
	@echo -e "$(OK_MSG)"

clean:
ifeq ($(pw_connector),)
	@echo "use -e pw_connector=connector1,connector2,...."
	@exit 1
endif
	$(eval space:= )
	$(eval space+= )
	$(eval comma:=,)
	$(eval connectors_list:=$(subst $(comma),$(space),$(pw_connector)))

	@$(foreach var,$(connectors_list), $(call clean_connectors,$(var));)

connectors: .check_license_env_var
ifeq ($(pw_connector),)
	@echo "use -e pw_connector=connector1,connector2,...."
	@exit 1
endif
	$(eval space:= )
	$(eval space+= )
	$(eval comma:=,)
	$(eval connectors_list:=$(subst $(comma),$(space),$(pw_connector)))

	@$(foreach var,$(connectors_list), $(call install_connectors,$(var));)
	$(call print_execute_time,Connectors)


all_connectors: default_connectors extra_connectors
	@echo "Install all connectors..."
	$(call print_execute_time,All connectors)

default_connectors: .check_license_env_var
	@echo "Installing default connectors..."
	@$(foreach var,$(DEFAULT_CONNECTORS), $(call install_connectors,$(var));)
	$(call print_execute_time,Default connectors)

extra_connectors: .check_license_env_var
	@echo "Installing extra connectors..."
	@$(foreach var,$(EXTRA_CONNECTORS), $(call install_connectors,$(var));)
	$(call print_execute_time,Extra connectors)


list_installed_components:
	$(call print_installed_connectors)

list_default_connectors:
	@echo
	@echo "   ============================"
	@echo "   Available Default Connectors"
	@echo "   ============================"
	@$(foreach var,$(DEFAULT_CONNECTORS), $(call print_list_of_connectors,$(var));)
	@echo "   ----------------------------"

list_extra_connectors:
	@echo
	@echo "   ============================"
	@echo "   Available Extra Connectors"
	@echo "   ============================"
	@$(foreach var,$(EXTRA_CONNECTORS), $(call print_list_of_connectors,$(var));)
	@echo "   ----------------------------"

.pw_logo:
	@echo -e "$(BLUE)"
	@(CURRENT_YEAR=$(shell date +"%Y") envsubst < motd)
	@echo -e "$(RESET_COLOR)"

.check_license_env_var:
	$(eval ACCEPT_LICENSES:=NO)
ifeq ($(pw_acceptlicenses),y)
	$(eval ACCEPT_LICENSES:=YES)
endif
ifeq ($(pw_acceptlicenses),Y)
	$(eval ACCEPT_LICENSES:=YES)
endif
ifeq ($(pw_acceptlicenses),Yes)
	$(eval ACCEPT_LICENSES:=YES)
endif
ifeq ($(pw_acceptlicenses),YES)
	$(eval ACCEPT_LICENSES:=YES)
endif

.check_gettext:
	@echo -n "Checking gettext..."
	@if ! ENVSUBST_LOC="$$(type -p "envsubst")" || [[ -z ENVSUBST_LOC ]]; then\
		echo -e "$(FAIL_MSG)" &&\
		echo "envsubst not found but it is required to run this script. Try to install gettext or gettext-base package" && exit 1;\
	fi
	@echo -e "$(OK_MSG)"

.set_pip_args:
	$(eval PIP_ARGS:="")