simple

#!/bin/sh
# simple - CLI for the SIMPLE Framework
# v2.1 - 2020/03/17
# authors: Maarten.Litmaath@cern.ch, Mayank.Sharma@cern.ch

logdir=/var/log/simple
# Add /opt/puppetlabs/bin to path
[[ ":$PATH:" != *":/opt/puppetlabs/bin:"* ]] && PATH="/opt/puppetlabs/bin:${PATH}"
mkdir -p $logdir || exit

usage()
{
    echo "Usage: $0 command [args]" >&2
    echo "Commands [args]:" >&2
    cat >&2 << EOF
    pre-install-cm
    create-template
    pre-compile
    extract-lc
    generate-key
    test-ssh
    pre-install-lc
    install
    install-cm
    install-lc CM_hostname
    cm-stage
    lc-stage
    check-stage [ install | config | pre_deploy | deploy | final ]
    rollback-to { config [lc] | pre_deploy | deploy [rmi] }
    pre-deploy
    deploy
    docker { ls | ps }
    validate { pre_install | install | config | pre_deploy | deploy }
EOF
    echo "Logs in $logdir" >&2
    echo "" >&2

    exit 1
}

cli_conf_dir=~/.simple

mkdir -p $cli_conf_dir || exit

simple_prvkey=$cli_conf_dir/simple_key
simple_pubkey=$simple_prvkey.pub
tested_ssh=$cli_conf_dir/.tested_ssh
compiler_dir=$cli_conf_dir/simple_grid_yaml_compiler
compiler_env=$compiler_dir/.env
installed_pip=$cli_conf_dir/.installed_pip
installed_compiler=$compiler_dir/.installed_compiler
installed_virtualenv=$cli_conf_dir/.installed_virtualenv
installed_git=$cli_conf_dir/.installed_git
installed_wget=$cli_conf_dir/.installed_wget
installed_lc_extractor=$cli_conf_dir/.installed_lc_extractor
validator_dir=$cli_conf_dir/validator
installed_validator=$validator_dir/.installed_validator
validator_env=$validator_dir/.env
validator_url='http://github.com/simple-framework/simple_grid_infra_validation_engine'
validator_revision=current
confdir=/etc/simple_grid/site_config
conf=$confdir/site_level_config_file.yaml
pre_compiled_conf=$compiler_dir/output.yaml
compiled_conf=$confdir/augmented_site_level_config_file.yaml
validation_cmd="simple_infra_validation_engine validate -vvv -i $simple_prvkey"
validation_overrule=0
lc_file=/etc/simple_grid/lc
lc_extractor_url="https://raw.githubusercontent.com/simple-framework/simple_bash_cli/current/extract-lc.py"
lc_extractor_dir=$cli_conf_dir/lc_extractor
lc_extractor_env=$lc_extractor_dir/.env

stamp=`date +%y%m%d-%H%M%S`
err_=$logdir/simple-$stamp-err-
out_=$logdir/simple-$stamp-out-
warn=3

puppet_repo=https://yum.puppetlabs.com/puppet5/puppet5-release-el-7.noarch.rpm
install_rpm="rpm -ivh $puppet_repo; yum -y --disablerepo='*' --enablerepo=puppet5 --enablerepo=base --enablerepo=updates install puppet-agent"
install_mod="(. /etc/profile.d/puppet-agent.sh; puppet module install maany-simple_grid)"
mp=/etc/puppetlabs/code/environments/production/modules
sg_i=simple_grid::install
cm_i=$sg_i::config_master::simple_installer
lc_i=$sg_i::lightweight_component::simple_installer


prepare_cli()
{
    [ -e $installed_pip ] || (
	mkdir -p $compiler_dir/.temp &&
	yum install -y epel-release
    yum install -y python-pip && pip install --upgrade pip &&
	touch $installed_pip
    ) || return

    [ -e $installed_virtualenv ] || (
	yum install -y python-virtualenv && touch $installed_virtualenv
    ) || return
}

run_validation()
{
    (
	prepare_cli || exit

	[ -e $installed_git ] || (
	    yum install -y git && touch $installed_git
	) || exit

	[ -e $installed_validator ] || (
	    mkdir -p $validator_dir && cd $validator_dir &&
	    git clone -b $validator_revision "$validator_url" $validator_dir &&
	    virtualenv $validator_env && source $validator_env/bin/activate &&
	    pip install -r requirements.txt && pip install $validator_dir &&
	    touch $installed_validator
	) || exit

	source $validator_env/bin/activate
	$validation_cmd "$@"
	val=$?

	if [ $val != 0 ] && [ "x$validation_overrule" = x1 ]
	then
	    echo "Overruling exit status $val with 0..."
	    val=0
	fi

	exit $val
   )
}

run_on_lc()
{
    op=$1
    shift

    cmd="$*"
    pids=

    for lc in `cat $lc_file`
    do
	echo "Launching on $lc"
	lcl=$op-$lc.txt
	ssh -i $simple_prvkey $lc "$cmd" < /dev/null > $out_$lcl 2> $err_$lcl &
	pids="$pids $!"
	eval lc_$!=\$lc
    done

    echo "Waiting for the launched commands to return..."

    while [ ${pids:+x} ]
    do
	echo -n .
	sleep 10
	tmp=

	for p in $pids
	do
	    if kill -0 $p 2> /dev/null
	    then
		tmp="$tmp $p"
	    else
		wait $p
		let "val |= $?"
		echo ''
		eval echo LC \$lc_$p finished.
	    fi
	done

	pids=$tmp
    done
}


case "${1:-?}" in
*[!a-z0-9-]*)
    usage
esac

log=$1.txt out=$out_$log err=$err_$log
val=0

case $1 in
pre-install-cm)
    echo "Pre-installing the CM..."

    (eval "$install_rpm puppetserver && $install_mod") > $out 2> $err || val=$?
    ;;

###############################################################################
create-template)
    c="class{'$cm_i::create_sample_site_level_config_file':}"

    puppet apply -e "$c" > $out 2> $err &&
    echo "The template has been created in $conf" && exit 0 || val=$?
    ;;

###############################################################################
pre-compile)
    (
	prepare_cli || exit

	[ -e $installed_compiler ] || (
	    virtualenv $compiler_env && source $compiler_env/bin/activate &&
	    pip install simple-grid-yaml-compiler && touch $installed_compiler
	) || exit

	cd $compiler_dir
	source $compiler_env/bin/activate
	simple_grid_yaml_compiler $conf -o $pre_compiled_conf -s schema.yaml
    ) > $out 2> $err

    val=$?
    ;;

###############################################################################
extract-lc)
    (
	prepare_cli || exit

	[ -e $pre_compiled_conf ] || {
	    echo "Please run '$0 pre-compile' to generate $pre_compiled_conf" >&2
	    exit 1
	}

	[ -e $installed_wget ] || (
	    yum install -y wget && touch $installed_wget
	) || exit

	[ -e $installed_lc_extractor ] || (
	    mkdir -p $lc_extractor_dir &&
	    wget -P  $lc_extractor_dir $lc_extractor_url &&
	    virtualenv $lc_extractor_env &&
            source $lc_extractor_env/bin/activate && pip install argparse pyyaml &&
	    touch $installed_lc_extractor
	) || exit

	source $lc_extractor_env/bin/activate
	python $lc_extractor_dir/extract-lc.py -f $pre_compiled_conf -o $lc_file &&
	echo "" >> $lc_file
    ) > $out 2> $err

    val=$?
    ;;

###############################################################################
generate-key)
    [ -e $lc_file ] || {
	echo "Please run '$0 extract-lc' to generate $lc_file" >&2
	exit 1
    }

    ssh-keygen -q -f $simple_prvkey -P '' || exit

    echo "Please add the public key in $simple_pubkey to your LC hosts:"
    echo ""
    cat $lc_file
    echo ""
    echo "For example, append it to /root/.ssh/authorized_keys on those hosts,"
    echo "then run '$0 test-ssh' to check if ssh works from the CM."
    exit
    ;;

###############################################################################
test-ssh)
    [ -e $simple_pubkey ] || {
	echo "Please run '$0 generate-key' to generate $simple_pubkey" >&2
	exit 1
    }

    for lc in `cat $lc_file`
    do
	echo "Trying $lc"
	ssh -i $simple_prvkey $lc echo success < /dev/null || val=$?
    done

    [ $val = 0 ] && touch $tested_ssh ||
	echo "It failed for at least 1 LC host: please fix that." >&2

    exit $val
    ;;

###############################################################################
pre-install-lc)
    [ -e $tested_ssh ] || {
	echo "Please run '$0 test-ssh' first" >&2
	exit 1
    }

    echo "Pre-installing the LC."
    echo 'This make take a few minutes...'

    cmd="$install_rpm && $install_mod && (grep -qF 'puppetlabs' /etc/environment) || (echo 'PATH=$PATH:/opt/puppetlabs/bin' >> /etc/environment)"

    run_on_lc $1 "$cmd"
    ;;

###############################################################################
install)
    [ -e $tested_ssh ] || {
	echo "Please run '$0 test-ssh' first" >&2
	exit 1
    }

    stage=pre_install
    echo "First validating the '$stage' stage..."

    (
	echo ""
	echo "==> Validating the '$stage' stage:"
	echo ""

	targets=$(tr '\n' ',' < $lc_file) &&
	run_validation -t $targets $stage
    ) > $out 2> $err

    val=$?
    [ $val = $warn ] && echo "Ignoring exit code $val" && val=0

    if [ $val = 0 ]
    then
	echo "Now installing the CM."
	echo 'This make take a few minutes...'
	puppet apply --modulepath $mp -e "class{'$cm_i':}" >> $out 2>> $err
	val=$?
    fi

    if [ $val = 0 ]
    then
	echo "Now installing the LC."
	echo 'This make take a few minutes...'

	CM=`hostname -f`
	c="class{'$lc_i':puppet_master => '$CM'}"
	cmd="puppet apply --modulepath $mp -e \"$c\""

	run_on_lc $1 "$cmd"
    fi
    ;;

###############################################################################
install-cm)
    echo 'This make take a few minutes...'
    puppet apply --modulepath $mp -e "class{'$cm_i':}" > $out 2> $err
    val=$?
    ;;

###############################################################################
install-lc)
    case $2 in
    *.*.*)
	:
	;;
    *)
	echo "$0: $1 requires CM hostname as argument" >&2
	exit 1
    esac

    c="class{'$lc_i':puppet_master => '$2'}"

    echo 'This make take a few minutes...'
    puppet apply --modulepath $mp -e "$c" > $out 2> $err
    val=$?
    ;;

###############################################################################
cm-stage)
    puppet facts| grep simple_stage
    exit
    ;;

###############################################################################
lc-stage)
    bolt command run 'puppet facts| grep simple_stage' -t @$lc_file
    exit
    ;;

###############################################################################
check-stage)
    stage=${2/-/_}
    def=any
    stage=${stage:-$def}

    case $stage in
    $def|install|config|pre_deploy|deploy|final)
	:
	;;
    *)
	echo "$0: invalid stage name: $stage" >&2
	usage
    esac

    bolt task run simple_grid::check_stage \
    augmented_site_level_config_file=$compiled_conf \
    site_infrastructure_key=site_infrastructure expected_stage=$stage -t localhost 2>&1 |
    sed -n '/^ *{ *$/,$p'
    exit
    ;;

###############################################################################
rollback-to)
    stage=${2/-/_}

    case $stage in
    config|pre_deploy|deploy)
	:
	;;
    *)
	echo "$0: $1 requires a valid stage name as argument" >&2
	usage
    esac

    case $stage:$3 in
    config:lc)
	c='class{"simple_grid::config::lightweight_component::rollback":}'

	echo 'This make take a few minutes...'
	bolt command run "puppet apply -e '$c'" -t @$lc_file > $out 2> $err
	val=$?
	echo "The command finished with exit code $? (logs in $logdir)"
	exit $val
	;;
    deploy:rmi)
	rmi='remove_images => true'
	;;
    *)
	rmi=
    esac

    c="class{'simple_grid::$stage::config_master::rollback':$rmi}"

    echo 'This make take a few minutes...'
    puppet apply -e "$c" > $out 2> $err
    val=$?
    ;;

###############################################################################
pre-deploy)
    stage=install
    echo "First validating the '$stage' stage..."

    (
	echo ""
	echo "==> Validating the '$stage' stage:"
	echo ""
	run_validation $stage
    ) > $out 2> $err

    val=$?
    [ $val = $warn ] && echo "Ignoring exit code $val" && val=0

    if [ $val = 0 ]
    then
	stage=config
	echo "Now validating the '$stage' stage..."

	(
	    echo ""
	    echo "==> Validating the '$stage' stage:"
	    echo ""
	    run_validation $stage
	) >> $out 2>> $err

	val=$?
	[ $val = $warn ] && echo "Ignoring exit code $val" && val=0
    fi

    if [ $val = 0 ]
    then
	stage=pre_deploy
	echo "Now running the '$stage' stage."
	echo 'This make take a few minutes...'

	(
	    echo ""
	    echo "==> Running the '$stage' stage:"
	    echo ""
	    puppet agent -t
	) >> $out 2>> $err

	val=$?
	[ $val = 2 ] && echo "Ignoring exit code $val" && val=0
    fi
    ;;

###############################################################################
deploy)
    stage=pre_deploy
    echo "First validating the '$stage' stage..."

    (
	echo ""
	echo "==> Validating the '$stage' stage:"
	echo ""
	run_validation $stage
    ) > $out 2> $err

    val=$?
    [ $val = $warn ] && echo "Ignoring exit code $val" && val=0

    stage=deploy

    if [ $val = 0 ]
    then
	msg='==> Finished with exit code'

	echo "Now running the '$stage' stage."
	echo 'This make take ~15 minutes per container...'
	echo "Please check stdout for '$msg' at the end:"
	echo "stdout: $out"
	echo "stderr: $err"

	(
	    echo ""
	    echo "==> Running the '$stage' stage:"
	    echo ""
	    puppet agent -t
	    val=$?

	    [ $val = 2 ] && echo "Ignoring exit code $val" && val=0

	    if [ $val = 0 ]
	    then
		echo ""
		echo "==> Validating the '$stage' stage:"
		echo ""
		run_validation $stage
		val=$?
	    fi

	    echo "$msg $val"
	) >> $out 2>> $err < /dev/null &

	echo 'Continuing in the background...'
	exit
    fi
    ;;

###############################################################################
docker)
    case $2 in
    ls)
	cmd='image ls'
	;;
    ps)
	cmd='ps -a'
	;;
    *)
	echo "$0: unexpected argument of docker command" >&2
	usage
    esac

    bolt command run "docker $cmd" -t @$lc_file
    exit
    ;;

###############################################################################
validate)
    stage=${2/-/_}

    case $stage in
    pre_install|install|config|pre_deploy|deploy)
	:
        ;;
    *)
	echo "$0: $1 requires a valid stage as argument" >&2
	usage
    esac

    shift 2
    validation_overrule=0
    run_validation "$@" $stage > $out 2> $err
    val=$?
    ;;

###############################################################################
*)
    usage
esac

echo "The command finished with exit code $val (logs in $logdir)"
exit $val