jenkins-utilities

#!/bin/bash
#
# Copyright (C) Extensible Service Proxy Authors
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
#
################################################################################
#
## Library of useful utilities for Jenkins scripts.

DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ESP_ROOT="$(cd "${DIR}/.." && pwd)"

. ${DIR}/all-utilities || { echo "Cannot load Bash utilities" ; exit 1 ; }

PROJECT_ID="$(${GCLOUD} config list core/project --format flattened | cut -d ' ' -f 2)"
DEFAULT_ESP_SERVICE="testing-dot-${PROJECT_ID}.appspot.com"

# End to End tests options
API_KEY=''
BOOKSTORE_IMAGE=''
BUCKET=''
COUPLING_OPTION='loose'
DEBIAN_PKG=''
DIRECT_REPO=''
DURATION_IN_HOUR=0
ENDPOINTS_RUNTIME_VERSION=''
ESP_IMAGE='gcr.io/endpoints-release/endpoints-runtime:1'
BACKEND=''
INSTANCE_NAME=''
NAMESPACE=''
REMOTE_LOG_DIR=''
SKIP_CLEANUP='false'
SWAGGER_TMPL="${ESP_ROOT}/test/bookstore/swagger_template.json"
TEST_TYPE='http'
UNIQUE_ID='test'
VM_IMAGE=''
ESP_ROLLOUT_STRATEGY=''
ESP_SERVICE_VERSION=''

# Echo and un command, exit on failure
function run_nonfatal() {
  echo ""
  echo "[$(date)] $@"
  "${@}"
  local status=${?}
  if [[ "${status}" != "0" ]]; then
    echo "Command failed with exit status ${status}: ${@}" >&2
  fi
  return ${status}
}

# Echo and run a shell command, exit on failure
function run() {
  run_nonfatal "${@}" || error_exit "command failed"
}

# Convenience method to sed files, works on both linux and mac
function sed_i() {
  # Incompatible sed parameter parsing.
  if sed -i 2>&1 | grep -q 'requires an argument'; then
    sed -i '' "${@}"
  else
    sed -i "${@}"
  fi
}

# Deleting and Deactivating a service
function delete_service() {
  local esp_service="${1}"
  if ! [[ "${esp_service}" =~ "${DEFAULT_ESP_SERVICE}" ]]; then
    ${GCLOUD} services disable \
      "${esp_service}" --project="${PROJECT_ID}"
    ${GCLOUD} endpoints services delete "${esp_service}" --quiet
  fi
}

# Parse regex from a given variable
function parse_output() {
  local output="${1}"
  local regex="${2}"
  local value=''
  # change delimiter (IFS) to new line.
  local ifs_backup=${IFS}
  IFS=$'\n'

  for line in ${output}; do
    if [[ "${line}" =~ ${regex} ]]; then
      value="${BASH_REMATCH[1]}"
      echo "${value}"
    fi
  done

  # return delimiter to previous value
  IFS=${ifs_backup}
}

function set_esp_service_version() {
  local esp_service="${1}"
  local json_path="$(mktemp /tmp/XXXXX.json)"
  local version=''
  ${GCLOUD} endpoints configs list \
    --service=${esp_service} --format=json > "${json_path}" || return 1
  version="$(python - << __EOF__ "${json_path}"
import json
import sys
json_path = sys.argv[1]
with open(json_path, 'r') as f:
  result = json.load(f)
print str(result[0]['id'])
__EOF__
  )" || version=''
  rm -f "${json_path}"
  [[ -n "${version}" ]] && ESP_SERVICE_VERSION="${version}" \
    && return 0
  return 1
}

# Creating and activating a service
function create_service() {
  local esp_service="${1}"
  local output=''
  local operation=''
  local version=''

  echo 'Deploying service'
  retry -n 3 run ${GCLOUD} endpoints services deploy ${@:2}

  retry -n 5 set_esp_service_version "${esp_service}" \
    || error_exit 'Could not fetch the service version.'

  echo 'Activating service'
  retry -n 3 run ${GCLOUD} services enable \
    "${esp_service}" --project="${PROJECT_ID}" \
    || error_exit 'Could not activate the service.'

  echo "ESP service version: ${ESP_SERVICE_VERSION}"
}

# Set current rolloutId to ESP_CURRENT_ROLLOUTS variable
function set_esp_remote_rollout_info_env() {
  local target_env=${1}
  local endpoint_status_url=${2}
  local endpoint_status=""

  case "${target_env}" in
    'gce' )
      set_ssh_private_key
      endpoint_status=`${GCLOUD} compute ssh "${INSTANCE_NAME}" --command="curl \"$endpoint_status_url\""`;;
    'gke' )
      endpoint_status=`curl "$endpoint_status_url"`;;
  esac

  ESP_CURRENT_ROLLOUTS=`echo ${endpoint_status} | python -c '
import json
import sys
try:
  status=json.loads("".join(sys.stdin.readlines()))
  if status["processes"][0]["espStatus"][0]["serviceConfigRollouts"]["percentages"] is None:
    print("")
  else:
    percetages=status["processes"][0]["espStatus"][0]["serviceConfigRollouts"]["percentages"]

    config_ids = [];
    for config_id, percentage in percetages.iteritems():
      config_ids.append([config_id,percentage])

    config_ids.sort(key=lambda x: x[0])

    result=[]
    for i in config_ids:
      result.append(i[0])
      result.append(str(i[1]))

    print(" ".join(result))
except:
  print("")
'`
  return ${?}
}

function wait_for_service_config_rollouts_update() {
  local target_env=${1}
  local endpoint_status_url=${2}
  local target_rollouts=${3}

  ESP_CURRENT_ROLLOUTS=""
  # set_esp_remote_rollout_info_env will set ESP_CURRENT_ROLLOUTS
  retry -n 10 -t 90  set_esp_remote_rollout_info_env "$target_env" "$endpoint_status_url" \
    || error_exit 'Could not fetch the remote rollouts'
  echo "Instance rollout information=$ESP_CURRENT_ROLLOUTS"

  if [ "$target_rollouts" == "$ESP_CURRENT_ROLLOUTS" ]; then
    return 0
  else
    return 1
  fi
}

# Check for host http return code.
function check_http_service () {
  local host=${1}
  local http_code="${2}"
  local errors="$(mktemp /tmp/curl.XXXXX)"
  local http_response="$(curl -k -m 20 --write-out %{http_code} --silent --output ${errors} ${host})"
  echo "Pinging host: ${host}, response: ${http_response}"
  if [[ "${http_response}" == "${http_code}" ]]; then
    echo "Service is available at: ${host}"
    return 0
  else
    echo "Response body:"
    cat $errors
    echo "Service ${host} is not ready"
    return 1
  fi
}

function check_grpc_service() {
  local host=${1}
  cat <<EOF | "${ESP_ROOT}/bazel-bin/test/grpc/grpc-test-client"
server_addr: "${host}"
plans {
  echo {
    request {
      text: "Hello, world!"
    }
  }
}
EOF
  local status=${?}
  if [[ ${status} -eq 0 ]]; then
    echo "Service is available at: ${host}"
  else
    echo "Service ${host} is not ready"
  fi
  return ${status}
}

# Run and upload logs
function long_running_test() {
  local host="${1}"
  local duration_in_hour=${2}
  local api_key="${3}"
  local esp_service="${4}"
  local log_dir="${5}"
  local test_id="${6}"
  local run_id="${7}"
  local test_type=''
  [[ ${duration_in_hour} -gt 0 ]] && test_type='long-run-test_'
  local final_test_id="${test_type}${test_id}"
  local log_file="${log_dir}/${final_test_id}.log"
  local json_file="${log_dir}/${final_test_id}.json"
  local status
  local http_code=200

  echo "Running ${BACKEND} long running test on ${host}"
  if [[ "${BACKEND}" == 'echo' ]]; then
    # We expect HTTP host to run at port 8080 for GRPC tests
    local http_host="http://${host%:*}:8080"
    retry -n 20 check_grpc_service "${host}"
    status=${?}
    if [[ ${status} -eq 0 ]]; then
      run_nonfatal "${ESP_ROOT}"/script/linux-grpc-test-long-run \
        -h "${http_host}" \
        -g "${host}" \
        -l "${duration_in_hour}" \
        -a "${api_key}" \
        -s "${esp_service}" 2>&1 | tee "${log_file}"
      status=${PIPESTATUS[0]}
    fi
  elif [[ "${BACKEND}" == 'interop' ]]; then
    run_nonfatal "${ESP_ROOT}"/script/test-grpc-interop \
      -h "${host}" \
      -l "${duration_in_hour}" \
      -s "${esp_service}" 2>&1 | tee "${log_file}"
    status=${PIPESTATUS[0]}
  else
    retry -n 20 check_http_service "${host}/shelves" ${http_code}
    status=${?}
    if [[ ${status} -eq 0 ]]; then
      echo 'Running long running test.'
      run_nonfatal "${ESP_ROOT}"/script/linux-test-kb-long-run \
        -h "${host}" \
        -l "${duration_in_hour}" \
        -a "${api_key}" \
        -s "${esp_service}" 2>&1 | tee "${log_file}"
      status=${PIPESTATUS[0]}
    fi
  fi

  create_status_file \
    -f "${json_file}" \
    -s ${status} \
    -t "${final_test_id}" \
    -r "${run_id}" \
    || { echo "Could not create ${json_file}."; return 1; }

  return ${status}
}

# Point SSH_PRIVATE_KEY to the private key path and make sure it exists.
function set_ssh_private_key() {
  local ssh_path="${HOME}/.ssh"
  SSH_PRIVATE_KEY="${ssh_path}/google_compute_engine"
  if ! [[ -a "${SSH_PRIVATE_KEY}" ]]; then
    mkdir -p "${ssh_path}"
    local remote_path="gs://${BUCKET}/ssh_keys/*"
    echo "Getting ssh key from ${remote_path}"
    retry -n 3 ${GSUTIL} cp "${remote_path}" "${ssh_path}/"
    chmod -R 700 "${ssh_path}"
  fi
}

# Extract logs from VM and save it to a given directory.
function save_vm_logs() {
  local vm="${1}"
  local log_dir="${2}"
  local tar_file="${log_dir}/${vm}.tar.gz"

  # Make sure we have the private key to SSH to the VM
  # Using ssh instead of gcloud compute ssh, as gcloud uses external IP
  # address which does not work with restrictive firewall rules
  set_ssh_private_key
  [[ ${?} -eq 0 ]] || { echo 'Failed to download SSH private key'; return 1; }

  # For GCE, remove huge access log before tar
  ssh -i "${SSH_PRIVATE_KEY}" \
    -o StrictHostKeyChecking=no \
    "${vm}" sudo rm /var/log/nginx/access.log

  ssh -i "${SSH_PRIVATE_KEY}" \
    -o StrictHostKeyChecking=no \
    "${vm}" sudo tar czf - /var/log/ > "${tar_file}"
  [[ ${?} -eq 0 ]] || { echo 'Failed to get logs'; return 1; }
}

# Upload logs remote directory
function upload_logs() {
  local remote_dir="${1}"
  local log_dir="${2}"

  echo "Uploading content of ${log_dir} to ${remote_dir}"
  retry -n 3 ${GSUTIL} -h 'Content-Type:text/plain' -m cp -r \
    "${log_dir}" "${remote_dir}" \
    || echo "Failed to upload ${log_dir}"
}

# End to End tests common options
function e2e_options() {
  local OPTIND OPTARG arg
  while getopts :a:b:B:c:d:e:g:i:k:l:r:R:st:v:V: arg; do
    case ${arg} in
      a) ESP_SERVICE="${OPTARG}";;
      b) BOOKSTORE_IMAGE="${OPTARG}";;
      B) BUCKET="${OPTARG}";;
      c) COUPLING_OPTION="$(echo ${OPTARG} | tr '[A-Z]' '[a-z]')";;
      d) DEBIAN_PKG="${OPTARG}";;
      e) ESP_IMAGE="${OPTARG}";;
      g) BACKEND="${OPTARG}";;
      i) UNIQUE_ID="${OPTARG}";;
      k) API_KEY="${OPTARG}";;
      l) DURATION_IN_HOUR="${OPTARG}";;
      r) DIRECT_REPO="${OPTARG}";;
      R) ESP_ROLLOUT_STRATEGY="${OPTARG}";;
      s) SKIP_CLEANUP='true';;
      t) TEST_TYPE="$(echo ${OPTARG} | tr '[A-Z]' '[a-z]')";;
      v) VM_IMAGE="${OPTARG}";;
      V) ENDPOINTS_RUNTIME_VERSION="${OPTARG}";;
      *) e2e_usage "Invalid option: -${OPTARG}";;
    esac
  done

  if [[ -z ${API_KEY} ]]; then
    # Setting APY_KEY
    set_api_keys
    API_KEY="${ENDPOINTS_JENKINS_API_KEY}"
    [[ -n "${API_KEY}" ]] || error_exit 'Could not set api key.'
  fi

  if [[ -n "${BUCKET}" ]]; then
    local git_commit="$(git rev-parse --verify HEAD)"
    REMOTE_LOG_DIR="gs://${BUCKET}/${git_commit}/logs/${UNIQUE_ID}"
  fi
}

function get_host_ip() {
  local hostname="${1}"
  local ip=''
  ip="$(${GCLOUD} compute instances describe "${hostname}" \
    --format='value(networkInterfaces.networkIP)')" || { ip=''; return 1; }
  export HOST_INTERNAL_IP="${ip}"
  return 0
}

function gke_namespace_cleanup {
  if [[ "${SKIP_CLEANUP}" == 'false' ]]; then
    run kubectl delete namespace "${NAMESPACE}"
    # Uncomment this line when the limit on #services is lifted or increased to > 20
    # run gcloud endpoints services delete ${ESP_SERVICE} --quiet
  fi
}

function get_gke_service_ip () {
  local ns="${1}"
  local name="${2}"
  local COUNT=30
  local SLEEP=15
  for i in $( seq 1 ${COUNT} ); do
    local host=$(kubectl -n "${ns}" get service "${name}" | awk '{print $4}' | grep -v EXTERNAL-IP)
      [ '<pending>' != $host ] && break
      echo "Waiting for server external ip. Attempt  #$i/${COUNT}... will try again in ${SLEEP} seconds" >&2
      sleep ${SLEEP}
  done
  if [[ '<pending>' == $host ]]; then
    echo 'Failed to get the GKE cluster host.'
    return 1
  else
    echo "$host"
    return 0
  fi
}

function e2e_usage() {
  local error_message="${1}"
  [[ -n "${error_message}" ]] && echo "${error_message}"
  echo "usage: ${BASH_SOURCE[1]}
  -a <esp service name to use>
  -b <Bookstore docker image>
  -B <bucket to upload files in>
  -c <coupling option: loose, tight>
  -d <debian package>
  -e <esp docker image>
  -i <Unique identifier for instances and kubernetes namespaces>
  -k <api key>
  -l <duration in hour>
  -r <debian repo>
  -s <skip cleanup>
  -t <test type: http, https, custom>
  -V <endpoints-runtime version>
  -v <vm image to use. Supported are [debian-8]>"
  exit 1
}