From 727a2914fc08af050b11c468e6a4fb8abf572953 Mon Sep 17 00:00:00 2001 From: Andrej Podhradsky Date: Fri, 30 Oct 2020 17:18:39 +0100 Subject: [PATCH] ONEOCPDEPL-43: Add support for OSD4 Signed-off-by: Andrej Podhradsky --- lib/bushslicer.rb | 1 + lib/launchers/amz.rb | 7 ++ lib/launchers/cloud_helper.rb | 2 + lib/launchers/ocm.rb | 198 ++++++++++++++++++++++++++++++++++ lib/launchers/ocm_test.rb | 92 ++++++++++++++++ 5 files changed, 300 insertions(+) create mode 100644 lib/launchers/ocm.rb create mode 100644 lib/launchers/ocm_test.rb diff --git a/lib/bushslicer.rb b/lib/bushslicer.rb index f4b5b0c5bf..ed198a95b5 100644 --- a/lib/bushslicer.rb +++ b/lib/bushslicer.rb @@ -16,6 +16,7 @@ module BushSlicer autoload :OpenStack, "launchers/openstack" autoload :VSphere, "launchers/v_sphere" autoload :Packet, "launchers/packet" + autoload :OCM, "launchers/ocm" autoload :EnvironmentLauncher, "launchers/environment_launcher" autoload :PolarShift, "polarshift/autoload" diff --git a/lib/launchers/amz.rb b/lib/launchers/amz.rb index 13613bac70..63305b67e1 100755 --- a/lib/launchers/amz.rb +++ b/lib/launchers/amz.rb @@ -47,6 +47,8 @@ def initialize(access_key: nil, secret_key: nil, service_name: nil, region: nil) ) }) ) Aws.config.update( config[:config_opts].merge({region: region})) if region + + @account_id = awscred["aws_account_id"] end private def client_ec2 @@ -523,6 +525,11 @@ def secret_key ec2.client.config.credentials.secret_access_key end + # @return [String] + def account_id + return @account_id + end + # @return [Object] undefined def terminate_instance(instance) # we don't really have root permission to terminate, we'll just label it diff --git a/lib/launchers/cloud_helper.rb b/lib/launchers/cloud_helper.rb index 03167dde55..f4c307c603 100644 --- a/lib/launchers/cloud_helper.rb +++ b/lib/launchers/cloud_helper.rb @@ -39,6 +39,8 @@ def iaas_by_service(service_name) BushSlicer::Alicloud.new(service_name: service_name) when "packet" BushSlicer::Packet.new(service_name: service_name) + when "ocm" + BushSlicer::OCM.new(service_name: service_name) else raise "unknown service type " \ "#{conf[:services, service_name, :cloud_type]} for cloud " \ diff --git a/lib/launchers/ocm.rb b/lib/launchers/ocm.rb new file mode 100644 index 0000000000..7af9de4b72 --- /dev/null +++ b/lib/launchers/ocm.rb @@ -0,0 +1,198 @@ +#!/usr/bin/env ruby + +lib_path = File.expand_path(File.dirname(File.dirname(__FILE__))) +unless $LOAD_PATH.any? {|p| File.expand_path(p) == lib_path} + $LOAD_PATH.unshift(lib_path) +end + +require 'common' +require 'json' +require 'tmpdir' + +module BushSlicer + class OCM + include Common::Helper + + attr_reader :config + attr_reader :token, :token_file, :url, :region, :version, :nodes, :lifespan, :cloud, :cloud_opts, :multi_az + + def initialize(**options) + service_name = ENV['OCM_SERVICE_NAME'] || options[:service_name] || 'ocm' + @opts = default_opts(service_name)&.merge options + unless @opts + @opts = options + end + + # OCM token is mandatory + # it can be defined by token or by token_file + @token = ENV['OCM_TOKEN'] || @opts[:token] + @token_file = @opts[:token_file] + unless @token + if @token_file + token_file_path = expand_private_path(@token_file) + @token = File.read(token_file_path) + else + raise "You need to specify OCM token by 'token' or by 'token_file'" + end + end + + # region is mandatory + # in the future we can extend support for other clouds, e.g. GCP and ARO + @region = ENV['OCM_REGION'] || ENV['AWS_REGION'] || @opts[:region] + + # url defines the OCM environment (prod, integration or stage) + # currently, the url is ignored as many teams use the stage environment + @url = ENV['OCM_URL'] || @opts[:url] || 'https://api.stage.openshift.com' + + # openshift version is optional + @version = ENV['OCM_VERSION'] || @opts[:version] + + # number of worker nodes + # minimum is 2 + # default value is 4 + @nodes = ENV['OCM_NODES'] || @opts[:nodes] + + # lifespan in hours + # default value is 24 hours + @lifespan = ENV['OCM_LIFESPAN'] || @opts[:lifespan] + + # multi_az is optional + # default value is false + @multi_az = ENV['OCM_MULTI_AZ'] || @opts[:multi_az] + + # BYOC (Bring Your Own Cloud) + # you can refer to already defined cloud in config.yaml + # currently, only AWS is supported + @cloud = ENV['OCM_CLOUD'] || @opts[:cloud] + if @cloud + @cloud_opts = default_opts(@cloud) + unless @cloud_opts + raise "Cannot find cloud '#{cloud}' defined in '#{service_name}'" + end + end + end + + # @param service_name [String] the service name of this openstack instance + # to lookup in configuration + def default_opts(service_name) + return conf[:services, service_name.to_sym] + end + + def to_seconds(string) + regex_m = /^(\d+)\s*(m|min|minutes|mins)+$/ + regex_h = /^(\d+)\s*(h|hour|hours|hrs)+$/ + regex_d = /^(\d+)\s*(d|day|days)+$/ + regex_w = /^(\d+)\s*(w|week|weeks|wks)+$/ + if string.match(regex_m) + return string.match(regex_m)[1].to_i * 60 + elsif string.match(regex_h) + return string.match(regex_h)[1].to_i * 60 * 60 + elsif string.match(regex_d) + return string.match(regex_d)[1].to_i * 24 * 60 * 60 + elsif string.match(regex_w) + return string.match(regex_w)[1].to_i * 7 * 24 * 60 * 60 + else + raise "Cannot convert '#{string}' to seconds!" + end + end + + # create a json which specifies OSD cluster + # in the future we plan to move the logic into the script 'osd-provision.sh' + def generate_json(name) + json_data = { + "name" => name, + "managed" => true, + "multi_az" => false, + "byoc" => false + } + + if @multi_az + json_data.merge!({"multi_az" => @multi_az}) + end + + if @region + json_data.merge!({"region" => {"id" => @region}}) + end + + if @version + json_data.merge!({"version" => {"id" => "openshift-v#{@version}"}}) + end + + if @nodes + json_data.merge!({"nodes" => {"compute" => @nodes.to_i}}) + end + + if @lifespan + expiration = Time.now + to_seconds(@lifespan) + json_data.merge!({"expiration_timestamp" => expiration.strftime("%Y-%m-%dT%H:%M:%SZ")}) + end + + if @cloud_opts + case @cloud_opts[:cloud_type] + when "aws" + aws = Amz_EC2.new(service_name: @cloud) + json_data.merge!({"aws" => {"access_key_id":aws.access_key, "secret_access_key":aws.secret_key, "account_id":aws.account_id}}) + json_data.merge!({"byoc" => true}) + end + end + + return json_data.to_json + end + + # download the script 'osd-provision.sh' which takes care of the OSD installation/uninstallation + def download_osd_script + default_osd_script_url = "https://gitlab.cee.redhat.com/mk-bin-packing/mk-performance-tests/-/raw/master/scripts/osd-provision.sh?inline=false" + osd_script_url = ENV['OSD_SCRIPT_URL'] || default_osd_script_url + # we need another parent dir as the script downloads ocm tool to its parent + osd_scripts = "#{Dir.tmpdir}/osd-scripts" + %x( + rm -rf #{osd_scripts} && mkdir #{osd_scripts} && \ + curl -s #{osd_script_url} --output #{osd_scripts}/osd-provision.sh && \ + chmod a+x #{osd_scripts}/osd-provision.sh + ) + return "#{osd_scripts}/osd-provision.sh" + end + + def shell(cmd) + if !system(cmd) + raise "Error when executing '#{cmd}'" + end + end + + # create OSD cluster + def create_osd(name) + # cerate a temp file with ocm-token + ocm_token_file = Tempfile.new("ocm-token-file", Host.localhost.workdir) + File.open(ocm_token_file, "w") do |f| + f.write(@token) + end + # create a temp file with cluster specification + puts File.read(ocm_token_file.path) + ocm_json_file = Tempfile.new("ocm-json-file", Host.localhost.workdir) + File.open(ocm_json_file, "w") do |f| + data = generate_json(name) + f.write(data) + puts data + end + # now, download the script which will take care of the OSD cluster installation + osd_script = download_osd_script + shell("#{osd_script} --create --cloud-token-file #{ocm_token_file.path} -f #{ocm_json_file.path} --wait") + shell("#{osd_script} --get api_url -f #{ocm_json_file.path}") + shell("#{osd_script} --get credentials -f #{ocm_json_file.path}") + end + + # delete OSD cluster + def delete_osd(name) + # create a temp file with ocm-token + ocm_token_file = Tempfile.new("ocm-token-file", Host.localhost.workdir) + File.open(ocm_token_file, "w") do |f| + f.write(@token) + end + # now, download the script which will take care of the OSD cluster installation + osd_script = download_osd_script + system("#{osd_script} --delete --cloud-token-file #{ocm_token_file.path} -n #{name}") + end + + end + +end diff --git a/lib/launchers/ocm_test.rb b/lib/launchers/ocm_test.rb new file mode 100644 index 0000000000..4cc94cabed --- /dev/null +++ b/lib/launchers/ocm_test.rb @@ -0,0 +1,92 @@ +ENV['BUSHSLICER_PRIVATE_DIR'] = nil +ENV['OCM_NAME'] = nil +ENV['OCM_TOKEN'] = nil +ENV['OCM_URL'] = nil +ENV['OCM_REGION'] = nil +ENV['OCM_VERSION'] = nil +ENV['OCM_LIFESPAN'] = nil + +lib_path = File.expand_path(File.dirname(File.dirname(__FILE__))) +unless $LOAD_PATH.any? {|p| File.expand_path(p) == lib_path} + $LOAD_PATH.unshift(lib_path) +end + +require 'test/unit' +require_relative './ocm' + +class MyTest < Test::Unit::TestCase + def setup + + end + + # def teardown + # end + + def test_default_url + options = { :token => "abc" } + ocm = BushSlicer::OCM.new(options) + assert_equal('https://api.stage.openshift.com', ocm.url) + end + + def test_generating_json + options = { :token => "abc" } + ocm = BushSlicer::OCM.new(options) + json = ocm.generate_json('myosd4') + assert_equal('{"name":"myosd4","managed":true,"multi_az":false,"byoc":false}', json) + end + + def test_generating_json_with_region + options = { :token => "abc", :region => "us-east-1" } + ocm = BushSlicer::OCM.new(options) + json = ocm.generate_json('myosd4') + assert_equal('{"name":"myosd4","managed":true,"multi_az":false,"byoc":false,"region":{"id":"us-east-1"}}', json) + end + + def test_generating_json_with_version + options = { :token => "abc", :version => "4.6.1" } + ocm = BushSlicer::OCM.new(options) + json = ocm.generate_json('myosd4') + assert_equal('{"name":"myosd4","managed":true,"multi_az":false,"byoc":false,"version":{"id":"openshift-v4.6.1"}}', json) + end + + def test_generating_json_with_lifespan + options = { :token => "abc", :lifespan => "25h" } + ocm = BushSlicer::OCM.new(options) + json = ocm.generate_json('myosd4') + time = Time.now + 60 * 60 * 25 + year = time.strftime("%Y") + month = time.strftime("%m") + day = time.strftime("%d") + assert_match(/.*"expiration_timestamp":"#{year}-#{month}-#{day}T[0-9][0-9]:[0-9][0-9]:[0-9][0-9]Z".*/, json) + end + + def test_generating_json_with_nodes + options = { :token => "abc", :nodes => "8" } + ocm = BushSlicer::OCM.new(options) + json = ocm.generate_json('myosd4') + assert_equal('{"name":"myosd4","managed":true,"multi_az":false,"byoc":false,"nodes":{"compute":8}}', json) + end + + def test_downloading_osd_script + options = { :token => "abc" } + ocm = BushSlicer::OCM.new(options) + osd_script = ocm.download_osd_script + assert(File.exists?("/tmp/osd-scripts/osd-provision.sh"), "File 'osd-provision.sh' was not downloaded") + content = File.read("/tmp/osd-scripts/osd-provision.sh") + assert_match(/.*ocm.*/, content) + end + + def test_downloading_osd_script_envvar + # Tempfile is not working here (I have no idea why) + # custom_osd_script = Tempfile.new("custom-osd-script.sh", Host.localhost.workdir) + File.open("/tmp/custom-osd-script.sh", "w") { |f| f.write "ocm command --option value" } + ENV["OSD_SCRIPT_URL"] = "file:///tmp/custom-osd-script.sh" + options = { :token => "abc" } + ocm = BushSlicer::OCM.new(options) + osd_script = ocm.download_osd_script + assert(File.exists?("/tmp/osd-scripts/osd-provision.sh"), "File 'osd-provision.sh' was not downloaded") + content = File.read("/tmp/osd-scripts/osd-provision.sh") + assert_equal('ocm command --option value', content) + end + +end