From 4817e126e7966015772243235a734047773c66a1 Mon Sep 17 00:00:00 2001 From: Hanwen Date: Wed, 28 Sep 2022 08:30:28 -0700 Subject: [PATCH 1/3] EFS Signed-off-by: Hanwen --- .../recipes/base.rb | 3 ++ .../recipes/efs.rb | 43 +++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 cookbooks/aws-parallelcluster-install/recipes/efs.rb diff --git a/cookbooks/aws-parallelcluster-install/recipes/base.rb b/cookbooks/aws-parallelcluster-install/recipes/base.rb index a289a831b6..939bb8c204 100644 --- a/cookbooks/aws-parallelcluster-install/recipes/base.rb +++ b/cookbooks/aws-parallelcluster-install/recipes/base.rb @@ -221,6 +221,9 @@ # Install FSx options include_recipe "aws-parallelcluster-install::lustre" +# Install EFS Utils +include_recipe "aws-parallelcluster-install::efs" + # Install the AWS cloudwatch agent include_recipe "aws-parallelcluster-install::cloudwatch_agent" diff --git a/cookbooks/aws-parallelcluster-install/recipes/efs.rb b/cookbooks/aws-parallelcluster-install/recipes/efs.rb new file mode 100644 index 0000000000..8994dae387 --- /dev/null +++ b/cookbooks/aws-parallelcluster-install/recipes/efs.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +# +# Cookbook:: aws-parallelcluster +# Recipe:: efs +# +# Copyright:: 2013-2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the +# License. A copy of the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "LICENSE.txt" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions and +# limitations under the License. + +case node['platform'] +when 'amazon' + package 'amazon-efs-utils' +when 'centos' + bash "install efs utils" do + cwd node['cluster']['sources_dir'] + code <<-EFSUTILSINSTALL + set -e + git clone https://github.com/aws/efs-utils + cd efs-utils + make rpm + yum -y install ./build/amazon-efs-utils*rpm + EFSUTILSINSTALL + end +when 'ubuntu' + bash "install efs utils" do + cwd node['cluster']['sources_dir'] + code <<-EFSUTILSINSTALL + set -e + git clone https://github.com/aws/efs-utils + cd efs-utils + ./build-deb.sh + apt-get -y install ./build/amazon-efs-utils*deb + EFSUTILSINSTALL + end +end From 2050c45292e9220f85878db408d3463cd08bf18a Mon Sep 17 00:00:00 2001 From: Hanwen Date: Thu, 6 Oct 2022 11:30:17 -0700 Subject: [PATCH 2/3] test Signed-off-by: Hanwen --- attributes/default.rb | 8 ++ .../recipes/efs.rb | 89 +++++++++++++++++-- 2 files changed, 90 insertions(+), 7 deletions(-) diff --git a/attributes/default.rb b/attributes/default.rb index 845ca2e59c..7378d3130e 100644 --- a/attributes/default.rb +++ b/attributes/default.rb @@ -262,6 +262,14 @@ default['cluster']['efa']['installer_url'] = "https://efa-installer.amazonaws.com/aws-efa-installer-#{node['cluster']['efa']['installer_version']}.tar.gz" default['cluster']['efa']['unsupported_aarch64_oses'] = %w(centos7) +# EFS Utils and its dependency stunnel +default['cluster']['efs_utils']['version'] = '1.33.3' +default['cluster']['efs_utils']['url'] = "https://github.com/aws/efs-utils/archive/v#{node['cluster']['efs_utils']['version']}.tar.gz" +default['cluster']['efs_utils']['sha256'] = '8bf9703d1dfc5cdd5e98539194cb9086873bdc2d938664356a94a727a78d40b6' +default['cluster']['stunnel']['version'] = '5.66' +default['cluster']['stunnel']['url'] = "https://www.stunnel.org/downloads/stunnel-#{node['cluster']['stunnel']['version']}.tar.gz" +default['cluster']['stunnel']['sha256'] = '558178704d1aa5f6883aac6cc5d6bbf2a5714c8a0d2e91da0392468cee9f579c' + # NICE DCV default['cluster']['dcv_port'] = 8443 default['cluster']['dcv']['installed'] = 'yes' diff --git a/cookbooks/aws-parallelcluster-install/recipes/efs.rb b/cookbooks/aws-parallelcluster-install/recipes/efs.rb index 8994dae387..2749eff4cc 100644 --- a/cookbooks/aws-parallelcluster-install/recipes/efs.rb +++ b/cookbooks/aws-parallelcluster-install/recipes/efs.rb @@ -15,16 +15,40 @@ # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions and # limitations under the License. +efs_utils_tarball = "#{node['cluster']['sources_dir']}/efs-utils-#{node['cluster']['efs_utils']['version']}.tar.gz" +stunnel_tarball = "#{node['cluster']['sources_dir']}/stunnel-#{node['cluster']['stunnel']['version']}.tar.gz" + +# Get EFS Utils tarball +remote_file efs_utils_tarball do + source node['cluster']['efs_utils']['url'] + mode '0644' + retries 3 + retry_delay 5 + not_if { ::File.exist?(efs_utils_tarball) } +end + +# Verify tarball +ruby_block "verify EFS Utils checksum" do + block do + require 'digest' + checksum = Digest::SHA256.file(efs_utils_tarball).hexdigest + raise "Downloaded EFS Utils package checksum #{checksum} does not match expected checksum #{node['cluster']['efs_utils']['sha256']}" if checksum != node['cluster']['efs_utils']['sha256'] + end +end + +# Install EFS Utils case node['platform'] -when 'amazon' - package 'amazon-efs-utils' -when 'centos' +when 'amazon', 'centos' bash "install efs utils" do cwd node['cluster']['sources_dir'] code <<-EFSUTILSINSTALL set -e - git clone https://github.com/aws/efs-utils - cd efs-utils + + # python3.4 or later is required + source #{node['cluster']['cookbook_virtualenv_path']}/bin/activate + + tar xf #{efs_utils_tarball} + cd efs-utils-#{node['cluster']['efs_utils']['version']} make rpm yum -y install ./build/amazon-efs-utils*rpm EFSUTILSINSTALL @@ -34,10 +58,61 @@ cwd node['cluster']['sources_dir'] code <<-EFSUTILSINSTALL set -e - git clone https://github.com/aws/efs-utils - cd efs-utils + + # python3.4 or later is required + source #{node['cluster']['cookbook_virtualenv_path']}/bin/activate + + tar xf #{efs_utils_tarball} + cd efs-utils-#{node['cluster']['efs_utils']['version']} ./build-deb.sh apt-get -y install ./build/amazon-efs-utils*deb EFSUTILSINSTALL end end + +# Get dependencies of stunnel +package "Install dependencies of stunnel" do + case node['platform'] + when 'amazon', 'centos' + package_name 'tcp_wrappers-devel' + when 'ubuntu' + package_name 'libwrap0-dev' + end + retries 3 + retry_delay 5 +end + +# Get stunnel tarball +remote_file stunnel_tarball do + source node['cluster']['stunnel']['url'] + mode '0644' + retries 3 + retry_delay 5 + not_if { ::File.exist?(stunnel_tarball) } +end + +# Verify tarball +ruby_block "verify stunnel checksum" do + block do + require 'digest' + checksum = Digest::SHA256.file(stunnel_tarball).hexdigest + raise "Downloaded stunnel package checksum #{checksum} does not match expected checksum #{node['cluster']['stunnel']['sha256']}" if checksum != node['cluster']['stunnel']['sha256'] + end +end + +bash "install stunnel" do + cwd node['cluster']['sources_dir'] + code <<-STUNNELINSTALL + set -e + + tar xvfz #{stunnel_tarball} + cd stunnel-#{node['cluster']['stunnel']['version']} + ./configure + make + if [[ -f /bin/stunnel ]]; then + rm /bin/stunnel + fi + make install + ln -s /usr/local/bin/stunnel /bin/stunnel + STUNNELINSTALL +end \ No newline at end of file From b60c74807500403e1d4952c852a7b4abb46f2c00 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 14 Dec 2022 04:03:23 +0000 Subject: [PATCH 3/3] Update kitchen-vagrant requirement from ~> 1.12.0 to ~> 1.13.0 Updates the requirements on [kitchen-vagrant](https://github.com/opscode/kitchen-vagrant) to permit the latest version. - [Release notes](https://github.com/opscode/kitchen-vagrant/releases) - [Changelog](https://github.com/test-kitchen/kitchen-vagrant/blob/main/CHANGELOG.md) - [Commits](https://github.com/opscode/kitchen-vagrant/compare/v1.12.0...v1.13.0) --- updated-dependencies: - dependency-name: kitchen-vagrant dependency-type: direct:development ... Signed-off-by: dependabot[bot] --- Gemfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile b/Gemfile index 506378c8bb..a23af58d92 100644 --- a/Gemfile +++ b/Gemfile @@ -13,7 +13,7 @@ end group :test do gem 'chefspec', '~> 9.3.0' - gem 'kitchen-vagrant', '~> 1.12.0' + gem 'kitchen-vagrant', '~> 1.13.0' gem 'safe_yaml', '~> 1.0.5' gem 'test-kitchen', '~> 3.0.0' end