[BuildImage] Download build image dependencies from s3 bucket (#2761)

* Retrieve dependencies from s3 bucket rather than from the open-internet

* Modify spec test to expect dependencies to be retrieved from an s3 bucket

* Get python dependency using https rather than aws cli

* Point s3 url for dependency download to production bucket

* Fix fabric manager install to install from a local package

* Update changelog to include changes to build-image

---------

Co-authored-by: Hanwen <68928867+hanwen-pcluste@users.noreply.github.com>

Author: hgreebe

CHANGELOG.md

@@ -8,6 +8,7 @@ This file is used to list changes made in each version of the AWS ParallelCluste
 
 **ENHANCEMENTS**
 - Add support for external Slurmdbd.
+- Allow build-image to be run in an isolated network.
 
 **CHANGES**
 - Upgrade Cinc Client to version to 18.4.12 from 18.2.7.

cookbooks/aws-parallelcluster-awsbatch/recipes/awsbatch_virtualenv.rb

@@ -28,5 +28,25 @@
   not_if { ::File.exist?("#{virtualenv_path}/bin/activate") }
 end
 
+remote_file "#{node['cluster']['base_dir']}/awsbatch-dependencies.tgz" do
+  source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/awsbatch-dependencies.tgz"
+  mode '0644'
+  retries 3
+  retry_delay 5
+  action :create_if_missing
+end
+
+bash 'pip install' do
+  user 'root'
+  group 'root'
+  cwd "#{node['cluster']['base_dir']}"
+  code <<-REQ
+    set -e
+    tar xzf awsbatch-dependencies.tgz
+    cd awsbatch
+    #{virtualenv_path}/bin/pip install * -f ./ --no-index
+    REQ
+end
+
 node.default['cluster']['awsbatch_virtualenv_path'] = virtualenv_path
 node_attributes "dump node attributes"

cookbooks/aws-parallelcluster-awsbatch/recipes/install.rb

@@ -43,6 +43,7 @@
       mkdir aws-parallelcluster-awsbatch-cli
       tar -xzf aws-parallelcluster.tgz --directory aws-parallelcluster-awsbatch-cli
       cd aws-parallelcluster-awsbatch-cli/*aws-parallelcluster-*
+
       #{node['cluster']['awsbatch_virtualenv_path']}/bin/pip install awsbatch-cli/
     CLI
   end

cookbooks/aws-parallelcluster-computefleet/recipes/install/custom_parallelcluster_node.rb

@@ -20,6 +20,14 @@
 # TODO: once the pyenv Chef resource supports installing packages from a path (e.g. `pip install .`), convert the
 # bash block to a recipe that uses the pyenv resource.
 
+remote_file "#{Chef::Config[:file_cache_path]}/node-dependencies.tgz" do
+  source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/node-dependencies.tgz"
+  mode '0644'
+  retries 3
+  retry_delay 5
+  action :create_if_missing
+end
+
 bash "install custom aws-parallelcluster-node" do
   cwd Chef::Config[:file_cache_path]
   code <<-NODE

cookbooks/aws-parallelcluster-computefleet/recipes/install/parallelcluster_node.rb

@@ -36,8 +36,7 @@
 if is_custom_node?
   include_recipe 'aws-parallelcluster-computefleet::custom_parallelcluster_node'
 else
-  pyenv_pip 'aws-parallelcluster-node' do
-    version node['cluster']['parallelcluster-node-version']
-    virtualenv virtualenv_path
+  execute "install official aws-parallelcluster-node" do
+    command "#{virtualenv_path}/bin/pip install aws-parallelcluster-node==#{node['cluster']['parallelcluster-node-version']}"
   end
 end

cookbooks/aws-parallelcluster-computefleet/spec/unit/recipes/node_spec.rb

@@ -36,10 +36,7 @@
         end
 
         it 'installs official node package' do
-          is_expected.to install_pyenv_pip('aws-parallelcluster-node').with(
-            version: node_version,
-            virtualenv: virtualenv_path
-          )
+          is_expected.to run_execute('install official aws-parallelcluster-node')
         end
       end
 

cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb

@@ -33,6 +33,26 @@
   not_if { ::File.exist?("#{virtualenv_path}/bin/activate") }
 end
 
+remote_file "#{node['cluster']['base_dir']}/cfn-dependencies.tgz" do
+  source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/cfn-dependencies.tgz"
+  mode '0644'
+  retries 3
+  retry_delay 5
+  action :create_if_missing
+end
+
+bash 'pip install' do
+  user 'root'
+  group 'root'
+  cwd "#{node['cluster']['base_dir']}"
+  code <<-REQ
+    set -e
+    tar xzf cfn-dependencies.tgz
+    cd cfn
+    #{virtualenv_path}/bin/pip install * -f ./ --no-index
+    REQ
+end
+
 cfnbootstrap_version = '2.0-28'
 cfnbootstrap_package = "aws-cfn-bootstrap-py3-#{cfnbootstrap_version}.tar.gz"
 

cookbooks/aws-parallelcluster-environment/spec/unit/recipes/cfn_bootstrap_spec.rb

@@ -30,6 +30,14 @@
           )
         end
 
+        it 'installs python packages' do
+          is_expected.to run_bash("pip install").with(
+            user: 'root',
+            group: 'root',
+            cwd: "#{node['cluster']['base_dir']}"
+          )
+        end
+
         it 'sets virtualenv path' do
           expect(node.default['cluster']['cfn_bootstrap_virtualenv_path']).to eq(virtualenv_path)
           is_expected.to write_node_attributes('dump node attributes')

cookbooks/aws-parallelcluster-platform/recipes/install/cookbook_virtualenv.rb

@@ -24,6 +24,25 @@
 activate_virtual_env cookbook_virtualenv_name do
   pyenv_path cookbook_virtualenv_path
   python_version cookbook_python_version
-  requirements_path "cookbook_virtualenv/requirements.txt"
   not_if { ::File.exist?("#{cookbook_virtualenv_path}/bin/activate") }
 end
+
+remote_file "#{node['cluster']['base_dir']}/cookbook-dependencies.tgz" do
+  source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/cookbook-dependencies.tgz"
+  mode '0644'
+  retries 3
+  retry_delay 5
+  action :create_if_missing
+end
+
+bash 'pip install' do
+  user 'root'
+  group 'root'
+  cwd "#{node['cluster']['base_dir']}"
+  code <<-REQ
+    set -e
+    tar xzf cookbook-dependencies.tgz
+    cd dependencies
+    #{virtualenv_path}/bin/pip install * -f ./ --no-index
+    REQ
+end

cookbooks/aws-parallelcluster-platform/recipes/install/cuda.rb

@@ -24,9 +24,9 @@
 cuda_complete_version = "#{cuda_version}.#{cuda_patch}"
 cuda_version_suffix = '535.104.05'
 cuda_arch = arm_instance? ? 'linux_sbsa' : 'linux'
-cuda_url = "https://developer.download.nvidia.com/compute/cuda/#{cuda_complete_version}/local_installers/cuda_#{cuda_complete_version}_#{cuda_version_suffix}_#{cuda_arch}.run"
+cuda_url = "#{node['cluster']['artifacts_s3_url']}/dependencies/cuda/cuda_#{cuda_complete_version}_#{cuda_version_suffix}_#{cuda_arch}.run"
 cuda_samples_version = '12.2'
-cuda_samples_url = "https://github.com/NVIDIA/cuda-samples/archive/refs/tags/v#{cuda_samples_version}.tar.gz"
+cuda_samples_url = "#{node['cluster']['artifacts_s3_url']}/dependencies/cuda/samples/v#{cuda_samples_version}.tar.gz"
 tmp_cuda_run = '/tmp/cuda.run'
 tmp_cuda_sample_archive = '/tmp/cuda-sample.tar.gz'