Use systemd timers to trigger updates

Author: hgreebe

cookbooks/aws-parallelcluster-computefleet/files/check_update/check-update.timer

@@ -0,0 +1,11 @@
+[Unit]
+Description=Check file modification time every minute
+
+[Timer]
+AccuracySec=1s
+OnActiveSec=60sec
+OnUnitActiveSec=60sec
+Unit=check-update.service
+
+[Install]
+WantedBy=timers.target

cookbooks/aws-parallelcluster-computefleet/recipes/config.rb

@@ -12,3 +12,7 @@
 # limitations under the License.
 
 include_recipe 'aws-parallelcluster-computefleet::fleet_status'
+
+if ['ComputeFleet'].include?(node['cluster']['node_type'])
+  include_recipe 'aws-parallelcluster-computefleet::config_check_update_systemd_service'
+end

cookbooks/aws-parallelcluster-computefleet/recipes/config/config_check_update_systemd_service.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+#
+# Cookbook:: aws-parallelcluster-slurm
+# Recipe:: config_compute
+#
+# Copyright:: 2013-2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the
+# License. A copy of the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "LICENSE.txt" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions and
+# limitations under the License.
+
+template '/etc/systemd/system/check-update.service' do
+  source 'check_update/check-update.service.erb'
+  owner 'root'
+  group 'root'
+  mode '0644'
+end
+
+cookbook_file '/etc/systemd/system/check-update.timer' do
+  source 'check_update/check-update.timer'
+  owner 'root'
+  group 'root'
+  mode '0644'
+  action :create
+end
+
+file node['cluster']['shared_update_path'] do
+  content ''
+  owner 'root'
+  group 'root'
+  mode '0644'
+  action :create_if_missing
+end
+
+file node['cluster']['update_checkpoint'] do
+  content ''
+  owner 'root'
+  group 'root'
+  mode '0644'
+  action :create_if_missing
+end
+
+service 'check-update.timer' do
+  action [:enable, :start]
+end

cookbooks/aws-parallelcluster-computefleet/templates/check_update/check-update.service.erb

@@ -0,0 +1,21 @@
+[Unit]
+Description=Check for recent file modifications
+
+[Service]
+Type=oneshot
+TimeoutStartSec=30
+ExecStart=/bin/bash -c '\
+SHARED_FILE="<%= node['cluster']['shared_update_path'] %>"; \
+LOCAL_CHECKPOINT="<%= node['cluster']['update_checkpoint'] %>"; \
+\
+[ ! -f "$SHARED_FILE" ] && exit 0; \
+\
+CURRENT_UPDATE=$(cat "$SHARED_FILE") || exit 0; \
+LAST_APPLIED=$([ -f "$LOCAL_CHECKPOINT" ] && cat "$LOCAL_CHECKPOINT" || echo ""); \
+\
+if [ "$CURRENT_UPDATE" != "$LAST_APPLIED" ]; then \
+echo "$CURRENT_UPDATE" > "$LOCAL_CHECKPOINT" && <%= node['cluster']['scripts_dir'] %>/cfn-hup-update-action.sh; \
+fi'
+
+[Install]
+WantedBy=multi-user.target

cookbooks/aws-parallelcluster-environment/resources/cfn_hup_configuration.rb

@@ -68,7 +68,8 @@
       cfn_init_role: instance_role_name,
       # ComputeFleet specific variables
       update_hook_script_dir: node['cluster']['scripts_dir'],
-      node_bootstrap_timeout: node['cluster']['compute_node_bootstrap_timeout'] || node['cluster']['Timeout']
+      node_bootstrap_timeout: node['cluster']['compute_node_bootstrap_timeout'] || node['cluster']['Timeout'],
+      update_dir: node['cluster']['shared_update_path']
     )
   end
 end
@@ -94,7 +95,8 @@
       mode '0700'
       variables(
         monitor_shared_dir: monitor_shared_dir,
-        launch_template_resource_id: node['cluster']['launch_template_id']
+        launch_template_resource_id: node['cluster']['launch_template_id'],
+        update_dir: node['cluster']['shared_update_path']
       )
     end
   end

cookbooks/aws-parallelcluster-environment/templates/cfn_hup_configuration/cfn-hook-update.conf.erb

@@ -3,9 +3,6 @@ triggers=post.update
 <% case node['cluster']['node_type'] -%>
 <% when 'HeadNode', 'LoginNode' -%>
 path=Resources.<%= @launch_template_resource_id %>.Metadata.AWS::CloudFormation::Init
-action=PATH=/usr/local/bin:/bin:/usr/bin:/opt/aws/bin; . /etc/parallelcluster/pcluster_cookbook_environment.sh; $CFN_BOOTSTRAP_VIRTUALENV_PATH/cfn-init -v --stack <%= @stack_id %> --resource <%= @launch_template_resource_id %> --configsets update --region <%= @region %> --url <%= @cloudformation_url %> --role <%= @cfn_init_role %>
-<% when 'ComputeFleet' -%>
-path=Resources.<%= @launch_template_resource_id %>
-action=timeout <%= @node_bootstrap_timeout %> <%= @update_hook_script_dir %>/cfn-hup-update-action.sh
+action=PATH=/usr/local/bin:/bin:/usr/bin:/opt/aws/bin;. /etc/parallelcluster/pcluster_cookbook_environment.sh; $CFN_BOOTSTRAP_VIRTUALENV_PATH/cfn-init -v --stack <%= @stack_id %> --resource <%= @launch_template_resource_id %> --configsets update --region <%= @region %> --url <%= @cloudformation_url %> --role <%= @cfn_init_role %>
 <% end %>
 runas=root

cookbooks/aws-parallelcluster-platform/templates/supervisord/parallelcluster_supervisord.conf.erb

@@ -1,7 +1,7 @@
 # Generated by Chef for AWS ParallelCluster <%= node['cluster']['node_type'] -%>
 # Local modifications could be overwritten.
-<%# HeadNode, ComputeFleet, LoginNode -%>
-<% if @cfnhup_enabled -%>
+<% case node['cluster']['node_type'] -%>
+<% when 'HeadNode', 'LoginNode' -%>
 [program:cfn-hup]
 command = <%= node['cluster']['scripts_dir']%>/cfn-hup-runner.sh
 autorestart = true

cookbooks/aws-parallelcluster-shared/attributes/cluster.rb

@@ -9,6 +9,10 @@
 default['cluster']['log_base_dir'] = '/var/log/parallelcluster'
 default['cluster']['etc_dir'] = '/etc/parallelcluster'
 
+# Shared file used to manage inplace updates
+default['cluster']['shared_update_path'] = "#{node['cluster']['shared_dir']}/check_update"
+default['cluster']['update_checkpoint'] = "#{node['cluster']['scripts_dir']}/update_checkpoint"
+
 # Slurm_plugin_dir is used by slurm cookbook and custom_actions recipe
 default['cluster']['slurm_plugin_dir'] = "#{node['cluster']['etc_dir']}/slurm_plugin"
 

cookbooks/aws-parallelcluster-slurm/libraries/helpers.rb

@@ -173,7 +173,7 @@ def wait_cluster_ready
               " --config-version #{node['cluster']['cluster_config_version']}" \
               " --region #{node['cluster']['region']}"
     timeout 30
-    retries 10
+    retries 20
     retry_delay 90
   end
 end

cookbooks/aws-parallelcluster-slurm/recipes/update/update_head_node.rb

@@ -20,6 +20,14 @@
   not_if { ::File.exist?(node['cluster']['previous_cluster_config_path']) && !are_queues_updated? && !are_bulk_custom_slurm_settings_updated? }
 end
 
+# Write the new config version to shared storage to signal compute nodes to update
+file node['cluster']['shared_update_path'] do
+  content node['cluster']['cluster_config_version']
+  owner 'root'
+  group 'root'
+  mode '0644'
+end
+
 ruby_block "update_shared_storages" do
   block do
     run_context.include_recipe 'aws-parallelcluster-environment::update_shared_storages'