From 2308bdbea3f7bbbfd884ed544b5d92f0b7cde754 Mon Sep 17 00:00:00 2001 From: Sebastian Cruz Date: Thu, 26 May 2016 13:12:11 +0100 Subject: [PATCH 01/12] Updated README with needed IAM policy --- load-balancing/elb/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/load-balancing/elb/README.md b/load-balancing/elb/README.md index 69938c4..88b4510 100644 --- a/load-balancing/elb/README.md +++ b/load-balancing/elb/README.md @@ -29,6 +29,8 @@ in the CLI's user guide. autoscaling:EnterStandby autoscaling:ExitStandby autoscaling:UpdateAutoScalingGroup + autoscaling:SuspendProcesses + autoscaling:ResumeProcesses ``` Note: the AWS CodeDeploy Agent requires that an instance profile be attached to all instances that From e54045339199f52630a7ed1bf4e79d0ab5965bad Mon Sep 17 00:00:00 2001 From: Sebastian Cruz Date: Fri, 27 May 2016 15:59:05 +0100 Subject: [PATCH 02/12] Revamped flagfile usage with extra functions preparing for extra functionality --- load-balancing/elb/common_functions.sh | 56 +++++++++++++++++++++++--- 1 file changed, 50 insertions(+), 6 deletions(-) diff --git a/load-balancing/elb/common_functions.sh b/load-balancing/elb/common_functions.sh index 7ee6dbe..40bb008 100644 --- a/load-balancing/elb/common_functions.sh +++ b/load-balancing/elb/common_functions.sh @@ -34,6 +34,9 @@ WAITER_INTERVAL=1 # AutoScaling Standby features at minimum require this version to work. MIN_CLI_VERSION='1.3.25' +# Create a flagfile for each deployment +FLAGFILE="/tmp/asg_codedeploy_flags-$DEPLOYMENT_GROUP_ID-$DEPLOYMENT_ID" + # Usage: get_instance_region # # Writes to STDOUT the AWS region as known by the local instance. @@ -49,6 +52,47 @@ get_instance_region() { AWS_CLI="aws --region $(get_instance_region)" +# Usage: set_flag +# +# Writes to FLAGFILE +set_flag() { + if echo "$1=true" >> $FLAGFILE; then + msg "$1 flag written to $FLAGFILE" + return 0 + else + error_exit "$1 flag NOT written to $FLAGFILE" + fi +} + +# Usage: get_flag +# +# Checks if is true in FLAGFILE +get_flag() { + if [ -e $FLAGFILE ]; then + if grep "$1=true" $FLAGFILE > /dev/null; then + msg "$1 flag found in $FLAGFILE" + return 0 + else + msg "$1 flag NOT found in $FLAGFILE" + return 1 + fi + else + msg "Flagfile $FLAGFILE doesn't exist." + return 1 + fi +} + +# Usage: remove_flagfile +# +# Removes FLAGFILE +remove_flagfile() { + if rm -f $FLAGFILE; then + msg "Successfully removed flagfile $FLAGFILE" + else + error_exit "Failed to remove flagfile $FLAGFILE." + fi +} + # Usage: autoscaling_group_name # # Prints to STDOUT the name of the AutoScaling group this instance is a part of and returns 0. If @@ -123,9 +167,9 @@ autoscaling_enter_standby() { msg "Failed to reduce ASG ${asg_name}'s minimum size to $new_min. Cannot put this instance into Standby." return 1 else - msg "ASG ${asg_name}'s minimum size has been decremented, creating flag file /tmp/asgmindecremented" - # Create a "flag" file to denote that the ASG min has been decremented - touch /tmp/asgmindecremented + msg "ASG ${asg_name}'s minimum size has been decremented, creating flag in file $FLAGFILE" + # Create a "flag" denote that the ASG min has been decremented + set_flag asgmindecremented fi fi @@ -192,7 +236,7 @@ autoscaling_exit_standby() { return 1 fi - if [ -a /tmp/asgmindecremented ]; then + if get_flag asgmindecremented; then local min_desired=$($AWS_CLI autoscaling describe-auto-scaling-groups \ --auto-scaling-group-name "${asg_name}" \ --query 'AutoScalingGroups[0].[MinSize, DesiredCapacity]' \ @@ -207,16 +251,16 @@ autoscaling_exit_standby() { --min-size $new_min) if [ $? != 0 ]; then msg "Failed to increase ASG ${asg_name}'s minimum size to $new_min." + remove_flagfile return 1 else msg "Successfully incremented ASG ${asg_name}'s minimum size" - msg "Removing /tmp/asgmindecremented flag file" - rm -f /tmp/asgmindecremented fi else msg "Auto scaling group was not decremented previously, not incrementing min value" fi + remove_flagfile return 0 } From d338710fe5fc479054c4115d012304384afc9d76 Mon Sep 17 00:00:00 2001 From: Sebastian Cruz Date: Fri, 27 May 2016 16:01:09 +0100 Subject: [PATCH 03/12] Moved final message with stats to a function to be able to call it from different places --- load-balancing/elb/common_functions.sh | 12 ++++++++++++ load-balancing/elb/deregister_from_elb.sh | 8 ++------ load-balancing/elb/register_with_elb.sh | 8 ++------ 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/load-balancing/elb/common_functions.sh b/load-balancing/elb/common_functions.sh index 40bb008..6050249 100644 --- a/load-balancing/elb/common_functions.sh +++ b/load-balancing/elb/common_functions.sh @@ -93,6 +93,18 @@ remove_flagfile() { fi } +# Usage: finish_msg +# +# Prints some finishing statistics +finish_msg() { + msg "Finished $(basename $0) at $(/bin/date "+%F %T")" + + end_sec=$(/bin/date +%s.%N) + elapsed_seconds=$(echo "$end_sec - $start_sec" | /usr/bin/bc) + + msg "Elapsed time: $elapsed_seconds" +} + # Usage: autoscaling_group_name # # Prints to STDOUT the name of the AutoScaling group this instance is a part of and returns 0. If diff --git a/load-balancing/elb/deregister_from_elb.sh b/load-balancing/elb/deregister_from_elb.sh index c497e57..70f5407 100755 --- a/load-balancing/elb/deregister_from_elb.sh +++ b/load-balancing/elb/deregister_from_elb.sh @@ -44,6 +44,7 @@ if [ $? == 0 -a -n "${asg}" ]; then error_exit "Failed to move instance into standby" else msg "Instance is in standby" + finish_msg exit 0 fi fi @@ -81,9 +82,4 @@ for elb in $ELB_LIST; do fi done -msg "Finished $(basename $0) at $(/bin/date "+%F %T")" - -end_sec=$(/bin/date +%s.%N) -elapsed_seconds=$(echo "$end_sec - $start_sec" | /usr/bin/bc) - -msg "Elapsed time: $elapsed_seconds" +finish_msg diff --git a/load-balancing/elb/register_with_elb.sh b/load-balancing/elb/register_with_elb.sh index fc4e1f2..0359f53 100755 --- a/load-balancing/elb/register_with_elb.sh +++ b/load-balancing/elb/register_with_elb.sh @@ -44,6 +44,7 @@ if [ $? == 0 -a -n "${asg}" ]; then error_exit "Failed to move instance out of standby" else msg "Instance is no longer in Standby" + finish_msg exit 0 fi fi @@ -81,9 +82,4 @@ for elb in $ELB_LIST; do fi done -msg "Finished $(basename $0) at $(/bin/date "+%F %T")" - -end_sec=$(/bin/date +%s.%N) -elapsed_seconds=$(echo "$end_sec - $start_sec" | /usr/bin/bc) - -msg "Elapsed time: $elapsed_seconds" +finish_msg From fbd80c744bce49fc3efb7b686699f813b7964c85 Mon Sep 17 00:00:00 2001 From: Sebastian Cruz Date: Fri, 27 May 2016 19:29:31 +0100 Subject: [PATCH 04/12] Make sure (AZRebalance AlarmNotification ScheduledActions ReplaceUnhealthy) are suspended while deploying and resume only the ones suspended by the script. Also abort if Launch process is disabled as it will not allow to return and instance from StandBy --- load-balancing/elb/common_functions.sh | 80 ++++++++++++++++++++++++-- 1 file changed, 76 insertions(+), 4 deletions(-) diff --git a/load-balancing/elb/common_functions.sh b/load-balancing/elb/common_functions.sh index 6050249..abfedbd 100644 --- a/load-balancing/elb/common_functions.sh +++ b/load-balancing/elb/common_functions.sh @@ -57,7 +57,7 @@ AWS_CLI="aws --region $(get_instance_region)" # Writes to FLAGFILE set_flag() { if echo "$1=true" >> $FLAGFILE; then - msg "$1 flag written to $FLAGFILE" + #msg "$1 flag written to $FLAGFILE" return 0 else error_exit "$1 flag NOT written to $FLAGFILE" @@ -70,18 +70,80 @@ set_flag() { get_flag() { if [ -e $FLAGFILE ]; then if grep "$1=true" $FLAGFILE > /dev/null; then - msg "$1 flag found in $FLAGFILE" + #msg "$1 flag found in $FLAGFILE" return 0 else - msg "$1 flag NOT found in $FLAGFILE" + #msg "$1 flag NOT found in $FLAGFILE" return 1 fi else - msg "Flagfile $FLAGFILE doesn't exist." + # FLAGFILE doesn't exist return 1 fi } +# Usage: check_suspended_processes +# +# Checks processes suspended on the ASG before beginning and store them in +# the FLAGFILE to avoid resuming afterwards. Also abort if Launch process +# is suspended. +check_suspended_processes() { + # Get suspended processes in an array + local suspended=($($AWS_CLI autoscaling describe-auto-scaling-groups \ + --auto-scaling-group-name "${asg_name}" \ + --query 'AutoScalingGroups[].SuspendedProcesses' \ + --output text | awk '{printf $1" "}')) + + msg "This processes were suspended on the ASG before starting ${suspended[*]}" + + # If "Launch" process is suspended abort because we will not be able to recover from StandBy + if [[ "${suspended[@]}" =~ "Launch" ]]; then + error_exit "Launch process of AutoScaling is suspended which will not allow us to recover the instance from StandBy. Aborting." + fi + + for process in ${suspended[@]}; do + set_flag $process + done +} + +# Usage: suspend_processes +# +# Suspend processes known to cause problems during deployments. +# The API call is idempotent so it doesn't matter if any were previously suspended. +suspend_processes() { + local -a processes=(AZRebalance AlarmNotification ScheduledActions ReplaceUnhealthy) + + msg "Suspending ${processes[*]} processes" + $AWS_CLI autoscaling suspend-processes \ + --auto-scaling-group-name "${asg_name}" \ + --scaling-processes ${processes[@]} + if [ $? != 0 ]; then + error_exit "Failed to suspend ${processes[*]} processes for ASG ${asg_name}. Aborting as this may cause issues." + fi +} + +# Usage: resume_processes +# +# Resume processes suspended, except for the one suspended before deployment. +resume_processes() { + local -a processes=(AZRebalance AlarmNotification ScheduledActions ReplaceUnhealthy) + local -a to_resume + + for p in ${processes[@]}; do + if ! get_flag "$p"; then + to_resume=("${to_resume[@]}" "$p") + fi + done + + msg "Resuming ${to_resume[*]} processes" + $AWS_CLI autoscaling resume-processes \ + --auto-scaling-group-name "${asg_name}" \ + --scaling-processes ${to_resume[@]} + if [ $? != 0 ]; then + error_exit "Failed to resume ${to_resume[*]} processes for ASG ${asg_name}. Aborting as this may cause issues." + fi +} + # Usage: remove_flagfile # # Removes FLAGFILE @@ -157,6 +219,12 @@ autoscaling_enter_standby() { return 0 fi + msg "Checking ASG ${asg_name} suspended processes" + check_suspended_processes + + # Suspend troublesome processes while deploying + suspend_processes + msg "Checking to see if ASG ${asg_name} will let us decrease desired capacity" local min_desired=$($AWS_CLI autoscaling describe-auto-scaling-groups \ --auto-scaling-group-name "${asg_name}" \ @@ -272,6 +340,10 @@ autoscaling_exit_standby() { msg "Auto scaling group was not decremented previously, not incrementing min value" fi + # Resume processes, except for the ones suspended before deployment + resume_processes + + # Clean up the FLAGFILE remove_flagfile return 0 } From 35818792526f1eced633c4604f54a12873471efb Mon Sep 17 00:00:00 2001 From: Sebastian Cruz Date: Mon, 30 May 2016 09:57:17 +0100 Subject: [PATCH 05/12] Update README.md - Added notice about suspended processes - Fixed some Markdown style issues - Fixed missing link to documentation --- load-balancing/elb/README.md | 49 ++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/load-balancing/elb/README.md b/load-balancing/elb/README.md index 88b4510..46da444 100644 --- a/load-balancing/elb/README.md +++ b/load-balancing/elb/README.md @@ -9,6 +9,8 @@ your instances are a part of, set the scripts in the appropriate lifecycle event will take care of deregistering the instance, waiting for connection draining, and re-registering after the deployment finishes. +*Please note that this scripts will suspend some AutoScaling processes (AZRebalance, AlarmNotification, ScheduledActions and ReplaceUnhealthy) while deploying, in order to avoid wrong interactions.* + ## Requirements The register and deregister scripts have a couple of dependencies in order to properly interact with @@ -19,36 +21,33 @@ AutoScaling's Standby feature, the CLI must be at least version 1.3.25. If you have Python and PIP already installed, the CLI can simply be installed with `pip install awscli`. Otherwise, follow the [installation instructions](http://docs.aws.amazon.com/cli/latest/userguide/installing.html) in the CLI's user guide. -1. An instance profile with a policy that allows, at minimum, the following actions: - -``` - elasticloadbalancing:Describe* - elasticloadbalancing:DeregisterInstancesFromLoadBalancer - elasticloadbalancing:RegisterInstancesWithLoadBalancer - autoscaling:Describe* - autoscaling:EnterStandby - autoscaling:ExitStandby - autoscaling:UpdateAutoScalingGroup - autoscaling:SuspendProcesses - autoscaling:ResumeProcesses -``` - -Note: the AWS CodeDeploy Agent requires that an instance profile be attached to all instances that -are to participate in AWS CodeDeploy deployments. For more information on creating an instance -profile for AWS CodeDeploy, see the [Create an IAM Instance Profile for Your Amazon EC2 Instances]() -topic in the documentation. -1. All instances are assumed to already have the AWS CodeDeploy Agent installed. +2. An instance profile with a policy that allows, at minimum, the following actions: + + elasticloadbalancing:Describe* + elasticloadbalancing:DeregisterInstancesFromLoadBalancer + elasticloadbalancing:RegisterInstancesWithLoadBalancer + autoscaling:Describe* + autoscaling:EnterStandby + autoscaling:ExitStandby + autoscaling:UpdateAutoScalingGroup + autoscaling:SuspendProcesses + autoscaling:ResumeProcesses + + **Note**: the AWS CodeDeploy Agent requires that an instance profile be attached to all instances that + are to participate in AWS CodeDeploy deployments. For more information on creating an instance + profile for AWS CodeDeploy, see the [Create an IAM Instance Profile for Your Amazon EC2 Instances](http://docs.aws.amazon.com/codedeploy/latest/userguide/how-to-create-iam-instance-profile.html) + topic in the documentation. +3. All instances are assumed to already have the AWS CodeDeploy Agent installed. ## Installing the Scripts To use these scripts in your own application: 1. Install the AWS CLI on all your instances. -1. Update the policies on the EC2 instance profile to allow the above actions. -1. Copy the `.sh` files in this directory into your application source. -1. Edit your application's `appspec.yml` to run `deregister_from_elb.sh` on the ApplicationStop event, +2. Update the policies on the EC2 instance profile to allow the above actions. +3. Copy the `.sh` files in this directory into your application source. +4. Edit your application's `appspec.yml` to run `deregister_from_elb.sh` on the ApplicationStop event, and `register_with_elb.sh` on the ApplicationStart event. -1. Edit `common_functions.sh` to set `ELB_LIST` to contain the name(s) of the Elastic Load +5. Edit `common_functions.sh` to set `ELB_LIST` to contain the name(s) of the Elastic Load Balancer(s) your deployment group is a part of. Make sure the entries in ELB_LIST are separated by space. -1. Deploy! - +6. Deploy! From 9cd22dbeb14fa9cbf5bdca97189fd1646e607a16 Mon Sep 17 00:00:00 2001 From: Sebastian Cruz Date: Mon, 30 May 2016 15:38:43 +0100 Subject: [PATCH 06/12] Fixed messaging if no previously suspended processes were found. --- load-balancing/elb/common_functions.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/load-balancing/elb/common_functions.sh b/load-balancing/elb/common_functions.sh index abfedbd..ecc08ec 100644 --- a/load-balancing/elb/common_functions.sh +++ b/load-balancing/elb/common_functions.sh @@ -94,11 +94,15 @@ check_suspended_processes() { --query 'AutoScalingGroups[].SuspendedProcesses' \ --output text | awk '{printf $1" "}')) - msg "This processes were suspended on the ASG before starting ${suspended[*]}" + if [ ${#suspended[@]} -eq 0 ]; then + msg "No processes were suspended on the ASG before starting." + else + msg "This processes were suspended on the ASG before starting: ${suspended[*]}" + fi # If "Launch" process is suspended abort because we will not be able to recover from StandBy if [[ "${suspended[@]}" =~ "Launch" ]]; then - error_exit "Launch process of AutoScaling is suspended which will not allow us to recover the instance from StandBy. Aborting." + error_exit "'Launch' process of AutoScaling is suspended which will not allow us to recover the instance from StandBy. Aborting." fi for process in ${suspended[@]}; do From f005de422687e9c79987a6eae33543a239478084 Mon Sep 17 00:00:00 2001 From: Sebastian Cruz Date: Tue, 14 Jun 2016 17:18:45 +0100 Subject: [PATCH 07/12] Added info about ELB_LIST options of _all_ and _any_ as well as style revamp. --- load-balancing/elb/README.md | 34 ++++++++++------------------------ 1 file changed, 10 insertions(+), 24 deletions(-) diff --git a/load-balancing/elb/README.md b/load-balancing/elb/README.md index 46da444..20432eb 100644 --- a/load-balancing/elb/README.md +++ b/load-balancing/elb/README.md @@ -1,26 +1,15 @@ # ELB and ASG lifecycle event scripts -Often when running a web service, you'll have your instances behind a load balancer. But when -deploying new code to these instances, you don't want the load balancer to continue sending customer -traffic to an instance while the deployment is in progress. Lifecycle event scripts give you the -ability to integrate your AWS CodeDeploy deployments with instances that are behind an Elastic Load -Balancer or in an Auto Scaling group. Simply set the name (or names) of the Elastic Load Balancer -your instances are a part of, set the scripts in the appropriate lifecycle events, and the scripts -will take care of deregistering the instance, waiting for connection draining, and re-registering -after the deployment finishes. +Often when running a web service, you'll have your instances behind a load balancer. But when deploying new code to these instances, you don't want the load balancer to continue sending customer traffic to an instance while the deployment is in progress. Lifecycle event scripts give you the ability to integrate your AWS CodeDeploy deployments with instances that are behind an Elastic Load Balancer or in an Auto Scaling group. Simply set the name (or names) of the Elastic Load Balancer your instances are a part of, set the scripts in the appropriate lifecycle events, and the scripts will take care of deregistering the instance, waiting for connection draining, and re-registering after the deployment finishes. *Please note that this scripts will suspend some AutoScaling processes (AZRebalance, AlarmNotification, ScheduledActions and ReplaceUnhealthy) while deploying, in order to avoid wrong interactions.* ## Requirements -The register and deregister scripts have a couple of dependencies in order to properly interact with -Elastic Load Balancing and AutoScaling: +The register and deregister scripts have a couple of dependencies in order to properly interact with Elastic Load Balancing and AutoScaling: + +1. The [AWS CLI](http://aws.amazon.com/cli/). In order to take advantage of AutoScaling's Standby feature, the CLI must be at least version 1.3.25. If you have Python and PIP already installed, the CLI can simply be installed with `pip install awscli`. Otherwise, follow the [installation instructions](http://docs.aws.amazon.com/cli/latest/userguide/installing.html) in the CLI's user guide. -1. The [AWS CLI](http://aws.amazon.com/cli/). In order to take advantage of -AutoScaling's Standby feature, the CLI must be at least version 1.3.25. If you -have Python and PIP already installed, the CLI can simply be installed with `pip -install awscli`. Otherwise, follow the [installation instructions](http://docs.aws.amazon.com/cli/latest/userguide/installing.html) -in the CLI's user guide. 2. An instance profile with a policy that allows, at minimum, the following actions: elasticloadbalancing:Describe* @@ -33,10 +22,8 @@ in the CLI's user guide. autoscaling:SuspendProcesses autoscaling:ResumeProcesses - **Note**: the AWS CodeDeploy Agent requires that an instance profile be attached to all instances that - are to participate in AWS CodeDeploy deployments. For more information on creating an instance - profile for AWS CodeDeploy, see the [Create an IAM Instance Profile for Your Amazon EC2 Instances](http://docs.aws.amazon.com/codedeploy/latest/userguide/how-to-create-iam-instance-profile.html) - topic in the documentation. + **Note**: the AWS CodeDeploy Agent requires that an instance profile be attached to all instances that are to participate in AWS CodeDeploy deployments. For more information on creating an instance profile for AWS CodeDeploy, see the [Create an IAM Instance Profile for Your Amazon EC2 Instances](http://docs.aws.amazon.com/codedeploy/latest/userguide/how-to-create-iam-instance-profile.html) topic in the documentation. + 3. All instances are assumed to already have the AWS CodeDeploy Agent installed. ## Installing the Scripts @@ -46,8 +33,7 @@ To use these scripts in your own application: 1. Install the AWS CLI on all your instances. 2. Update the policies on the EC2 instance profile to allow the above actions. 3. Copy the `.sh` files in this directory into your application source. -4. Edit your application's `appspec.yml` to run `deregister_from_elb.sh` on the ApplicationStop event, -and `register_with_elb.sh` on the ApplicationStart event. -5. Edit `common_functions.sh` to set `ELB_LIST` to contain the name(s) of the Elastic Load -Balancer(s) your deployment group is a part of. Make sure the entries in ELB_LIST are separated by space. -6. Deploy! +4. Edit your application's `appspec.yml` to run `deregister_from_elb.sh` on the ApplicationStop event, and `register_with_elb.sh` on the ApplicationStart event. +5. If your instance is not in an Auto Scaling Group, edit `common_functions.sh` to set `ELB_LIST` to contain the name(s) of the Elastic Load Balancer(s) your deployment group is a part of. Make sure the entries in ELB_LIST are separated by space. +Alternatively, you can set `ELB_LIST` to `_all_` to automatically use all load balancers the instance is registered to, or `_any_` to get the same behaviour as `_all_` but without failing your deployments if the instance is not part of any ASG or ELB. This is more flexible in heterogeneous tag-based Deployment Groups. +6. Deploy! \ No newline at end of file From d30c8ffdc93001ffb9eb631611f9a0fc4e946da1 Mon Sep 17 00:00:00 2001 From: Sebastian Cruz Date: Tue, 14 Jun 2016 17:35:47 +0100 Subject: [PATCH 08/12] - Improved set_flag/get_flag functions to support storing/getting arbitrary values. - Removed dependency on bc by using awk instead. - Removed ASG checks from reset_waiter_timeout as it's only considered for ELBs. - Further improved the query string on reset_waiter_timeout function (and tested it working on Ubuntu Trusty) --- load-balancing/elb/common_functions.sh | 83 ++++++++++++-------------- 1 file changed, 37 insertions(+), 46 deletions(-) diff --git a/load-balancing/elb/common_functions.sh b/load-balancing/elb/common_functions.sh index ecc08ec..c33061b 100644 --- a/load-balancing/elb/common_functions.sh +++ b/load-balancing/elb/common_functions.sh @@ -14,7 +14,10 @@ # permissions and limitations under the License. # ELB_LIST defines which Elastic Load Balancers this instance should be part of. -# The elements in ELB_LIST should be seperated by space. +# The elements in ELB_LIST should be separated by space. Safe default is "". +# Set to "_all_" to automatically find all load balancers the instance is registered to. +# Set to "_any_" will work as "_all_" but will not fail if instance is not attached to +# any ASG or ELB, giving flexibility. ELB_LIST="" # Under normal circumstances, you shouldn't need to change anything below this line. @@ -52,30 +55,25 @@ get_instance_region() { AWS_CLI="aws --region $(get_instance_region)" -# Usage: set_flag +# Usage: set_flag # -# Writes to FLAGFILE +# Writes = to FLAGFILE set_flag() { - if echo "$1=true" >> $FLAGFILE; then - #msg "$1 flag written to $FLAGFILE" + if echo "$1=$2" >> $FLAGFILE; then return 0 else - error_exit "$1 flag NOT written to $FLAGFILE" + error_exit "Unable to write flag \"$1=$2\" to $FLAGFILE" fi } # Usage: get_flag # -# Checks if is true in FLAGFILE +# Checks for in FLAGFILE. Echoes it's value and returns 0 on success or non-zero if it fails to read the file. get_flag() { - if [ -e $FLAGFILE ]; then - if grep "$1=true" $FLAGFILE > /dev/null; then - #msg "$1 flag found in $FLAGFILE" - return 0 - else - #msg "$1 flag NOT found in $FLAGFILE" - return 1 - fi + if [ -r $FLAGFILE ]; then + local result=$(awk -F= -v flag="$1" '{if ( $1 == flag ) {print $2}}' $FLAGFILE) + echo "${result}" + return 0 else # FLAGFILE doesn't exist return 1 @@ -100,13 +98,13 @@ check_suspended_processes() { msg "This processes were suspended on the ASG before starting: ${suspended[*]}" fi - # If "Launch" process is suspended abort because we will not be able to recover from StandBy + # If "Launch" process is suspended abort because we will not be able to recover from StandBy. Note the "[[ ... =~" bashism. if [[ "${suspended[@]}" =~ "Launch" ]]; then error_exit "'Launch' process of AutoScaling is suspended which will not allow us to recover the instance from StandBy. Aborting." fi for process in ${suspended[@]}; do - set_flag $process + set_flag "$process" "true" done } @@ -134,7 +132,9 @@ resume_processes() { local -a to_resume for p in ${processes[@]}; do - if ! get_flag "$p"; then + if ! local tmp_flag_value=$(get_flag "$p"); then + error_exit "$FLAGFILE doesn't exist or is unreadable" + elif [ ! "$tmp_flag_value" = "true" ] ; then to_resume=("${to_resume[@]}" "$p") fi done @@ -150,12 +150,13 @@ resume_processes() { # Usage: remove_flagfile # -# Removes FLAGFILE +# Removes FLAGFILE. Returns non-zero if failure. remove_flagfile() { - if rm -f $FLAGFILE; then + if rm $FLAGFILE; then msg "Successfully removed flagfile $FLAGFILE" + return 0 else - error_exit "Failed to remove flagfile $FLAGFILE." + msg "WARNING: Failed to remove flagfile $FLAGFILE." fi } @@ -166,7 +167,7 @@ finish_msg() { msg "Finished $(basename $0) at $(/bin/date "+%F %T")" end_sec=$(/bin/date +%s.%N) - elapsed_seconds=$(echo "$end_sec - $start_sec" | /usr/bin/bc) + elapsed_seconds=$(echo "$end_sec" "$start_sec" | awk '{ print $1 - $2 }') msg "Elapsed time: $elapsed_seconds" } @@ -241,6 +242,7 @@ autoscaling_enter_standby() { if [ -z "$min_cap" -o -z "$desired_cap" ]; then msg "Unable to determine minimum and desired capacity for ASG ${asg_name}." msg "Attempting to put this instance into standby regardless." + set_flag "asgmindecremented" "false" elif [ $min_cap == $desired_cap -a $min_cap -gt 0 ]; then local new_min=$(($min_cap - 1)) msg "Decrementing ASG ${asg_name}'s minimum size to $new_min" @@ -253,8 +255,11 @@ autoscaling_enter_standby() { else msg "ASG ${asg_name}'s minimum size has been decremented, creating flag in file $FLAGFILE" # Create a "flag" denote that the ASG min has been decremented - set_flag asgmindecremented + set_flag "asgmindecremented" "true" fi + else + msg "No need to decrement ASG ${asg_name}'s minimum size" + set_flag "asgmindecremented" "false" fi msg "Putting instance $instance_id into Standby" @@ -320,7 +325,9 @@ autoscaling_exit_standby() { return 1 fi - if get_flag asgmindecremented; then + if ! local tmp_flag_value=$(get_flag "asgmindecremented"); then + error_exit "$FLAGFILE doesn't exist or is unreadable" + elif [ "$tmp_flag_value" = "true" ]; then local min_desired=$($AWS_CLI autoscaling describe-auto-scaling-groups \ --auto-scaling-group-name "${asg_name}" \ --query 'AutoScalingGroups[0].[MinSize, DesiredCapacity]' \ @@ -372,6 +379,9 @@ get_instance_state_asg() { fi } +# Usage: reset_waiter_timeout +# +# Resets the timeout value to account for the ELB timeout and also connection draining. reset_waiter_timeout() { local elb=$1 local state_name=$2 @@ -528,30 +538,11 @@ validate_elb() { get_elb_list() { local instance_id=$1 - local asg_name=$($AWS_CLI autoscaling describe-auto-scaling-instances \ - --instance-ids $instance_id \ - --query AutoScalingInstances[*].AutoScalingGroupName \ - --output text | sed -e $'s/\t/ /g') local elb_list="" - if [ -z "${asg_name}" ]; then - msg "Instance is not part of an ASG. Looking up from ELB." - local all_balancers=$($AWS_CLI elb describe-load-balancers \ - --query LoadBalancerDescriptions[*].LoadBalancerName \ - --output text | sed -e $'s/\t/ /g') - for elb in $all_balancers; do - local instance_health - instance_health=$(get_instance_health_elb $instance_id $elb) - if [ $? == 0 ]; then - elb_list="$elb_list $elb" - fi - done - else - elb_list=$($AWS_CLI autoscaling describe-auto-scaling-groups \ - --auto-scaling-group-names "${asg_name}" \ - --query AutoScalingGroups[*].LoadBalancerNames \ - --output text | sed -e $'s/\t/ /g') - fi + elb_list=$($AWS_CLI elb describe-load-balancers \ + --query $'LoadBalancerDescriptions[].[join(`,`,Instances[?InstanceId==`'$instance_id'`].InstanceId),LoadBalancerName]' \ + --output text | grep $instance_id | awk '{ORS=" ";print $2}') if [ -z "$elb_list" ]; then return 1 From d8c7a452b2236fbe70c9f98107d15611c1ed92af Mon Sep 17 00:00:00 2001 From: Sebastian Cruz Date: Tue, 14 Jun 2016 17:46:34 +0100 Subject: [PATCH 09/12] - Added support for _all_/_any_ in ELB_LIST --- load-balancing/elb/deregister_from_elb.sh | 26 +++++++++++++++++---- load-balancing/elb/register_with_elb.sh | 28 +++++++++++++++++++---- 2 files changed, 44 insertions(+), 10 deletions(-) diff --git a/load-balancing/elb/deregister_from_elb.sh b/load-balancing/elb/deregister_from_elb.sh index 70f5407..76551ec 100755 --- a/load-balancing/elb/deregister_from_elb.sh +++ b/load-balancing/elb/deregister_from_elb.sh @@ -49,11 +49,27 @@ if [ $? == 0 -a -n "${asg}" ]; then fi fi -msg "Instance is not part of an ASG, continuing..." +msg "Instance is not part of an ASG, trying with ELB..." -msg "Checking that user set at least one load balancer" -if test -z "$ELB_LIST"; then - error_exit "Must have at least one load balancer to deregister from" +if [ -z "$ELB_LIST" ]; then + error_exit "ELB_LIST is empty. Must have at least one load balancer to deregister from, or \"_all_\", \"_any_\" values." +elif [ "${ELB_LIST}" = "_all_" ]; then + msg "Automatically finding all the ELBs that this instance is registered to..." + get_elb_list $INSTANCE_ID + if [ $? != 0 ]; then + error_exit "Couldn't find any. Must have at least one load balancer to deregister from." + fi + set_flag "ELBs" "$ELB_LIST" +elif [ "${ELB_LIST}" = "_any_" ]; then + msg "Automatically finding all the ELBs that this instance is registered to..." + get_elb_list $INSTANCE_ID + if [ $? != 0 ]; then + msg "Couldn't find any, but ELB_LIST=any so finishing successfully without deregistering." + set_flag "ELBs" "" + finish_msg + exit 0 + fi + set_flag "ELBs" "$ELB_LIST" fi # Loop through all LBs the user set, and attempt to deregister this instance from them. @@ -73,7 +89,7 @@ for elb in $ELB_LIST; do fi done -# Wait for all Deregistrations to finish +# Wait for all deregistrations to finish msg "Waiting for instance to de-register from its load balancers" for elb in $ELB_LIST; do wait_for_state "elb" $INSTANCE_ID "OutOfService" $elb diff --git a/load-balancing/elb/register_with_elb.sh b/load-balancing/elb/register_with_elb.sh index 0359f53..d20879f 100755 --- a/load-balancing/elb/register_with_elb.sh +++ b/load-balancing/elb/register_with_elb.sh @@ -49,11 +49,27 @@ if [ $? == 0 -a -n "${asg}" ]; then fi fi -msg "Instance is not part of an ASG, continuing..." +msg "Instance is not part of an ASG, continuing with ELB" -msg "Checking that user set at least one load balancer" -if test -z "$ELB_LIST"; then - error_exit "Must have at least one load balancer to register to" +if [ -z "$ELB_LIST" ]; then + error_exit "ELB_LIST is empty. Must have at least one load balancer to register to, or \"_all_\", \"_any_\" values." +elif [ "${ELB_LIST}" = "_all_" ]; then + msg "Finding all the ELBs that this instance was previously registered to" + if ! ELB_LIST=$(get_flag "ELBs"); then + error_exit "$FLAGFILE doesn't exist or is unreadble" + elif [ -z $ELB_LIST ]; then + error_exit "Couldn't find any. Must have at least one load balancer to register to." + fi +elif [ "${ELB_LIST}" = "_any_" ]; then + msg "Finding all the ELBs that this instance was previously registered to" + if ! ELB_LIST=$(get_flag "ELBs"); then + error_exit "$FLAGFILE doesn't exist or is unreadble" + elif [ -z $ELB_LIST ]; then + msg "Couldn't find any, but ELB_LIST=any so finishing successfully without registering." + remove_flagfile + finish_msg + exit 0 + fi fi # Loop through all LBs the user set, and attempt to register this instance to them. @@ -73,7 +89,7 @@ for elb in $ELB_LIST; do fi done -# Wait for all Registrations to finish +# Wait for all registrations to finish msg "Waiting for instance to register to its load balancers" for elb in $ELB_LIST; do wait_for_state "elb" $INSTANCE_ID "InService" $elb @@ -82,4 +98,6 @@ for elb in $ELB_LIST; do fi done +remove_flagfile + finish_msg From 2da4b757d4c362df10083eb7ea4aaea416473020 Mon Sep 17 00:00:00 2001 From: Sebastian Cruz Date: Thu, 7 Jul 2016 17:47:15 +0100 Subject: [PATCH 10/12] Added a validation to avoid failing when it's the first deployment and deregister_from_elb.sh was not run on ApplicationStop --- load-balancing/elb/deregister_from_elb.sh | 2 ++ load-balancing/elb/register_with_elb.sh | 34 +++++++++++++++-------- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/load-balancing/elb/deregister_from_elb.sh b/load-balancing/elb/deregister_from_elb.sh index 76551ec..39d89cb 100755 --- a/load-balancing/elb/deregister_from_elb.sh +++ b/load-balancing/elb/deregister_from_elb.sh @@ -51,6 +51,8 @@ fi msg "Instance is not part of an ASG, trying with ELB..." +set_flag "dereg" "true" + if [ -z "$ELB_LIST" ]; then error_exit "ELB_LIST is empty. Must have at least one load balancer to deregister from, or \"_all_\", \"_any_\" values." elif [ "${ELB_LIST}" = "_all_" ]; then diff --git a/load-balancing/elb/register_with_elb.sh b/load-balancing/elb/register_with_elb.sh index d20879f..2a88c65 100755 --- a/load-balancing/elb/register_with_elb.sh +++ b/load-balancing/elb/register_with_elb.sh @@ -54,19 +54,31 @@ msg "Instance is not part of an ASG, continuing with ELB" if [ -z "$ELB_LIST" ]; then error_exit "ELB_LIST is empty. Must have at least one load balancer to register to, or \"_all_\", \"_any_\" values." elif [ "${ELB_LIST}" = "_all_" ]; then - msg "Finding all the ELBs that this instance was previously registered to" - if ! ELB_LIST=$(get_flag "ELBs"); then - error_exit "$FLAGFILE doesn't exist or is unreadble" - elif [ -z $ELB_LIST ]; then - error_exit "Couldn't find any. Must have at least one load balancer to register to." + if [ "$(get_flag "dereg")" = "true" ]; then + msg "Finding all the ELBs that this instance was previously registered to" + if ! ELB_LIST=$(get_flag "ELBs"); then + error_exit "$FLAGFILE doesn't exist or is unreadble" + elif [ -z $ELB_LIST ]; then + error_exit "Couldn't find any. Must have at least one load balancer to register to." + fi + else + msg "Assuming this is the first deployment and ELB_LIST=_all_ so finishing successfully without registering." + finish_msg + exit 0 fi elif [ "${ELB_LIST}" = "_any_" ]; then - msg "Finding all the ELBs that this instance was previously registered to" - if ! ELB_LIST=$(get_flag "ELBs"); then - error_exit "$FLAGFILE doesn't exist or is unreadble" - elif [ -z $ELB_LIST ]; then - msg "Couldn't find any, but ELB_LIST=any so finishing successfully without registering." - remove_flagfile + if [ "$(get_flag "dereg")" = "true" ]; then + msg "Finding all the ELBs that this instance was previously registered to" + if ! ELB_LIST=$(get_flag "ELBs"); then + error_exit "$FLAGFILE doesn't exist or is unreadble" + elif [ -z $ELB_LIST ]; then + msg "Couldn't find any, but ELB_LIST=_any_ so finishing successfully without registering." + remove_flagfile + finish_msg + exit 0 + fi + else + msg "Assuming this is the first deployment and ELB_LIST=_any_ so finishing successfully without registering." finish_msg exit 0 fi From 4e3cf224c044855f7c7a58b599984f7657b1a884 Mon Sep 17 00:00:00 2001 From: Sebastian Cruz Date: Fri, 8 Jul 2016 14:39:38 +0100 Subject: [PATCH 11/12] Made suspension of processes optional (default to false) and added clarification to README.md --- load-balancing/elb/README.md | 20 +++++++++++++++++--- load-balancing/elb/common_functions.sh | 19 +++++++++++++------ 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/load-balancing/elb/README.md b/load-balancing/elb/README.md index 20432eb..bf6411b 100644 --- a/load-balancing/elb/README.md +++ b/load-balancing/elb/README.md @@ -2,8 +2,6 @@ Often when running a web service, you'll have your instances behind a load balancer. But when deploying new code to these instances, you don't want the load balancer to continue sending customer traffic to an instance while the deployment is in progress. Lifecycle event scripts give you the ability to integrate your AWS CodeDeploy deployments with instances that are behind an Elastic Load Balancer or in an Auto Scaling group. Simply set the name (or names) of the Elastic Load Balancer your instances are a part of, set the scripts in the appropriate lifecycle events, and the scripts will take care of deregistering the instance, waiting for connection draining, and re-registering after the deployment finishes. -*Please note that this scripts will suspend some AutoScaling processes (AZRebalance, AlarmNotification, ScheduledActions and ReplaceUnhealthy) while deploying, in order to avoid wrong interactions.* - ## Requirements The register and deregister scripts have a couple of dependencies in order to properly interact with Elastic Load Balancing and AutoScaling: @@ -36,4 +34,20 @@ To use these scripts in your own application: 4. Edit your application's `appspec.yml` to run `deregister_from_elb.sh` on the ApplicationStop event, and `register_with_elb.sh` on the ApplicationStart event. 5. If your instance is not in an Auto Scaling Group, edit `common_functions.sh` to set `ELB_LIST` to contain the name(s) of the Elastic Load Balancer(s) your deployment group is a part of. Make sure the entries in ELB_LIST are separated by space. Alternatively, you can set `ELB_LIST` to `_all_` to automatically use all load balancers the instance is registered to, or `_any_` to get the same behaviour as `_all_` but without failing your deployments if the instance is not part of any ASG or ELB. This is more flexible in heterogeneous tag-based Deployment Groups. -6. Deploy! \ No newline at end of file +6. Optionally set up `HANDLE_PROCS=true` in `common_functions.sh`. See note below. +7. Deploy! + +## Important notice about handling AutoScaling processes + +When using AutoScaling with CodeDeploy you have to consider some edge cases during the deployment time window: + +1. If you have a scale up event, the new instance(s) will get the latest successful *Revision*, and not the one you are currently deploying. You will end up with a fleet of mixed revisions. +2. If you have a scale down event, instances are going to be terminated, and your deployment will (probably) fail. +3. If your instances are not balanced accross Availability Zones **and you are** using these scripts, AutoScaling may terminate some instances or create new ones to maintain balance (see [this doc](http://docs.aws.amazon.com/autoscaling/latest/userguide/as-suspend-resume-processes.html#process-types)), interfering with your deployment. +4. If you have the health checks of your AutoScaling Group based off the ELB's ([documentation](http://docs.aws.amazon.com/autoscaling/latest/userguide/healthcheck.html)) **and you are not** using these scripts, then instances will be marked as unhealthy and terminated, interfering with your deployment. + +In an effort to solve these cases, the scripts can suspend some AutoScaling processes (AZRebalance, AlarmNotification, ScheduledActions and ReplaceUnhealthy) while deploying, to avoid those events happening in the middle of your deployment. You only have to set up `HANDLE_PROCS=true` in `common_functions.sh`. + +A record of the previously (to the start of the deployment) suspended process is kept by the scripts (on each instance), so when finishing the deployment the status of the processes on the AutoScaling Group should be returned to the same status as before. I.e. if AZRebalance was suspended manually it will not be resumed. However, if the scripts don't run (failed deployment) you may end up with stale suspended processes. + +**WARNING**: If you are using this functionality you should only use *CodeDepoyDefault.OneAtATime* deployment configuration to ensure a serial execution of the scripts. Concurrent runs are not supported. \ No newline at end of file diff --git a/load-balancing/elb/common_functions.sh b/load-balancing/elb/common_functions.sh index c33061b..7e3f21a 100644 --- a/load-balancing/elb/common_functions.sh +++ b/load-balancing/elb/common_functions.sh @@ -40,6 +40,9 @@ MIN_CLI_VERSION='1.3.25' # Create a flagfile for each deployment FLAGFILE="/tmp/asg_codedeploy_flags-$DEPLOYMENT_GROUP_ID-$DEPLOYMENT_ID" +# Handle ASG processes +HANDLE_PROCS=false + # Usage: get_instance_region # # Writes to STDOUT the AWS region as known by the local instance. @@ -224,11 +227,13 @@ autoscaling_enter_standby() { return 0 fi - msg "Checking ASG ${asg_name} suspended processes" - check_suspended_processes + if [ "$HANDLE_PROCS" = "true" ]; then + msg "Checking ASG ${asg_name} suspended processes" + check_suspended_processes - # Suspend troublesome processes while deploying - suspend_processes + # Suspend troublesome processes while deploying + suspend_processes + fi msg "Checking to see if ASG ${asg_name} will let us decrease desired capacity" local min_desired=$($AWS_CLI autoscaling describe-auto-scaling-groups \ @@ -351,8 +356,10 @@ autoscaling_exit_standby() { msg "Auto scaling group was not decremented previously, not incrementing min value" fi - # Resume processes, except for the ones suspended before deployment - resume_processes + if [ "$HANDLE_PROCS" = "true" ]; then + # Resume processes, except for the ones suspended before deployment + resume_processes + fi # Clean up the FLAGFILE remove_flagfile From 99c91fe3b588c8cb23a1e3a2a0094784e6ff5b60 Mon Sep 17 00:00:00 2001 From: Sebastian Cruz Date: Fri, 8 Jul 2016 15:46:42 +0100 Subject: [PATCH 12/12] Added disclaimer about gaps. --- load-balancing/elb/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/load-balancing/elb/README.md b/load-balancing/elb/README.md index bf6411b..bda717d 100644 --- a/load-balancing/elb/README.md +++ b/load-balancing/elb/README.md @@ -50,4 +50,6 @@ In an effort to solve these cases, the scripts can suspend some AutoScaling proc A record of the previously (to the start of the deployment) suspended process is kept by the scripts (on each instance), so when finishing the deployment the status of the processes on the AutoScaling Group should be returned to the same status as before. I.e. if AZRebalance was suspended manually it will not be resumed. However, if the scripts don't run (failed deployment) you may end up with stale suspended processes. -**WARNING**: If you are using this functionality you should only use *CodeDepoyDefault.OneAtATime* deployment configuration to ensure a serial execution of the scripts. Concurrent runs are not supported. \ No newline at end of file +Disclaimer: There's a small chance that an event is triggered while the deployment is progressing from one instance to another. The only way to avoid that completely whould be to monitor the deployment externally to CodeDeploy/AutoScaling and act accordingly. The effort on doing that compared to this depends on the each use case. + +**WARNING**: If you are using this functionality you should only use *CodeDepoyDefault.OneAtATime* deployment configuration to ensure a serial execution of the scripts. Concurrent runs are not supported.