Addition of function to calculate rally point instance in AWS Auto Scaling Groups (#33)

yardbirdsax · web-flow · commit 391ad6d0b392 · 2021-04-08T09:52:33.000+01:00
* Dockerfiles have been updated to use Python 3 due to multiple issues with running under Python 2.

* A new function, aws_wrapper_get_asg_rally_point, has been added. This allows you to calculate a single 'rally point' instance that is part of an auto-scaling group, useful for executing bootstrapping commands once per ASG.

* Refactored new unit tests to include a custom setup function for each set, and split out public vs private hostname call scenarios to be consistent with other tests.

* The Dockerfiles for tests have been updated to include setting some encoding environment variables, which resolves some issues when running tests, as well as removing some un-necessary 'sudo' calls.

* Steps to install some new dependencies have been moved to the 'Install basic dependencies' section of the test Dockerfiles per comments from maintainer.

* The comment for the new function has been updated per maintainer comment.

* Variable usage and whitespace have been updated per comments from the maintainer.

* Additional tests have been added for the 'aws_wrapper_get_asg_rally_point' function to ensure that the function fails when the ASG does not contain the requisite number of instances after the given retry / wait period has elapsed.

* The Dockerfiles for the tests have been updated to include better comments / details around the use of the locale environment variables.
diff --git a/Dockerfile.ubuntu16.04.bats b/Dockerfile.ubuntu16.04.bats
@@ -3,7 +3,11 @@ MAINTAINER Gruntwork <info@gruntwork.io>
 
 # Install basic dependencies
 RUN DEBIAN_FRONTEND=noninteractive apt-get update && \
-    apt-get install -y vim python-pip jq sudo curl
+    apt-get install -y vim python-pip jq sudo curl libffi-dev python3-dev && \
+    update-alternatives --install /usr/bin/python python /usr/bin/python3 1 && \
+    update-alternatives --config python && \
+    curl https://bootstrap.pypa.io/pip/3.5/get-pip.py -o /tmp/get-pip.py && \
+    python /tmp/get-pip.py
 
 # Install Bats
 RUN apt-get install -y software-properties-common && \
@@ -25,3 +29,9 @@ RUN apt-get install -y net-tools iptables
 
 # Copy mock AWS CLI into the PATH
 COPY ./.circleci/aws-local.sh /usr/local/bin/aws
+
+# These have been added to resolve some encoding error issues with the tests. These were introduced during the upgrade to Python 3.6, 
+# which is known to have some sensitivity around locale issues. These variables should correct that, per examples like this SO thread: 
+# https://stackoverflow.com/questions/51026315/how-to-solve-unicodedecodeerror-in-python-3-6/51027262#51027262.
+ENV LC_ALL=C.UTF-8
+ENV LANG=C.UTF-8
diff --git a/Dockerfile.ubuntu18.04.bats b/Dockerfile.ubuntu18.04.bats
@@ -3,7 +3,7 @@ MAINTAINER Gruntwork <info@gruntwork.io>
 
 # Install basic dependencies
 RUN DEBIAN_FRONTEND=noninteractive apt-get update && \
-    apt-get install -y vim python-pip jq sudo curl
+    apt-get install -y vim python-pip jq sudo curl libffi-dev python3-dev
 
 # Install Bats
 RUN apt-get install -y software-properties-common && \
@@ -12,7 +12,12 @@ RUN apt-get install -y software-properties-common && \
     apt-get install -y bats
 
 # Install AWS CLI
-RUN pip install awscli --upgrade --user
+RUN apt install python3-distutils python3-apt -y && \
+    update-alternatives --install /usr/bin/python python /usr/bin/python3 1 && \
+    update-alternatives --config python && \
+    curl https://bootstrap.pypa.io/get-pip.py -o /tmp/get-pip.py && \
+    python /tmp/get-pip.py && \
+    pip install awscli --upgrade --user
 
 # Install moto: https://github.com/spulec/moto
 # Lock cfn-lint and pysistent to last known working versions
@@ -23,3 +28,9 @@ RUN apt-get install -y net-tools iptables
 
 # Copy mock AWS CLI into the PATH
 COPY ./.circleci/aws-local.sh /usr/local/bin/aws
+
+# These have been added to resolve some encoding error issues with the tests. These were introduced during the upgrade to Python 3.6, 
+# which is known to have some sensitivity around locale issues. These variables should correct that, per examples like this SO thread: 
+# https://stackoverflow.com/questions/51026315/how-to-solve-unicodedecodeerror-in-python-3-6/51027262#51027262.
+ENV LC_ALL=C.UTF-8
+ENV LANG=C.UTF-8
diff --git a/modules/bash-commons/src/aws-wrapper.sh b/modules/bash-commons/src/aws-wrapper.sh
@@ -215,3 +215,36 @@ function aws_wrapper_get_hostname {
     aws_get_instance_private_hostname
   fi
 }
+
+# Calculates a "rally point" instance in an ASG and returns its hostname. This is a deterministic way for the instances in an ASG to all pick the same single instance to perform some action: e.g., this instance could become the leader in a cluster or run some initialization script that should only be run once for the entire ASG. Under the hood, this method picks the instance in the ASG with the earliest launch time; in the case of ties, the instance with the earliest instance ID (lexicographically) is returned. This method assumes jq is installed.
+function aws_wrapper_get_asg_rally_point {
+  local -r asg_name="$1"
+  local -r aws_region="$2"
+  local -r use_public_hostname="$3"
+  local -r retries="${4:-60}"
+  local -r sleep_between_retries="${5:-5}"
+
+  log_info "Calculating rally point for ASG $asg_name in $aws_region"
+
+  local instances
+  log_info "Waiting for all instances to be available..."
+  instances=$(aws_wrapper_wait_for_instances_in_asg $asg_name $aws_region $retries $sleep_between_retries)
+  assert_not_empty_or_null "$instances" "Wait for instances in ASG $asg_name in $aws_region"
+
+  local rally_point
+  rally_point=$(echo "$instances" | jq -r '[.Reservations[].Instances[]] | sort_by(.LaunchTime, .InstanceId) | .[0]')
+  assert_not_empty_or_null "$rally_point" "Select rally point server in ASG $asg_name"
+
+  local hostname_field=".PrivateDnsName"
+  if [[ "$use_public_hostname" == "true" ]]; then
+    hostname_field=".PublicDnsName"
+  fi
+
+  log_info "Hostname field is $hostname_field"
+
+  local hostname
+  hostname=$(echo "$rally_point" | jq -r "$hostname_field")
+  assert_not_empty_or_null "$hostname" "Get hostname from field $hostname_field for rally point in $asg_name: $rally_point"
+  
+  echo -n "$hostname"
+}
diff --git a/test/aws-wrapper.bats b/test/aws-wrapper.bats
@@ -1311,4 +1311,289 @@ END_HEREDOC
 
   assert_success
   assert_equal "ip-10-251-50-12.ec2.internal" "$out"
+}
+
+function setup_rally_point_by_instance {
+  local -r asg_name="$1"
+  local -r aws_region="$2"
+  local -r size=3
+
+  local -r asg=$(cat <<END_HEREDOC
+{
+    "AutoScalingGroups": [
+        {
+            "AutoScalingGroupARN": "arn:aws:autoscaling:us-west-2:123456789012:autoScalingGroup:930d940e-891e-4781-a11a-7b0acd480f03:autoScalingGroupName/$asg_name",
+            "DesiredCapacity": $size,
+            "AutoScalingGroupName": "$asg_name",
+            "MinSize": 0,
+            "MaxSize": 10,
+            "LaunchConfigurationName": "my-launch-config",
+            "CreatedTime": "2013-08-19T20:53:25.584Z",
+            "AvailabilityZones": [
+                "${aws_region}c"
+            ]
+        }
+    ]
+}
+END_HEREDOC
+)
+
+  local -r instances=$(cat <<END_HEREDOC
+{
+  "Reservations": [
+    {
+      "Instances": [
+        {
+          "LaunchTime": "2013-08-19T20:53:25.584Z",
+          "InstanceId": "i-1234567890abcdef0",
+          "PublicIpAddress": "55.66.77.88",
+          "PrivateIpAddress": "11.22.33.44",
+          "PrivateDnsName": "ip-10-251-50-12.ec2.internal",
+          "PublicDnsName": "ec2-203-0-113-25.compute-1.amazonaws.com",
+          "Tags": [
+            {
+              "Value": "$asg_name",
+              "Key": "aws:autoscaling:groupName"
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "Instances": [
+        {
+          "LaunchTime": "2013-08-19T20:53:25.584Z",
+          "InstanceId": "i-1234567890abcdef1",
+          "PublicIpAddress": "55.66.77.881",
+          "PrivateIpAddress": "11.22.33.441",
+          "PrivateDnsName": "ip-10-251-50-121.ec2.internal",
+          "PublicDnsName": "ec2-203-0-113-252.compute-1.amazonaws.com",
+          "Tags": [
+            {
+              "Value": "$asg_name",
+              "Key": "aws:autoscaling:groupName"
+            }
+          ]
+        },
+        {
+          "LaunchTime": "2013-08-19T20:53:25.584Z",
+          "InstanceId": "i-1234567890abcdef2",
+          "PublicIpAddress": "55.66.77.882",
+          "PrivateIpAddress": "11.22.33.442",
+          "PrivateDnsName": "ip-10-251-50-122.ec2.internal",
+          "PublicDnsName": "ec2-203-0-113-253.compute-1.amazonaws.com",
+          "Tags": [
+            {
+              "Value": "$asg_name",
+              "Key": "aws:autoscaling:groupName"
+            }
+          ]
+        }
+      ]
+    }
+  ]
+}
+END_HEREDOC
+)
+
+  load_aws_mock "" "$asg" "$instances"
+}
+
+@test "aws_wrapper_get_asg_rally_point by instance id, private" {
+  local -r asg_name="foo"
+  local -r aws_region="us-west-2"
+
+  setup_rally_point_by_instance $asg_name $aws_region
+
+  local out
+
+  out=$(aws_wrapper_get_asg_rally_point $asg_name $aws_region)
+  assert_success
+  assert_equal "$out" "ip-10-251-50-12.ec2.internal"
+}
+
+@test "aws_wrapper_get_asg_rally_point by instance id, public" {
+  local -r asg_name="foo"
+  local -r aws_region="us-west-2"
+
+  setup_rally_point_by_instance $asg_name $aws_region
+
+  local out
+
+  out=$(aws_wrapper_get_asg_rally_point $asg_name $aws_region true)
+  assert_success
+  assert_equal "$out" "ec2-203-0-113-25.compute-1.amazonaws.com"
+}
+
+function setup_rally_point_by_launch_time {
+  local -r asg_name="$1"
+  local -r aws_region="$2"
+  local -r size=3
+
+  local -r asg=$(cat <<END_HEREDOC
+{
+    "AutoScalingGroups": [
+        {
+            "AutoScalingGroupARN": "arn:aws:autoscaling:us-west-2:123456789012:autoScalingGroup:930d940e-891e-4781-a11a-7b0acd480f03:autoScalingGroupName/$asg_name",
+            "DesiredCapacity": $size,
+            "AutoScalingGroupName": "$asg_name",
+            "MinSize": 0,
+            "MaxSize": 10,
+            "LaunchConfigurationName": "my-launch-config",
+            "CreatedTime": "2013-08-19T20:53:25.584Z",
+            "AvailabilityZones": [
+                "${aws_region}c"
+            ]
+        }
+    ]
+}
+END_HEREDOC
+)
+
+local -r instances=$(cat <<END_HEREDOC
+{
+  "Reservations": [
+    {
+      "Instances": [
+        {
+          "LaunchTime": "2013-08-19T20:53:25.584Z",
+          "InstanceId": "i-1234567890abcdef0",
+          "PublicIpAddress": "55.66.77.88",
+          "PrivateIpAddress": "11.22.33.44",
+          "PrivateDnsName": "ip-10-251-50-12.ec2.internal",
+          "PublicDnsName": "ec2-203-0-113-25.compute-1.amazonaws.com",
+          "Tags": [
+            {
+              "Value": "$asg_name",
+              "Key": "aws:autoscaling:groupName"
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "Instances": [
+        {
+          "LaunchTime": "2013-08-19T20:53:25.584Z",
+          "InstanceId": "i-1234567890abcdef1",
+          "PublicIpAddress": "55.66.77.881",
+          "PrivateIpAddress": "11.22.33.441",
+          "PrivateDnsName": "ip-10-251-50-121.ec2.internal",
+          "PublicDnsName": "ec2-203-0-113-252.compute-1.amazonaws.com",
+          "Tags": [
+            {
+              "Value": "$asg_name",
+              "Key": "aws:autoscaling:groupName"
+            }
+          ]
+        },
+        {
+          "LaunchTime": "2013-08-19T20:53:24.584Z",
+          "InstanceId": "i-1234567890abcdef2",
+          "PublicIpAddress": "55.66.77.882",
+          "PrivateIpAddress": "11.22.33.442",
+          "PrivateDnsName": "ip-10-251-50-122.ec2.internal",
+          "PublicDnsName": "ec2-203-0-113-253.compute-1.amazonaws.com",
+          "Tags": [
+            {
+              "Value": "$asg_name",
+              "Key": "aws:autoscaling:groupName"
+            }
+          ]
+        }
+      ]
+    }
+  ]
+}
+END_HEREDOC
+)
+
+  load_aws_mock "" "$asg" "$instances"
+}
+
+@test "aws_wrapper_get_asg_rally_point by launch time, private" {
+  local -r asg_name="foo"
+  local -r aws_region="us-west-2"
+
+  setup_rally_point_by_launch_time $asg_name $aws_region
+
+  local out
+
+  out=$(aws_wrapper_get_asg_rally_point $asg_name $aws_region)
+  assert_success
+  assert_equal "$out" "ip-10-251-50-122.ec2.internal"
+}
+
+@test "aws_wrapper_get_asg_rally_point by launch time, public" {
+  local -r asg_name="foo"
+  local -r aws_region="us-west-2"
+
+  setup_rally_point_by_launch_time $asg_name $aws_region
+
+  local out
+
+  out=$(aws_wrapper_get_asg_rally_point $asg_name $aws_region true)
+  assert_success
+  assert_equal "$out" "ec2-203-0-113-253.compute-1.amazonaws.com"
+}
+
+function setup_rally_point_empty {
+  local -r asg_name="$1"
+  local -r aws_region="$2"
+  local -r size=3
+
+  local -r asg=$(cat <<END_HEREDOC
+{
+    "AutoScalingGroups": [
+        {
+            "AutoScalingGroupARN": "arn:aws:autoscaling:us-west-2:123456789012:autoScalingGroup:930d940e-891e-4781-a11a-7b0acd480f03:autoScalingGroupName/$asg_name",
+            "DesiredCapacity": $size,
+            "AutoScalingGroupName": "$asg_name",
+            "MinSize": 0,
+            "MaxSize": 10,
+            "LaunchConfigurationName": "my-launch-config",
+            "CreatedTime": "2013-08-19T20:53:25.584Z",
+            "AvailabilityZones": [
+                "${aws_region}c"
+            ]
+        }
+    ]
+}
+END_HEREDOC
+)
+
+local -r instances=$(cat <<END_HEREDOC
+{
+  "Reservations": []
+}
+END_HEREDOC
+)
+
+  load_aws_mock "" "$asg" "$instances"
+}
+
+@test "aws_wrapper_get_asg_rally_point empty asg, private" {
+  local -r asg_name="foo"
+  local -r aws_region="us-west-2"
+  local -r use_public_hostname="false"
+  local -r retries=5
+  local -r sleep_between_retries=2
+
+  setup_rally_point_empty $asg_name $aws_region
+
+  run aws_wrapper_get_asg_rally_point $asg_name $aws_region $use_public_hostname $retries $sleep_between_retries
+  assert_failure
+}
+
+@test "aws_wrapper_get_asg_rally_point empty asg, public" {
+  local -r asg_name="foo"
+  local -r aws_region="us-west-2"
+  local -r use_public_hostname="true"
+  local -r retries=5
+  local -r sleep_between_retries=2
+
+  setup_rally_point_empty $asg_name $aws_region
+
+  run aws_wrapper_get_asg_rally_point $asg_name $aws_region $use_public_hostname $retries $sleep_between_retries
+  assert_failure
 }
