update_asg.py 14.2 KB
Newer Older
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
1 2 3
#!/usr/bin/env python3

from time import sleep
4 5
from akinaka_libs import helpers
from akinaka_libs import exceptions
6
from botocore.exceptions import ParamValidationError
7
from botocore.exceptions import ClientError
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
8
from akinaka_client.aws_client import AWS_Client
9
import logging
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
10 11 12 13

aws_client = AWS_Client()

class ASG():
14 15
    """All the methods needed to perform a blue/green deploy"""

16
    def __init__(self, region, role_arn):
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
17 18 19
        self.region = region
        self.role_arn = role_arn

20
    def get_application_name(self, asg, loadbalancer=None, target_group=None):
21
        """
22 23
        Returns the application name that we're deploying, worked out from the target group
        (via the load balancer if applicable)
24
        """
25

26 27
        if asg:
            return asg
28

29
        target_group_arn = self.get_target_group_arn(loadbalancer, target_group)
30 31
        active_asg = self.get_active_asg(target_group_arn)
        asg_split = active_asg.split('-')[0:-1]
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
32

33
        return '-'.join(asg_split)
34

35
    def asgs_by_liveness(self, asg=None, loadbalancer=None, target_group=None):
36 37
        """Return dict of '{inactive: ASG, active: ASG}'"""

38
        if asg is not None:
39
            logging.info("We've been given the ASG name as an argument")
40 41
            # NOTE: "inactive_asg" is a misnomer at this point, since when we already have the ASG
            #       name, it is also the active ASG, because this case is used for non-blue/green ASGs
42
            return {"inactive_asg": asg, "active_asg": asg}
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
43

44
        target_group_arn = self.get_target_group_arn(loadbalancer=loadbalancer, target_group=target_group)
45
        active_asg = self.get_active_asg(target_group_arn)
46 47 48
        inactive_asg = self.get_inactive_asg(active_asg)

        return {"inactive_asg": inactive_asg, "active_asg": active_asg}
49

50
    def scale_down_inactive(self, asg):
51 52
        # NOTE: We're making the heavy presumption here that the _active_ ASG is the one we've been
        #       given from the command line
53
        inactive_asg = self.get_inactive_asg(asg)
54
        self.scale(inactive_asg, 0, 0, 0)
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
55

56 57 58 59 60
    def do_update(self, ami, asg=None, loadbalancer=None, target_group=None):
        asg_liveness_info = self.asgs_by_liveness(asg=asg, loadbalancer=loadbalancer, target_group=target_group)

        inactive_asg = asg_liveness_info['inactive_asg']
        active_asg = asg_liveness_info['active_asg']
61
        new_ami = ami
62

63 64
        logging.info("New ASG was worked out as {}. Now updating it's Launch Template".format(inactive_asg))
        self.update_launch_template(inactive_asg, new_ami, self.get_lt_name(inactive_asg))
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
65

66
        scale_to = self.get_current_scale(active_asg)
67

68
        logging.info("Scaling ASG down")
69 70
        self.scale(inactive_asg, 0, 0, 0)
        while not self.asg_is_empty(inactive_asg):
71 72 73 74
            logging.info("Waiting for instances in ASG to terminate")
            sleep(10)

        logging.info("Scaling ASG back up")
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
75
        self.scale(
76
            auto_scaling_group_id = inactive_asg,
77 78
            min_size = scale_to['min'],
            max_size = scale_to['max'],
79
            desired =  scale_to['desired']
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
80
        )
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
81

82
        while len(self.get_auto_scaling_group_instances(inactive_asg)) < 1:
83 84
            logging.info("Waiting for instances in ASG to start ...")
            sleep(10)
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
85

86 87 88 89 90
        logging.info("First instance has started")

        # Try to get information for an instance in the new ASG 20 times
        for i in range(20):
            try:
91
                first_new_instance = self.get_first_new_instance(inactive_asg)
92 93 94 95 96 97
                if i == 20:
                    raise exceptions.AkinakaCriticalException
                break
            except exceptions.AkinakaLoggingError as error:
                logging.info("Retry {}".format(i))
                logging.info("Problem in getting data for first new ASG instance")
98
                logging.info("get_auto_scaling_group_instances() returned: {}".format(self.get_auto_scaling_group_instances(inactive_asg)))
99 100 101
                logging.info("Error was: {}".format(error))

        # Show console output for first instance up until it's Lifecycle Hook has passed
102
        while self.get_auto_scaling_group_instances(auto_scaling_group_id=inactive_asg, instance_ids=[first_new_instance])[0]['LifecycleState'] != "InService":
103 104 105 106 107 108 109 110
            try:
                logging.info("Attempting to retrieve console output from first instance up -- this will not work for non-nitro hypervisor VMs")
                logging.info(self.get_instance_console_output(first_new_instance)['Output'])
            except KeyError:
                logging.info("No output from instance yet. Trying again in 10 seconds.")
            sleep(10)

        # Wait for remaining instances (if any) to come up too
111
        while len(self.asgs_healthy_instances(inactive_asg)) < scale_to['desired']:
112 113
            logging.info("Waiting for all instances to be healthy ...")

114 115
        logging.info("ASG fully healthy. Logging new ASG name to \"inactive_asg.txt\"")
        open("inactive_asg.txt", "w").write(inactive_asg)
116 117 118 119 120 121 122 123 124 125 126 127 128 129

    def get_first_new_instance(self, new_asg):
        asg_instances = self.get_auto_scaling_group_instances(new_asg)
        if len(asg_instances) < 1: raise exceptions.AkinakaLoggingError

        return asg_instances[0]['InstanceId']

    def get_instance_console_output(self, instance):
        ec2_client = aws_client.create_client('ec2', self.region, self.role_arn)
        return ec2_client.get_console_output(InstanceId=instance)

    def instance_state(self, instance):
        ec2_client = aws_client.create_client('ec2', self.region, self.role_arn)
        instance_states = ec2_client.describe_instance_status(IncludeAllInstances=True, InstanceIds=[instance])
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
130

131 132
        while 'InstanceStatuses' not in instance_states:
            instance_states = ec2_client.describe_instance_status(InstanceIds=[instance])
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
133

134
        return instance_states['InstanceStatuses'][0]['InstanceState']['Name']
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
135 136 137 138 139

    def get_lt_name(self, asg):
        asg_client = aws_client.create_client('autoscaling', self.region, self.role_arn)
        lt_info = asg_client.describe_auto_scaling_groups(AutoScalingGroupNames=[asg])

140
        try:
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
141
            return lt_info['AutoScalingGroups'][0]['LaunchTemplate']['LaunchTemplateName']
142 143 144
        except Exception as e:
            raise exceptions.AkinakaCriticalException("{}: Likely couldn't find the ASG you're trying to update".format(e))

145
    def get_target_group_arn(self, loadbalancer=None, target_group=None):
146
        """
147 148 149 150 151 152
        Returns a string containing the ARN of the target group, either by using:

        * targetgroup from --targetgroup
        * loadbalancer from --lb

        Both are mutually exclusive, and at least one must be supplied
153
        """
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
154 155

        alb_client = aws_client.create_client('elbv2', self.region, self.role_arn)
156 157
        if target_group is not None:
            return alb_client.describe_target_groups(Names=[target_group])['TargetGroups'][0]['TargetGroupArn']
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
158

159
        loadbalancer_raw_info = alb_client.describe_load_balancers(Names=[loadbalancer])
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
160 161 162 163 164
        loadbalancer_arn = loadbalancer_raw_info['LoadBalancers'][0]['LoadBalancerArn']

        target_groups_raw_info = alb_client.describe_target_groups(LoadBalancerArn=loadbalancer_arn)['TargetGroups']
        target_group_arns = [targetgroup['TargetGroupArn'] for targetgroup in target_groups_raw_info]

165 166 167 168 169 170 171 172 173 174 175 176 177
        # If we get this far, then the LB has more than a single target group, and we can't work
        # out which one the caller wants
        if len(target_group_arns) == 1:
            return target_group_arns[0]
        elif len(target_group_arns) > 1:
            error_message = "Load balancer has {} target groups".format(len(target_group_arns))

            for target_group in target_group_arns:
                error_message += "> " + target_group
        else:
            error_message = "Load balancer has no target groups"

        raise exceptions.AkinakaCriticalException(error_message)
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
178

179 180
    def get_target_groups_instances(self, target_group_arn):
        """Returns an array of instance IDs belonging to the target group specified"""
181
        alb_client = aws_client.create_client('elbv2', self.region, self.role_arn)
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
182

Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
183 184
        target_groups_instances = []

185
        these_target_group_instances = alb_client.describe_target_health(TargetGroupArn=target_group_arn)['TargetHealthDescriptions']
186 187 188 189
        these_target_group_instances = [
            instance for instance in these_target_group_instances if not instance['TargetHealth']['State'] == "unused"
        ]

190 191 192 193
        # NOTE: This presumes some robustness from your target groups. If they contain instances
        #       from stale ASGs, or anything else is off in them, you will get unexpected results
        for instance in these_target_group_instances:
            target_groups_instances.append(instance['Target']['Id'])
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
194 195

        return target_groups_instances
196

197
    def get_active_asg(self, target_groups):
198
        ec2_client = aws_client.create_client('ec2', self.region, self.role_arn)
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
199

200
        target_groups_instances = self.get_target_groups_instances(target_groups)
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
201

Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
202 203 204
        instances_with_tags = {}

        raw_instances_reservations = ec2_client.describe_instances(InstanceIds=target_groups_instances)['Reservations']
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
205

Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
206 207 208 209 210 211 212 213 214 215 216 217 218
        for reservation in raw_instances_reservations:
            for instance in reservation['Instances']:
                instances_with_tags[instance['InstanceId']] = instance['Tags']

        instances_with_asg = {}

        # Create dictionary with Instance -> AG
        for (key,value) in instances_with_tags.items():
            for tag in value:

                if tag['Key'] == 'aws:autoscaling:groupName':
                    instances_with_asg[key] = tag['Value']

219
        return next(iter(instances_with_asg.values()))
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
220

Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
221
    def get_inactive_asg(self, active_asg):
222 223
        asg_parts = active_asg.split('-')
        active_colour = asg_parts[-1]
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
224 225 226 227 228 229

        if active_colour == "blue":
            inactive_colour = "green"
        else:
            inactive_colour = "blue"

230
        asg_parts[-1] = inactive_colour
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
231

232
        return "-".join(asg_parts)
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280

    def get_launch_template_id(self, lt_name):
        ec2_client = aws_client.create_client('ec2', self.region, self.role_arn)

        response = ec2_client.describe_launch_templates(
            DryRun=False,
            Filters=[
                {
                    'Name': 'launch-template-name',
                    'Values': [ lt_name ]
                }
            ]
        )

        # This is hackish because we are assuming that there is going to be only 1 (one) launch template
        return {
            "id": response['LaunchTemplates'][0]['LaunchTemplateId'],
            "version": response['LaunchTemplates'][0]['LatestVersionNumber']
        }

    def update_launch_template(self, inactive_asg, ami, lt_name):
        lt = self.get_launch_template_id(lt_name)
        lt_client  = aws_client.create_client('ec2', self.region, self.role_arn)

        lt_id = lt['id']
        lt_source_version = str(lt['version'])

        response = lt_client.create_launch_template_version(
            DryRun = False,
            LaunchTemplateId = lt_id,
            SourceVersion = lt_source_version,
            LaunchTemplateData = {
                "ImageId": ami
            }
        )

        launch_template_new_version = str(response['LaunchTemplateVersion']['VersionNumber'])

        lt_client.modify_launch_template(
            LaunchTemplateId=lt_id,
            DefaultVersion=launch_template_new_version
        )

        return {
            "id": lt_id,
            "version": launch_template_new_version
        }

Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
281 282 283
    def scale(self, auto_scaling_group_id, min_size, max_size, desired):
        """Scale an ASG to {'min_size', 'max_size', 'desired'}"""

Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
284 285 286 287 288 289
        asg_client = aws_client.create_client('autoscaling', self.region, self.role_arn)

        response = asg_client.update_auto_scaling_group(
            AutoScalingGroupName=auto_scaling_group_id,
            MinSize=min_size,
            MaxSize=max_size,
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
290
            DesiredCapacity=desired
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
291 292 293 294
        )

        return response

295 296
    def get_current_scale(self, asg):
        """
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
297 298
        Returns the current scales of the given ASG as dict {'desired', 'min', 'max'},
        expects ASG ID as argument
299 300
        """

301 302
        asg_client = aws_client.create_client('autoscaling', self.region, self.role_arn)
        asg = asg_client.describe_auto_scaling_groups(AutoScalingGroupNames=[asg])['AutoScalingGroups'][0]
303

304 305 306 307 308
        return {
            "desired": asg['DesiredCapacity'],
            "min": asg['MinSize'],
            "max": asg['MaxSize']
        }
309

310
    def get_auto_scaling_group_instances(self, auto_scaling_group_id, instance_ids=None):
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
311
        asg_client = aws_client.create_client('autoscaling', self.region, self.role_arn)
312 313
        instance_ids = instance_ids if instance_ids else []
        asg_instances = asg_client.describe_auto_scaling_instances(InstanceIds=instance_ids)
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
314 315 316
        target_instances = []

        for i in asg_instances['AutoScalingInstances']:
317
            if i['AutoScalingGroupName'] == auto_scaling_group_id:
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
318 319
                target_instances.append(i)

320
                logging.debug("Instance {instance_id} has state = {instance_state},"
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
321 322 323 324
                    "Lifecycle is at {instance_lifecycle_state}".format(
                        instance_id = i['InstanceId'],
                        instance_state = i['HealthStatus'],
                        instance_lifecycle_state = i['LifecycleState']
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
325
                    )
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
326
                )
327 328 329 330 331 332

                logging.info("Instance {instance_id} is {instance_lifecycle_state}".format(
                        instance_id = i['InstanceId'],
                        instance_lifecycle_state = i['LifecycleState']
                    )
                )
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
333

Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
334 335

        return target_instances
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
336

337 338 339 340 341 342 343 344
    def asg_is_empty(self, auto_scaling_group_id):
        asg_instances = self.get_auto_scaling_group_instances(auto_scaling_group_id)

        return all(instance.get('LifecycleState') == 'Terminated' for instance in asg_instances)

    def asgs_healthy_instances(self, auto_scaling_group_id):
        asg_instances = self.get_auto_scaling_group_instances(auto_scaling_group_id)
        healthy_instances = []
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
345

346 347 348 349
        for i in asg_instances:
            if i['LifecycleState'] == "InService":
                healthy_instances.append(i)

Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
350
        return healthy_instances