update_asg.py 15.5 KB
Newer Older
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
1 2 3
#!/usr/bin/env python3

from time import sleep
Afraz Ahmadzadeh's avatar
WIP  
Afraz Ahmadzadeh committed
4 5
from akinaka.libs import helpers
from akinaka.libs import exceptions
6
from botocore.exceptions import ParamValidationError
7
from botocore.exceptions import ClientError
Afraz Ahmadzadeh's avatar
WIP  
Afraz Ahmadzadeh committed
8
from akinaka.client.aws_client import AWS_Client
9
import logging
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
10 11 12 13

aws_client = AWS_Client()

class ASG():
14 15
    """All the methods needed to perform a blue/green deploy"""

16
    def __init__(self, region, role_arn):
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
17 18 19
        self.region = region
        self.role_arn = role_arn

20
    def get_application_name(self, asg, loadbalancer=None, target_group=None):
21
        """
22 23
        Returns the application name that we're deploying, worked out from the target group
        (via the load balancer if applicable)
24
        """
25

26 27
        if asg:
            return asg
28

29
        target_group_arn = self.get_target_group_arn(loadbalancer, target_group)
30 31
        active_asg = self.get_active_asg(target_group_arn)
        asg_split = active_asg.split('-')[0:-1]
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
32

33
        return '-'.join(asg_split)
34

35
    def asgs_by_liveness(self, asg=None, loadbalancer=None, target_group=None):
36 37
        """Return dict of '{inactive: ASG, active: ASG}'"""

38
        if asg is not None:
39
            logging.info("We've been given the ASG name as an argument")
40 41
            # NOTE: "inactive_asg" is a misnomer at this point, since when we already have the ASG
            #       name, it is also the active ASG, because this case is used for non-blue/green ASGs
42
            return {"inactive_asg": asg, "active_asg": asg}
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
43

44
        target_group_arn = self.get_target_group_arn(loadbalancer=loadbalancer, target_group=target_group)
45
        active_asg = self.get_active_asg(target_group_arn)
46 47 48
        inactive_asg = self.get_inactive_asg(active_asg)

        return {"inactive_asg": inactive_asg, "active_asg": active_asg}
49

50
    def scale_down_inactive(self, asg):
51 52
        # NOTE: We're making the heavy presumption here that the _active_ ASG is the one we've been
        #       given from the command line
53
        inactive_asg = self.get_inactive_asg(asg)
54
        self.scale(inactive_asg, 0, 0, 0)
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
55

56 57 58 59 60 61 62 63 64 65 66
    def set_asg_launch_template_version(self, asg, lt_id, lt_version):
        asg_client = aws_client.create_client('autoscaling', self.region, self.role_arn)

        asg_client.update_auto_scaling_group(
            AutoScalingGroupName = asg,
            LaunchTemplate = {
                "LaunchTemplateId": lt_id,
                "Version": lt_version
            }
        )

67 68 69 70 71
    def do_update(self, ami, asg=None, loadbalancer=None, target_group=None):
        asg_liveness_info = self.asgs_by_liveness(asg=asg, loadbalancer=loadbalancer, target_group=target_group)

        inactive_asg = asg_liveness_info['inactive_asg']
        active_asg = asg_liveness_info['active_asg']
72
        new_ami = ami
73
        current_lt_info = self.get_launch_template_info(self.get_lt_name(inactive_asg))
74

75
        logging.info("New ASG was worked out as {}. Now updating it's Launch Template".format(inactive_asg))
76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91

        # Set the current ASG to the current launch template version. Shouldn't actually be necessary,
        # since it's done during the previous deploy, but including for completeness and paranoia
        self.set_asg_launch_template_version(
            asg=active_asg,
            lt_id=current_lt_info["id"],
            lt_version=current_lt_info["version"],
        )

        # Update the lt and set the soon to be new ASG to the new launch template version
        updated_lt = self.update_launch_template(new_ami, self.get_lt_name(inactive_asg))
        self.set_asg_launch_template_version(
            asg=inactive_asg,
            lt_id=updated_lt["id"],
            lt_version=updated_lt["version"]
        )
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
92

93
        scale_to = self.get_current_scale(active_asg)
94

95
        logging.info("Scaling ASG down")
96 97
        self.scale(inactive_asg, 0, 0, 0)
        while not self.asg_is_empty(inactive_asg):
98 99 100 101
            logging.info("Waiting for instances in ASG to terminate")
            sleep(10)

        logging.info("Scaling ASG back up")
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
102
        self.scale(
103
            auto_scaling_group_id = inactive_asg,
104 105
            min_size = scale_to['min'],
            max_size = scale_to['max'],
106
            desired =  scale_to['desired']
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
107
        )
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
108

109
        while len(self.get_auto_scaling_group_instances(inactive_asg)) < 1:
110 111
            logging.info("Waiting for instances in ASG to start ...")
            sleep(10)
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
112

113 114 115 116 117
        logging.info("First instance has started")

        # Try to get information for an instance in the new ASG 20 times
        for i in range(20):
            try:
118
                first_new_instance = self.get_first_new_instance(inactive_asg)
119 120 121 122 123 124
                if i == 20:
                    raise exceptions.AkinakaCriticalException
                break
            except exceptions.AkinakaLoggingError as error:
                logging.info("Retry {}".format(i))
                logging.info("Problem in getting data for first new ASG instance")
125
                logging.info("get_auto_scaling_group_instances() returned: {}".format(self.get_auto_scaling_group_instances(inactive_asg)))
126 127 128
                logging.info("Error was: {}".format(error))

        # Show console output for first instance up until it's Lifecycle Hook has passed
129
        while self.get_auto_scaling_group_instances(auto_scaling_group_id=inactive_asg, instance_ids=[first_new_instance])[0]['LifecycleState'] != "InService":
130 131 132 133 134 135 136 137
            try:
                logging.info("Attempting to retrieve console output from first instance up -- this will not work for non-nitro hypervisor VMs")
                logging.info(self.get_instance_console_output(first_new_instance)['Output'])
            except KeyError:
                logging.info("No output from instance yet. Trying again in 10 seconds.")
            sleep(10)

        # Wait for remaining instances (if any) to come up too
138
        while len(self.asgs_healthy_instances(inactive_asg)) < scale_to['desired']:
139 140
            logging.info("Waiting for all instances to be healthy ...")

141 142
        logging.info("ASG fully healthy. Logging new ASG name to \"inactive_asg.txt\"")
        open("inactive_asg.txt", "w").write(inactive_asg)
143 144 145 146 147 148 149 150 151 152 153 154 155 156

    def get_first_new_instance(self, new_asg):
        asg_instances = self.get_auto_scaling_group_instances(new_asg)
        if len(asg_instances) < 1: raise exceptions.AkinakaLoggingError

        return asg_instances[0]['InstanceId']

    def get_instance_console_output(self, instance):
        ec2_client = aws_client.create_client('ec2', self.region, self.role_arn)
        return ec2_client.get_console_output(InstanceId=instance)

    def instance_state(self, instance):
        ec2_client = aws_client.create_client('ec2', self.region, self.role_arn)
        instance_states = ec2_client.describe_instance_status(IncludeAllInstances=True, InstanceIds=[instance])
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
157

158 159
        while 'InstanceStatuses' not in instance_states:
            instance_states = ec2_client.describe_instance_status(InstanceIds=[instance])
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
160

161
        return instance_states['InstanceStatuses'][0]['InstanceState']['Name']
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
162 163 164 165 166

    def get_lt_name(self, asg):
        asg_client = aws_client.create_client('autoscaling', self.region, self.role_arn)
        lt_info = asg_client.describe_auto_scaling_groups(AutoScalingGroupNames=[asg])

167
        try:
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
168
            return lt_info['AutoScalingGroups'][0]['LaunchTemplate']['LaunchTemplateName']
169 170 171
        except Exception as e:
            raise exceptions.AkinakaCriticalException("{}: Likely couldn't find the ASG you're trying to update".format(e))

172
    def get_target_group_arn(self, loadbalancer=None, target_group=None):
173
        """
174 175 176 177 178 179
        Returns a string containing the ARN of the target group, either by using:

        * targetgroup from --targetgroup
        * loadbalancer from --lb

        Both are mutually exclusive, and at least one must be supplied
180
        """
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
181 182

        alb_client = aws_client.create_client('elbv2', self.region, self.role_arn)
183 184
        if target_group is not None:
            return alb_client.describe_target_groups(Names=[target_group])['TargetGroups'][0]['TargetGroupArn']
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
185

186
        loadbalancer_raw_info = alb_client.describe_load_balancers(Names=[loadbalancer])
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
187 188 189 190 191
        loadbalancer_arn = loadbalancer_raw_info['LoadBalancers'][0]['LoadBalancerArn']

        target_groups_raw_info = alb_client.describe_target_groups(LoadBalancerArn=loadbalancer_arn)['TargetGroups']
        target_group_arns = [targetgroup['TargetGroupArn'] for targetgroup in target_groups_raw_info]

192 193 194 195 196 197 198 199 200 201 202 203 204
        # If we get this far, then the LB has more than a single target group, and we can't work
        # out which one the caller wants
        if len(target_group_arns) == 1:
            return target_group_arns[0]
        elif len(target_group_arns) > 1:
            error_message = "Load balancer has {} target groups".format(len(target_group_arns))

            for target_group in target_group_arns:
                error_message += "> " + target_group
        else:
            error_message = "Load balancer has no target groups"

        raise exceptions.AkinakaCriticalException(error_message)
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
205

206 207
    def get_target_groups_instances(self, target_group_arn):
        """Returns an array of instance IDs belonging to the target group specified"""
208
        alb_client = aws_client.create_client('elbv2', self.region, self.role_arn)
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
209

Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
210 211
        target_groups_instances = []

212
        these_target_group_instances = alb_client.describe_target_health(TargetGroupArn=target_group_arn)['TargetHealthDescriptions']
213 214 215 216
        these_target_group_instances = [
            instance for instance in these_target_group_instances if not instance['TargetHealth']['State'] == "unused"
        ]

217 218 219 220
        # NOTE: This presumes some robustness from your target groups. If they contain instances
        #       from stale ASGs, or anything else is off in them, you will get unexpected results
        for instance in these_target_group_instances:
            target_groups_instances.append(instance['Target']['Id'])
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
221 222

        return target_groups_instances
223

224
    def get_active_asg(self, target_groups):
225
        ec2_client = aws_client.create_client('ec2', self.region, self.role_arn)
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
226

227
        target_groups_instances = self.get_target_groups_instances(target_groups)
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
228

Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
229 230 231
        instances_with_tags = {}

        raw_instances_reservations = ec2_client.describe_instances(InstanceIds=target_groups_instances)['Reservations']
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
232

Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
233 234 235 236 237 238 239 240 241 242 243 244 245
        for reservation in raw_instances_reservations:
            for instance in reservation['Instances']:
                instances_with_tags[instance['InstanceId']] = instance['Tags']

        instances_with_asg = {}

        # Create dictionary with Instance -> AG
        for (key,value) in instances_with_tags.items():
            for tag in value:

                if tag['Key'] == 'aws:autoscaling:groupName':
                    instances_with_asg[key] = tag['Value']

246
        return next(iter(instances_with_asg.values()))
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
247

Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
248
    def get_inactive_asg(self, active_asg):
249 250
        asg_parts = active_asg.split('-')
        active_colour = asg_parts[-1]
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
251 252 253 254 255 256

        if active_colour == "blue":
            inactive_colour = "green"
        else:
            inactive_colour = "blue"

257
        asg_parts[-1] = inactive_colour
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
258

259
        return "-".join(asg_parts)
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
260

261 262 263
    def get_launch_template_info(self, lt_name):
        """Returns the [id] and (latest) [version] number for lt_name"""

Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
264 265 266 267 268 269 270 271 272 273 274 275
        ec2_client = aws_client.create_client('ec2', self.region, self.role_arn)

        response = ec2_client.describe_launch_templates(
            DryRun=False,
            Filters=[
                {
                    'Name': 'launch-template-name',
                    'Values': [ lt_name ]
                }
            ]
        )

276
        # FIXME: This is hackish because we are assuming that there is going to be only 1 (one) launch template
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
277 278
        return {
            "id": response['LaunchTemplates'][0]['LaunchTemplateId'],
279
            "version": str(response['LaunchTemplates'][0]['LatestVersionNumber'])
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
280 281
        }

282 283 284 285
    def update_launch_template(self, ami, lt_name):
        """
        Creates a new template version for [lt_name] which uses [ami], and sets the default
        default template version to be that version.
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
286

287 288 289 290 291
        Returns [id] and (new) [version] number
        """

        lt = self.get_launch_template_info(lt_name)
        lt_client  = aws_client.create_client('ec2', self.region, self.role_arn)
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
292 293 294

        response = lt_client.create_launch_template_version(
            DryRun = False,
295 296
            LaunchTemplateId = lt['id'],
            SourceVersion = str(lt['version']),
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
297 298 299 300 301 302 303 304
            LaunchTemplateData = {
                "ImageId": ami
            }
        )

        launch_template_new_version = str(response['LaunchTemplateVersion']['VersionNumber'])

        lt_client.modify_launch_template(
305
            LaunchTemplateId=lt['id'],
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
306 307 308 309
            DefaultVersion=launch_template_new_version
        )

        return {
310
            "id": lt['id'],
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
311 312 313
            "version": launch_template_new_version
        }

Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
314 315 316
    def scale(self, auto_scaling_group_id, min_size, max_size, desired):
        """Scale an ASG to {'min_size', 'max_size', 'desired'}"""

Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
317 318 319 320 321 322
        asg_client = aws_client.create_client('autoscaling', self.region, self.role_arn)

        response = asg_client.update_auto_scaling_group(
            AutoScalingGroupName=auto_scaling_group_id,
            MinSize=min_size,
            MaxSize=max_size,
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
323
            DesiredCapacity=desired
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
324 325 326 327
        )

        return response

328 329
    def get_current_scale(self, asg):
        """
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
330 331
        Returns the current scales of the given ASG as dict {'desired', 'min', 'max'},
        expects ASG ID as argument
332 333
        """

334 335
        asg_client = aws_client.create_client('autoscaling', self.region, self.role_arn)
        asg = asg_client.describe_auto_scaling_groups(AutoScalingGroupNames=[asg])['AutoScalingGroups'][0]
336

337 338 339 340 341
        return {
            "desired": asg['DesiredCapacity'],
            "min": asg['MinSize'],
            "max": asg['MaxSize']
        }
342

343
    def get_auto_scaling_group_instances(self, auto_scaling_group_id, instance_ids=None):
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
344
        asg_client = aws_client.create_client('autoscaling', self.region, self.role_arn)
345 346
        instance_ids = instance_ids if instance_ids else []
        asg_instances = asg_client.describe_auto_scaling_instances(InstanceIds=instance_ids)
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
347 348 349
        target_instances = []

        for i in asg_instances['AutoScalingInstances']:
350
            if i['AutoScalingGroupName'] == auto_scaling_group_id:
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
351 352
                target_instances.append(i)

353
                logging.debug("Instance {instance_id} has state = {instance_state},"
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
354 355 356 357
                    "Lifecycle is at {instance_lifecycle_state}".format(
                        instance_id = i['InstanceId'],
                        instance_state = i['HealthStatus'],
                        instance_lifecycle_state = i['LifecycleState']
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
358
                    )
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
359
                )
360 361 362 363 364 365

                logging.info("Instance {instance_id} is {instance_lifecycle_state}".format(
                        instance_id = i['InstanceId'],
                        instance_lifecycle_state = i['LifecycleState']
                    )
                )
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
366

Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
367 368

        return target_instances
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
369

370 371 372 373 374 375 376 377
    def asg_is_empty(self, auto_scaling_group_id):
        asg_instances = self.get_auto_scaling_group_instances(auto_scaling_group_id)

        return all(instance.get('LifecycleState') == 'Terminated' for instance in asg_instances)

    def asgs_healthy_instances(self, auto_scaling_group_id):
        asg_instances = self.get_auto_scaling_group_instances(auto_scaling_group_id)
        healthy_instances = []
Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
378

379 380 381 382
        for i in asg_instances:
            if i['LifecycleState'] == "InService":
                healthy_instances.append(i)

Afraz Ahmadzadeh's avatar
Afraz Ahmadzadeh committed
383
        return healthy_instances