From 8ad676e10d1a49d736bdf483281ad007030dcb75 Mon Sep 17 00:00:00 2001 From: Afraz Ahmadzadeh Date: Wed, 18 Dec 2019 09:49:00 +0100 Subject: [PATCH] Added --keep for snapshots to keep --- README.md | 3 ++- akinaka/dr/commands.py | 14 +++++++++++--- akinaka/dr/rds/transfer_snapshot.py | 29 +++++++++++++++++++---------- 3 files changed, 32 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 32bae24..6f20180 100644 --- a/README.md +++ b/README.md @@ -369,7 +369,8 @@ The following further policies need to be attached to the assume roles to backup "rds:ModifyDBClusterSnapshotAttribute", "rds:ModifyDBSnapshotAttribute", "rds:DescribeDBClusters", - "rds:DeleteDBSnapshot" + "rds:DeleteDBSnapshot", + "rds:DeleteDBClusterSnapshot" ], "Resource": "*" } diff --git a/akinaka/dr/commands.py b/akinaka/dr/commands.py index 7994c31..0a5ce92 100644 --- a/akinaka/dr/commands.py +++ b/akinaka/dr/commands.py @@ -71,11 +71,12 @@ def create_kms_key(region, assumable_role_arn): @dr.command() @click.pass_context @click.option("--take-snapshot", is_flag=True, help="Boolean, default false. Take a live snapshot now, or take the existing latest snapshot. Relevant only for RDS") -@click.option("--names", required=False, help="Comma separated list of DB/S3 names to transfer") +@click.option("--names", required=False, help="Comma separated list in quotes of DB/S3 names to transfer") @click.option("--service", type=click.Choice(['rds', 'aurora', 's3']), required=False, help="The service to transfer backups for. Defaults to all (RDS, S3)") @click.option("--retention", required=False, help="Number of days of backups to keep") @click.option("--rotate", is_flag=True, required=False, help="Only rotate backups so [retention] number of days is kept, don't do any actual backups. Relevant for RDS only") -def transfer(ctx, take_snapshot, names, service, retention, rotate): +@click.option("--keep", required=False, help="Comma separated list in quotes. Do not delete these snapshot IDs as part of the rotation policy.") +def transfer(ctx, take_snapshot, names, service, retention, keep, rotate): """ Creates and passes shared KMS keys to the subcommands which wish to tranfer data between eachother. @@ -103,6 +104,9 @@ def transfer(ctx, take_snapshot, names, service, retention, rotate): scanner = scan_resources_storage.ScanResources(region, source_role_arn) db_names = scanner.scan_rds_instances()['db_names'] + if keep: + keep = [keep.replace(' ','')] + rds( dry_run, region, @@ -115,6 +119,7 @@ def transfer(ctx, take_snapshot, names, service, retention, rotate): source_account, destination_account, retention, + keep, rotate) if service == 'aurora': @@ -136,6 +141,7 @@ def transfer(ctx, take_snapshot, names, service, retention, rotate): source_account, destination_account, retention, + keep, rotate) if service == 's3': @@ -199,6 +205,7 @@ def rds( source_account, destination_account, retention, + keep, rotate): """ Call the RDS class to transfer snapshots @@ -223,7 +230,7 @@ def rds( if rotate: for db_name in db_names: - rds.rotate_snapshots(retention, db_name) + rds.rotate_snapshots(retention, db_name, keep) exit() rds.transfer_snapshot( @@ -231,5 +238,6 @@ def rds( db_names=db_names, source_account=source_account, destination_account=destination_account, + keep=keep, retention=retention ) diff --git a/akinaka/dr/rds/transfer_snapshot.py b/akinaka/dr/rds/transfer_snapshot.py index dc31888..c929965 100644 --- a/akinaka/dr/rds/transfer_snapshot.py +++ b/akinaka/dr/rds/transfer_snapshot.py @@ -38,7 +38,7 @@ class TransferSnapshot(): self.source_kms_key = source_kms_key self.destination_kms_key = destination_kms_key - def transfer_snapshot(self, take_snapshot, db_names, source_account, destination_account, retention): + def transfer_snapshot(self, take_snapshot, db_names, source_account, destination_account, keep, retention): """ For every DB in [db_names], call methods to perform the actions listed in this module's docstring. Additionally, rotate the oldest snapshot out, if there are more than [retention] @@ -61,28 +61,37 @@ class TransferSnapshot(): destination_rds_client = aws_client.create_client('rds', self.region, self.destination_role_arn, valid_for=14400) self.recrypt_snapshot(destination_rds_client, recrypted_snapshot, self.destination_kms_key, destination_account) - self.rotate_snapshots(retention, db_name) + self.rotate_snapshots(retention, db_name, keep=None) - def rotate_snapshots(self, retention, db_name): + def rotate_snapshots(self, retention, db_name, keep): """ Get all the snapshots for [db_name], and delete the oldest one if there are more than - [retention] of them. + [retention] of them. Ignore any in the list [keep]. Beware, this does not take distinct days into account, only the number of snapshots. So if you take more than [retention] snapshots in one day, all previous snapshots will be deleted """ + keep = keep or [] + destination_rds_client = aws_client.create_client('rds', self.region, self.destination_role_arn, valid_for=14400) snapshots = destination_rds_client.describe_db_snapshots(DBInstanceIdentifier=db_name)['DBSnapshots'] if len(snapshots) > retention: oldest_snapshot = sorted(snapshots, key=itemgetter('SnapshotCreateTime'))[-1] - logging.info("There are more than the given retention number of snapshots in the account," \ - "so we're going to delete the oldes: {}".format(oldest_snapshot['DBSnapshotIdentifier']) - ) - destination_rds_client.delete_db_snapshot( - DBSnapshotIdentifier=oldest_snapshot['DBSnapshotIdentifier'] - ) + + if oldest_snapshot['DBSnapshotIdentifier'] not in keep: + logging.info("There are more than the given retention number of snapshots in the account," \ + "so we're going to delete the oldest: {}".format(oldest_snapshot['DBSnapshotIdentifier']) + ) + + destination_rds_client.delete_db_snapshot( + DBSnapshotIdentifier=oldest_snapshot['DBSnapshotIdentifier'] + ) + else: + logging.info("Oldest snapshot ({}) is older than" \ + "the retention period allows, but it's " \ + "the --keep list so it will not be deleted".format(oldest_snapshot['DBSnapshotIdentifier'])) def get_latest_snapshot(self, db_name): """ -- GitLab