Added --keep for snapshots to keep

parent f2b5d69a
Pipeline #2462 failed with stage
in 19 seconds
...@@ -369,7 +369,8 @@ The following further policies need to be attached to the assume roles to backup ...@@ -369,7 +369,8 @@ The following further policies need to be attached to the assume roles to backup
"rds:ModifyDBClusterSnapshotAttribute", "rds:ModifyDBClusterSnapshotAttribute",
"rds:ModifyDBSnapshotAttribute", "rds:ModifyDBSnapshotAttribute",
"rds:DescribeDBClusters", "rds:DescribeDBClusters",
"rds:DeleteDBSnapshot" "rds:DeleteDBSnapshot",
"rds:DeleteDBClusterSnapshot"
], ],
"Resource": "*" "Resource": "*"
} }
......
...@@ -71,11 +71,12 @@ def create_kms_key(region, assumable_role_arn): ...@@ -71,11 +71,12 @@ def create_kms_key(region, assumable_role_arn):
@dr.command() @dr.command()
@click.pass_context @click.pass_context
@click.option("--take-snapshot", is_flag=True, help="Boolean, default false. Take a live snapshot now, or take the existing latest snapshot. Relevant only for RDS") @click.option("--take-snapshot", is_flag=True, help="Boolean, default false. Take a live snapshot now, or take the existing latest snapshot. Relevant only for RDS")
@click.option("--names", required=False, help="Comma separated list of DB/S3 names to transfer") @click.option("--names", required=False, help="Comma separated list in quotes of DB/S3 names to transfer")
@click.option("--service", type=click.Choice(['rds', 'aurora', 's3']), required=False, help="The service to transfer backups for. Defaults to all (RDS, S3)") @click.option("--service", type=click.Choice(['rds', 'aurora', 's3']), required=False, help="The service to transfer backups for. Defaults to all (RDS, S3)")
@click.option("--retention", required=False, help="Number of days of backups to keep") @click.option("--retention", required=False, help="Number of days of backups to keep")
@click.option("--rotate", is_flag=True, required=False, help="Only rotate backups so [retention] number of days is kept, don't do any actual backups. Relevant for RDS only") @click.option("--rotate", is_flag=True, required=False, help="Only rotate backups so [retention] number of days is kept, don't do any actual backups. Relevant for RDS only")
def transfer(ctx, take_snapshot, names, service, retention, rotate): @click.option("--keep", required=False, help="Comma separated list in quotes. Do not delete these snapshot IDs as part of the rotation policy.")
def transfer(ctx, take_snapshot, names, service, retention, keep, rotate):
""" """
Creates and passes shared KMS keys to the subcommands which wish to tranfer data between eachother. Creates and passes shared KMS keys to the subcommands which wish to tranfer data between eachother.
...@@ -103,6 +104,9 @@ def transfer(ctx, take_snapshot, names, service, retention, rotate): ...@@ -103,6 +104,9 @@ def transfer(ctx, take_snapshot, names, service, retention, rotate):
scanner = scan_resources_storage.ScanResources(region, source_role_arn) scanner = scan_resources_storage.ScanResources(region, source_role_arn)
db_names = scanner.scan_rds_instances()['db_names'] db_names = scanner.scan_rds_instances()['db_names']
if keep:
keep = [keep.replace(' ','')]
rds( rds(
dry_run, dry_run,
region, region,
...@@ -115,6 +119,7 @@ def transfer(ctx, take_snapshot, names, service, retention, rotate): ...@@ -115,6 +119,7 @@ def transfer(ctx, take_snapshot, names, service, retention, rotate):
source_account, source_account,
destination_account, destination_account,
retention, retention,
keep,
rotate) rotate)
if service == 'aurora': if service == 'aurora':
...@@ -136,6 +141,7 @@ def transfer(ctx, take_snapshot, names, service, retention, rotate): ...@@ -136,6 +141,7 @@ def transfer(ctx, take_snapshot, names, service, retention, rotate):
source_account, source_account,
destination_account, destination_account,
retention, retention,
keep,
rotate) rotate)
if service == 's3': if service == 's3':
...@@ -199,6 +205,7 @@ def rds( ...@@ -199,6 +205,7 @@ def rds(
source_account, source_account,
destination_account, destination_account,
retention, retention,
keep,
rotate): rotate):
""" """
Call the RDS class to transfer snapshots Call the RDS class to transfer snapshots
...@@ -223,7 +230,7 @@ def rds( ...@@ -223,7 +230,7 @@ def rds(
if rotate: if rotate:
for db_name in db_names: for db_name in db_names:
rds.rotate_snapshots(retention, db_name) rds.rotate_snapshots(retention, db_name, keep)
exit() exit()
rds.transfer_snapshot( rds.transfer_snapshot(
...@@ -231,5 +238,6 @@ def rds( ...@@ -231,5 +238,6 @@ def rds(
db_names=db_names, db_names=db_names,
source_account=source_account, source_account=source_account,
destination_account=destination_account, destination_account=destination_account,
keep=keep,
retention=retention retention=retention
) )
...@@ -38,7 +38,7 @@ class TransferSnapshot(): ...@@ -38,7 +38,7 @@ class TransferSnapshot():
self.source_kms_key = source_kms_key self.source_kms_key = source_kms_key
self.destination_kms_key = destination_kms_key self.destination_kms_key = destination_kms_key
def transfer_snapshot(self, take_snapshot, db_names, source_account, destination_account, retention): def transfer_snapshot(self, take_snapshot, db_names, source_account, destination_account, keep, retention):
""" """
For every DB in [db_names], call methods to perform the actions listed in this module's For every DB in [db_names], call methods to perform the actions listed in this module's
docstring. Additionally, rotate the oldest snapshot out, if there are more than [retention] docstring. Additionally, rotate the oldest snapshot out, if there are more than [retention]
...@@ -61,28 +61,37 @@ class TransferSnapshot(): ...@@ -61,28 +61,37 @@ class TransferSnapshot():
destination_rds_client = aws_client.create_client('rds', self.region, self.destination_role_arn, valid_for=14400) destination_rds_client = aws_client.create_client('rds', self.region, self.destination_role_arn, valid_for=14400)
self.recrypt_snapshot(destination_rds_client, recrypted_snapshot, self.destination_kms_key, destination_account) self.recrypt_snapshot(destination_rds_client, recrypted_snapshot, self.destination_kms_key, destination_account)
self.rotate_snapshots(retention, db_name) self.rotate_snapshots(retention, db_name, keep=None)
def rotate_snapshots(self, retention, db_name): def rotate_snapshots(self, retention, db_name, keep):
""" """
Get all the snapshots for [db_name], and delete the oldest one if there are more than Get all the snapshots for [db_name], and delete the oldest one if there are more than
[retention] of them. [retention] of them. Ignore any in the list [keep].
Beware, this does not take distinct days into account, only the number of snapshots. So if you Beware, this does not take distinct days into account, only the number of snapshots. So if you
take more than [retention] snapshots in one day, all previous snapshots will be deleted take more than [retention] snapshots in one day, all previous snapshots will be deleted
""" """
keep = keep or []
destination_rds_client = aws_client.create_client('rds', self.region, self.destination_role_arn, valid_for=14400) destination_rds_client = aws_client.create_client('rds', self.region, self.destination_role_arn, valid_for=14400)
snapshots = destination_rds_client.describe_db_snapshots(DBInstanceIdentifier=db_name)['DBSnapshots'] snapshots = destination_rds_client.describe_db_snapshots(DBInstanceIdentifier=db_name)['DBSnapshots']
if len(snapshots) > retention: if len(snapshots) > retention:
oldest_snapshot = sorted(snapshots, key=itemgetter('SnapshotCreateTime'))[-1] oldest_snapshot = sorted(snapshots, key=itemgetter('SnapshotCreateTime'))[-1]
if oldest_snapshot['DBSnapshotIdentifier'] not in keep:
logging.info("There are more than the given retention number of snapshots in the account," \ logging.info("There are more than the given retention number of snapshots in the account," \
"so we're going to delete the oldes: {}".format(oldest_snapshot['DBSnapshotIdentifier']) "so we're going to delete the oldest: {}".format(oldest_snapshot['DBSnapshotIdentifier'])
) )
destination_rds_client.delete_db_snapshot( destination_rds_client.delete_db_snapshot(
DBSnapshotIdentifier=oldest_snapshot['DBSnapshotIdentifier'] DBSnapshotIdentifier=oldest_snapshot['DBSnapshotIdentifier']
) )
else:
logging.info("Oldest snapshot ({}) is older than" \
"the retention period allows, but it's " \
"the --keep list so it will not be deleted".format(oldest_snapshot['DBSnapshotIdentifier']))
def get_latest_snapshot(self, db_name): def get_latest_snapshot(self, db_name):
""" """
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment