From d5b086c81952aaf78dd7febfedc6b424e63e3697 Mon Sep 17 00:00:00 2001 From: Nicholas Chammas Date: Fri, 19 Feb 2016 22:01:21 -0500 Subject: [PATCH] error out if spot request fails --- flintrock/ec2.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/flintrock/ec2.py b/flintrock/ec2.py index 17f60db4..7144aeef 100644 --- a/flintrock/ec2.py +++ b/flintrock/ec2.py @@ -340,6 +340,8 @@ def get_ec2_block_device_mappings( ec2 = boto3.resource(service_name='ec2', region_name=region) block_device_mappings = [] + # An IndexError here is probably a sign of this problem: + # https://github.com/boto/boto3/issues/496 image = list( ec2.images.filter(ImageIds=[ami]))[0] @@ -453,9 +455,19 @@ def launch( time.sleep(30) spot_requests = client.describe_spot_instance_requests( SpotInstanceRequestIds=request_ids)['SpotInstanceRequests'] + + failed_requests = [r for r in spot_requests if r['State'] == 'failed'] + if failed_requests: + failure_reasons = {r['Status']['Code'] for r in failed_requests} + raise Error( + "The spot request failed for the following reason{s}: {reasons}" + .format( + s='' if len(failure_reasons) == 1 else 's', + reasons=', '.join(failure_reasons))) + pending_request_ids = [ r['SpotInstanceRequestId'] for r in spot_requests - if r['State'] != 'active'] + if r['State'] == 'open'] print("All {c} instances granted.".format(c=num_instances)) @@ -518,7 +530,7 @@ def launch( except (Exception, KeyboardInterrupt) as e: # TODO: Cleanup cluster security group here. - print(e, file=sys.stderr) + print("There was a problem with the launch. Cleaning up...", file=sys.stderr) if spot_requests: request_ids = [r['SpotInstanceRequestId'] for r in spot_requests]