From 13d13ee6a165b44645f92ff3f613b843820c0829 Mon Sep 17 00:00:00 2001 From: Sayan Chowdhury Date: Wed, 12 Jul 2017 20:00:55 +0530 Subject: [PATCH] ec2: Create a script to initiate the complete process Signed-off-by: Sayan Chowdhury --- fedimg/__init__.py | 55 ------------------------ fedimg/config.py | 34 +++++++++++++++ fedimg/consumers.py | 2 +- fedimg/services/ec2/config.py | 37 +++++++++++++++++ fedimg/services/ec2/ec2imguploader.py | 2 +- fedimg/services/ec2/ec2initiate.py | 60 +++++++++++++++++++++++++++ fedimg/uploader.py | 13 +++--- 7 files changed, 141 insertions(+), 62 deletions(-) create mode 100644 fedimg/config.py create mode 100644 fedimg/services/ec2/config.py create mode 100644 fedimg/services/ec2/ec2initiate.py diff --git a/fedimg/__init__.py b/fedimg/__init__.py index b5bf33e..8b13789 100644 --- a/fedimg/__init__.py +++ b/fedimg/__init__.py @@ -1,56 +1 @@ -# This file is part of fedimg. -# Copyright (C) 2014-2017 Red Hat, Inc. -# -# fedimg is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# fedimg is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public -# License along with fedimg; if not, see http://www.gnu.org/licenses, -# or write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -# -# Authors: David Gay -# Sayan Chowdhury - -import toml - -# Read in config file -with open("/etc/fedimg/fedimg-conf.toml") as conffile: - config = toml.loads(conffile.read()) - -# Fedimg Consumer configurations -PROCESS_COUNT = 4 -STATUS_FILTER = ('FINISHED_INCOMPLETE', 'FINISHED') - -CLEAN_UP_ON_FAILURE = config.get('general', 'clean_up_on_failure') -DELETE_IMAGES_ON_FAILURE = config.get('general', 'delete_images_on_failure') - -# AMAZON WEB SERVICES (EC2) -AWS_UTIL_USER = config.get('aws', 'util_username') -AWS_TEST_USER = config.get('aws', 'test_username') -AWS_ACCESS_ID = config.get('aws', 'access_id') -AWS_SECRET_KEY = config.get('aws', 'secret_key') -AWS_VOL_SIZE = config.get('aws', 'volume_size') -AWS_IAM_PROFILE = config.get('aws', 'iam_profile') - -# RACKSPACE -RACKSPACE_USER = config.get('rackspace', 'username') -RACKSPACE_API_KEY = config.get('rackspace', 'api_key') - -# GCE -GCE_EMAIL = config.get('gce', 'email') -GCE_KEYPATH = config.get('gce', 'keypath') -GCE_PROJECT_ID = config.get('gce', 'project_id') - -# HP -HP_USER = config.get('hp', 'username') -HP_PASSWORD = config.get('hp', 'password') -HP_TENANT = config.get('hp', 'tenant') diff --git a/fedimg/config.py b/fedimg/config.py new file mode 100644 index 0000000..58dafdf --- /dev/null +++ b/fedimg/config.py @@ -0,0 +1,34 @@ +# This file is part of fedimg. +# Copyright (C) 2014-2017 Red Hat, Inc. +# +# fedimg is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# fedimg is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public +# License along with fedimg; if not, see http://www.gnu.org/licenses, +# or write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +# +# Authors: David Gay +# Sayan Chowdhury + +import toml + +# Read in config file +with open("/etc/fedimg/fedimg-conf.toml") as conffile: + config = toml.loads(conffile.read()) + +# Fedimg Consumer configurations +PROCESS_COUNT = 4 +STATUS_FILTER = ('FINISHED_INCOMPLETE', 'FINISHED') + +ACTIVE_SERVICES = config.get('general', 'active_services') +CLEAN_UP_ON_FAILURE = config.get('general', 'clean_up_on_failure') +DELETE_IMAGES_ON_FAILURE = config.get('general', 'delete_images_on_failure') diff --git a/fedimg/consumers.py b/fedimg/consumers.py index e891729..e0731fa 100644 --- a/fedimg/consumers.py +++ b/fedimg/consumers.py @@ -67,7 +67,7 @@ def consume(self, msg): upload_urls = get_rawxz_urls(location, images_meta) if len(upload_urls) > 0: log.info("Processing compose id: %s" % compose_id) - fedimg.uploader.upload(self.upload_pool, upload_urls, compose_id) + fedimg.uploader.upload(self.upload_pool, upload_urls) class FedimgStagingConsumer(FedimgConsumer): diff --git a/fedimg/services/ec2/config.py b/fedimg/services/ec2/config.py new file mode 100644 index 0000000..28bc067 --- /dev/null +++ b/fedimg/services/ec2/config.py @@ -0,0 +1,37 @@ +# This file is part of fedimg. +# Copyright (C) 2014-2017 Red Hat, Inc. +# +# fedimg is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# fedimg is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public +# License along with fedimg; if not, see http://www.gnu.org/licenses, +# or write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +# +# Authors: David Gay +# Sayan Chowdhury + + +import toml + +# Read in config file +with open("/etc/fedimg/fedimg-conf.toml") as conffile: + config = toml.loads(conffile.read()) + +UTIL_USER = config.get('aws', 'util_username') +TEST_USER = config.get('aws', 'test_username') +ACCESS_ID = config.get('aws', 'access_id') +SECRET_KEY = config.get('aws', 'secret_key') +VOLUME_SIZE = config.get('aws', 'volume_size') +VOLUME_TYPES = config.get('aws', 'volume_types') +VOLUME_VIA_S3 = config.get('aws', 'volume_via_s3') +IAM_PROFILE = config.get('aws', 'iam_profile') +REGIONS = config.get('aws', 'regions') diff --git a/fedimg/services/ec2/ec2imguploader.py b/fedimg/services/ec2/ec2imguploader.py index ef60697..08fb9d0 100644 --- a/fedimg/services/ec2/ec2imguploader.py +++ b/fedimg/services/ec2/ec2imguploader.py @@ -28,7 +28,7 @@ class EC2ImageUploader(EC2Base): """ Comment goes here """ - def __init__(self, **kwargs): + def __init__(self, *args, **kwargs): defaults = { 'access_key': None, 'image_name': 'Fedora-AMI', diff --git a/fedimg/services/ec2/ec2initiate.py b/fedimg/services/ec2/ec2initiate.py new file mode 100644 index 0000000..2563d15 --- /dev/null +++ b/fedimg/services/ec2/ec2initiate.py @@ -0,0 +1,60 @@ +# This file is part of fedimg. +# Copyright (C) 2014-2017 Red Hat, Inc. +# +# fedimg is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# fedimg is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public +# License along with fedimg; if not, see http://www.gnu.org/licenses, +# or write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +# +# Authors: Sayan Chowdhury +# + +from itertools import product as itertools_product + +from fedimg.ec2.config import ACCESS_ID, SECRET_KEY, REGIONS +from fedimg.ec2.config import VOLUME_TYPES, VOLUME_VIA_S3 +from fedimg.services.ec2.ec2imageuploader import EC2ImageUploader +from fedimg.services.ec2.ec2imagepublisher import EC2ImagePublisher +from fedimg.utils import get_virt_types_from_url, get_source_for_image + + +def main(image_url): + access_id = ACCESS_ID + secret_key = SECRET_KEY + regions = REGIONS + volume_types = VOLUME_TYPES + volume_via_s3 = VOLUME_VIA_S3 + virt_types = get_virt_types_from_url(image_url) + + source = get_source_for_image(image_url) + + uploader = EC2ImageUploader( + access_key=access_id, + secret_key=secret_key, + volume_via_s3=volume_via_s3) + + publisher = EC2ImagePublisher( + access_key=access_id, + secret_key=secret_key) + + combinations = itertools_product(*[regions, virt_types, volume_types]) + for region, virt_type, volume_type in combinations: + uploader.set_region(region) + uploader.set_image_virt_type(virt_type) + uploader.set_image_volume_type(volume_type) + + publisher.set_region(region) + + image = uploader.create_image(source) + + publisher.publish_images(image_ids=[image.id]) diff --git a/fedimg/uploader.py b/fedimg/uploader.py index bf76b7a..d367182 100644 --- a/fedimg/uploader.py +++ b/fedimg/uploader.py @@ -22,12 +22,15 @@ import logging log = logging.getLogger("fedmsg") -from fedimg.services.ec2 import EC2Service -from fedimg.util import virt_types_from_url +from fedimg.config import ACTIVE_SERVICES +from fedimg.services.ec2.ec2initiate import main as ec2main - -def upload(pool, urls, compose_id): +def upload(pool, urls): """ Takes a list (urls) of one or more .raw.xz image files and sends them off to cloud services for registration. The upload jobs threadpool must be passed as `pool`.""" - pass + + active_services = ACTIVE_SERVICES + + if 'aws' in active_services: + ec2main(urls)