ec2: Create a script to initiate the complete process

Signed-off-by: Sayan Chowdhury <[email protected]>

fedimg/__init__.py

@@ -1,56 +1 @@
-# This file is part of fedimg.
-# Copyright (C) 2014-2017 Red Hat, Inc.
-#
-# fedimg is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# fedimg is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public
-# License along with fedimg; if not, see http://www.gnu.org/licenses,
-# or write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-#
-# Authors:  David Gay <[email protected]>
-#           Sayan Chowdhury <[email protected]>
-
-import toml
-
-# Read in config file
-with open("/etc/fedimg/fedimg-conf.toml") as conffile:
-    config = toml.loads(conffile.read())
-
-# Fedimg Consumer configurations
-PROCESS_COUNT = 4
-STATUS_FILTER = ('FINISHED_INCOMPLETE', 'FINISHED')
-
-CLEAN_UP_ON_FAILURE = config.get('general', 'clean_up_on_failure')
-DELETE_IMAGES_ON_FAILURE = config.get('general', 'delete_images_on_failure')
-
-# AMAZON WEB SERVICES (EC2)
-AWS_UTIL_USER = config.get('aws', 'util_username')
-AWS_TEST_USER = config.get('aws', 'test_username')
-AWS_ACCESS_ID = config.get('aws', 'access_id')
-AWS_SECRET_KEY = config.get('aws', 'secret_key')
-AWS_VOL_SIZE = config.get('aws', 'volume_size')
-AWS_IAM_PROFILE = config.get('aws', 'iam_profile')
-
-# RACKSPACE
-RACKSPACE_USER = config.get('rackspace', 'username')
-RACKSPACE_API_KEY = config.get('rackspace', 'api_key')
-
-# GCE
-GCE_EMAIL = config.get('gce', 'email')
-GCE_KEYPATH = config.get('gce', 'keypath')
-GCE_PROJECT_ID = config.get('gce', 'project_id')
-
-# HP
-HP_USER = config.get('hp', 'username')
-HP_PASSWORD = config.get('hp', 'password')
-HP_TENANT = config.get('hp', 'tenant')
 

fedimg/config.py

@@ -0,0 +1,34 @@
+# This file is part of fedimg.
+# Copyright (C) 2014-2017 Red Hat, Inc.
+#
+# fedimg is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# fedimg is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public
+# License along with fedimg; if not, see http://www.gnu.org/licenses,
+# or write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+#
+# Authors:  David Gay <[email protected]>
+#           Sayan Chowdhury <[email protected]>
+
+import toml
+
+# Read in config file
+with open("/etc/fedimg/fedimg-conf.toml") as conffile:
+    config = toml.loads(conffile.read())
+
+# Fedimg Consumer configurations
+PROCESS_COUNT = 4
+STATUS_FILTER = ('FINISHED_INCOMPLETE', 'FINISHED')
+
+ACTIVE_SERVICES = config.get('general', 'active_services')
+CLEAN_UP_ON_FAILURE = config.get('general', 'clean_up_on_failure')
+DELETE_IMAGES_ON_FAILURE = config.get('general', 'delete_images_on_failure')

fedimg/consumers.py

@@ -67,7 +67,7 @@ def consume(self, msg):
         upload_urls = get_rawxz_urls(location, images_meta)
         if len(upload_urls) > 0:
             log.info("Processing compose id: %s" % compose_id)
-            fedimg.uploader.upload(self.upload_pool, upload_urls, compose_id)
+            fedimg.uploader.upload(self.upload_pool, upload_urls)
 
 
 class FedimgStagingConsumer(FedimgConsumer):

fedimg/services/ec2/config.py

@@ -0,0 +1,37 @@
+# This file is part of fedimg.
+# Copyright (C) 2014-2017 Red Hat, Inc.
+#
+# fedimg is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# fedimg is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public
+# License along with fedimg; if not, see http://www.gnu.org/licenses,
+# or write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+#
+# Authors:  David Gay <[email protected]>
+#           Sayan Chowdhury <[email protected]>
+
+
+import toml
+
+# Read in config file
+with open("/etc/fedimg/fedimg-conf.toml") as conffile:
+    config = toml.loads(conffile.read())
+
+UTIL_USER = config.get('aws', 'util_username')
+TEST_USER = config.get('aws', 'test_username')
+ACCESS_ID = config.get('aws', 'access_id')
+SECRET_KEY = config.get('aws', 'secret_key')
+VOLUME_SIZE = config.get('aws', 'volume_size')
+VOLUME_TYPES = config.get('aws', 'volume_types')
+VOLUME_VIA_S3 = config.get('aws', 'volume_via_s3')
+IAM_PROFILE = config.get('aws', 'iam_profile')
+REGIONS = config.get('aws', 'regions')

fedimg/services/ec2/ec2imguploader.py

@@ -28,7 +28,7 @@
 class EC2ImageUploader(EC2Base):
     """ Comment goes here """
 
-    def __init__(self, **kwargs):
+    def __init__(self, *args, **kwargs):
         defaults = {
             'access_key': None,
             'image_name': 'Fedora-AMI',

fedimg/services/ec2/ec2initiate.py

@@ -0,0 +1,60 @@
+# This file is part of fedimg.
+# Copyright (C) 2014-2017 Red Hat, Inc.
+#
+# fedimg is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# fedimg is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public
+# License along with fedimg; if not, see http://www.gnu.org/licenses,
+# or write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+#
+# Authors:  Sayan Chowdhury <[email protected]>
+#
+
+from itertools import product as itertools_product
+
+from fedimg.ec2.config import ACCESS_ID, SECRET_KEY, REGIONS
+from fedimg.ec2.config import VOLUME_TYPES, VOLUME_VIA_S3
+from fedimg.services.ec2.ec2imageuploader import EC2ImageUploader
+from fedimg.services.ec2.ec2imagepublisher import EC2ImagePublisher
+from fedimg.utils import get_virt_types_from_url, get_source_for_image
+
+
+def main(image_url):
+    access_id = ACCESS_ID
+    secret_key = SECRET_KEY
+    regions = REGIONS
+    volume_types = VOLUME_TYPES
+    volume_via_s3 = VOLUME_VIA_S3
+    virt_types = get_virt_types_from_url(image_url)
+
+    source = get_source_for_image(image_url)
+
+    uploader = EC2ImageUploader(
+            access_key=access_id,
+            secret_key=secret_key,
+            volume_via_s3=volume_via_s3)
+
+    publisher = EC2ImagePublisher(
+            access_key=access_id,
+            secret_key=secret_key)
+
+    combinations = itertools_product(*[regions, virt_types, volume_types])
+    for region, virt_type, volume_type in combinations:
+        uploader.set_region(region)
+        uploader.set_image_virt_type(virt_type)
+        uploader.set_image_volume_type(volume_type)
+
+        publisher.set_region(region)
+
+        image = uploader.create_image(source)
+
+        publisher.publish_images(image_ids=[image.id])

fedimg/uploader.py

@@ -22,12 +22,15 @@
 import logging
 log = logging.getLogger("fedmsg")
 
-from fedimg.services.ec2 import EC2Service
-from fedimg.util import virt_types_from_url
+from fedimg.config import ACTIVE_SERVICES
+from fedimg.services.ec2.ec2initiate import main as ec2main
 
-
-def upload(pool, urls, compose_id):
+def upload(pool, urls):
     """ Takes a list (urls) of one or more .raw.xz image files and
     sends them off to cloud services for registration. The upload
     jobs threadpool must be passed as `pool`."""
-    pass
+
+    active_services = ACTIVE_SERVICES
+
+    if 'aws' in active_services:
+        ec2main(urls)