logging: Change the source to consistent logging

Signed-off-by: Sayan Chowdhury <[email protected]>
fedora-infra · Apr 13, 2018 · 15e41e5 · 15e41e5
1 parent 3624b4b
commit 15e41e5
Show file tree

Hide file tree

Showing 7 changed files with 54 additions and 49 deletions.
diff --git a/fedimg/consumers.py b/fedimg/consumers.py
@@ -37,7 +37,7 @@
 from fedimg.config import PROCESS_COUNT, STATUS_FILTER
 from fedimg.utils import get_rawxz_urls, get_value_from_dict
 
-LOG = logging.getLogger(__name__)
+_log = logging.getLogger(__name__)
 
 
 class FedimgConsumer(fedmsg.consumers.FedmsgConsumer):
@@ -55,15 +55,15 @@ class FedimgConsumer(fedmsg.consumers.FedmsgConsumer):
     config_key = "fedimgconsumer.prod.enabled"
 
     def __init__(self, *args, **kwargs):
-        LOG.info("FedimgConsumer initializing")
+        _log.info("FedimgConsumer initializing")
         super(FedimgConsumer, self).__init__(*args, **kwargs)
 
         # Threadpool for upload jobs
-        LOG.info("Creating thread pool of %s process", PROCESS_COUNT)
+        _log.info("Creating thread pool of %s process", PROCESS_COUNT)
         self.upload_pool = multiprocessing.pool.ThreadPool(
             processes=PROCESS_COUNT
         )
-        LOG.info("FedimgConsumer initialized")
+        _log.info("FedimgConsumer initialized")
 
     def consume(self, msg):
         """
@@ -72,11 +72,11 @@ def consume(self, msg):
         Args:
             msg (dict): The raw message from fedmsg.
         """
-        LOG.info('Received %r %r', msg['topic'], msg['body']['msg_id'])
+        _log.info('Received %r %r', msg['topic'], msg['body']['msg_id'])
 
         msg_info = msg['body']['msg']
         if msg_info['status'] not in STATUS_FILTER:
-            LOG.debug('%s is not valid status' % msg_info['status'])
+            _log.debug('%s is not valid status' % msg_info['status'])
             return
 
         location = msg_info['location']
@@ -93,12 +93,12 @@ def consume(self, msg):
         )
 
         if images_meta is None:
-            LOG.debug('No compatible image found to process')
+            _log.debug('No compatible image found to process')
             return
 
         upload_urls = get_rawxz_urls(location, images_meta)
         if len(upload_urls) > 0:
-            LOG.info("Start processing compose id: %s", compose_id)
+            _log.info("Start processing compose id: %s", compose_id)
             fedimg.uploader.upload(
                 pool=self.upload_pool,
                 urls=upload_urls,

diff --git a/fedimg/services/ec2/ec2imgpublisher.py b/fedimg/services/ec2/ec2imgpublisher.py
@@ -20,7 +20,7 @@
 #
 
 import logging
-log = logging.getLogger("fedmsg")
+_log = logging.getLogger(__name__)
 
 import re
 
@@ -162,16 +162,16 @@ def publish_images(self, region_image_mapping=None):
         for region, image_id in region_image_mapping:
             self.set_region(region)
 
-            log.info('Publish image (%s) in %s started' % (image_id, region))
+            _log.info('Publish image (%s) in %s started' % (image_id, region))
             image = self._connect().get_image(image_id=image_id)
             is_image_public = self._retry_till_image_is_public(image)
-            log.info('Publish image (%s) in %s completed' % (image_id, region))
+            _log.info('Publish image (%s) in %s completed' % (image_id, region))
 
-            log.info('Publish snaphsot for image (%s) in %s started' % (image_id, region))
+            _log.info('Publish snaphsot for image (%s) in %s started' % (image_id, region))
             snapshot = self.get_snapshot_from_image(image)
-            log.info('Fetched snapshot for image (%s): %s' % (image_id, snapshot.id))
+            _log.info('Fetched snapshot for image (%s): %s' % (image_id, snapshot.id))
             is_snapshot_public = self._retry_till_snapshot_is_public(snapshot)
-            log.info('Publish snaphsot for image (%s) in %s completed' % (image_id, region))
+            _log.info('Publish snaphsot for image (%s) in %s completed' % (image_id, region))
 
             volume_type = self.get_volume_type_from_image(image)
             virt_type = self.get_virt_type_from_image(image)
@@ -217,7 +217,7 @@ def copy_images_to_regions(self, image_id=None, base_region=None, regions=None):
             return []
 
         for region in regions:
-            log.info('Copy %s to %s started' % (image_id, region))
+            _log.info('Copy %s to %s started' % (image_id, region))
             self.set_region(region)
             self.image_name = get_image_name_from_ami_name(image.name, region)
 
@@ -255,20 +255,20 @@ def copy_images_to_regions(self, image_id=None, base_region=None, regions=None):
                             )
                         )
 
-                    log.info('Copy %s to %s is completed.' % (image_id, region))
+                    _log.info('Copy %s to %s is completed.' % (image_id, region))
                     copied_images.append({
                         'region': region,
                         'copied_image_id': copied_image.id
                     })
                     break
 
                 except Exception as e:
-                    log.info('Could not register '
+                    _log.info('Could not register '
                              'with name: %r' % self.image_name)
                     if 'InvalidAMIName.Duplicate' in str(e):
                         counter = counter + 1
                     else:
-                        log.info('Failed')
+                        _log.info('Failed')
                         break
 
         return copied_images

diff --git a/fedimg/services/ec2/ec2imguploader.py b/fedimg/services/ec2/ec2imguploader.py
@@ -20,7 +20,7 @@
 #
 
 import logging
-log = logging.getLogger("fedmsg")
+_log = logging.getLogger(__name__)
 
 import re
 
@@ -85,7 +85,7 @@ def _determine_root_device_name(self):
         if self.image_virtualization_type == 'hvm':
             root_device_name = '/dev/sda1'
 
-        log.debug('Root device name is set to %r for %r' % (
+        _log.debug('Root device name is set to %r for %r' % (
             root_device_name, self.image_virtualization_type))
 
         return root_device_name
@@ -102,7 +102,7 @@ def _create_block_device_map(self, snapshot):
             }
         }
 
-        log.debug('Block device map created for %s' % snapshot.id)
+        _log.debug('Block device map created for %s' % snapshot.id)
 
         return [block_device_map]
 
@@ -116,15 +116,15 @@ def _retry_and_get_volume_id(self, task_id):
             ])
 
             if 'completed' in output:
-                log.debug('Task %r complete. Fetching volume id...' % task_id)
+                _log.debug('Task %r complete. Fetching volume id...' % task_id)
                 match = re.search('\s(vol-\w{17})', output)
                 volume_id = match.group(1)
 
-                log.debug('The id of the created volume: %r' % volume_id)
+                _log.debug('The id of the created volume: %r' % volume_id)
 
                 return volume_id
 
-            log.debug('Failed to find complete. Task %r still running. '
+            _log.debug('Failed to find complete. Task %r still running. '
                       'Sleeping for 10 seconds.' % task_id)
 
     def _create_snapshot(self, volume):
@@ -169,22 +169,22 @@ def _create_volume(self, source):
             ])
 
             if retcode != 0:
-                log.error('Unable to import volume. Out: %s, err: %s, ret: %s',
+                _log.error('Unable to import volume. Out: %s, err: %s, ret: %s',
                           output,
                           err,
                           retcode)
                 raise Exception('Creating the volume failed')
 
-            log.debug('Initiate task to upload the image via S3. '
+            _log.debug('Initiate task to upload the image via S3. '
                       'Fetching task id...')
 
             task_id = get_item_from_regex(output, regex='\s(import-vol-\w{8})')
-            log.info('Fetched task_id: %r. Listening to the task.' % task_id)
+            _log.info('Fetched task_id: %r. Listening to the task.' % task_id)
 
             volume_id = self._retry_and_get_volume_id(task_id)
 
             volume = self.get_volume_from_volume_id(volume_id)
-            log.info('Finish fetching volume object using volume_id')
+            _log.info('Finish fetching volume object using volume_id')
 
             return volume
 
@@ -198,7 +198,7 @@ def _register_image(self, snapshot):
                                          lambda x: str(int(x.group(0))+1),
                                          self.image_name)
             try:
-                log.info('Registering the image in %r (snapshot id: %r) with '
+                _log.info('Registering the image in %r (snapshot id: %r) with '
                          'name %r' % (self.region, snapshot.id,
                                       self.image_name))
                 image = self._connect().ex_register_image(
@@ -232,14 +232,14 @@ def _register_image(self, snapshot):
                 return image
 
             except Exception as e:
-                log.info('Could not register with name: %r' % self.image_name)
+                _log.info('Could not register with name: %r' % self.image_name)
                 if 'InvalidAMIName.Duplicate' in str(e):
                     counter = counter + 1
                 else:
                     raise
 
     def _remove_volume(self, volume):
-        log.info('[CLEAN] Destroying volume: %r' % volume.id)
+        _log.info('[CLEAN] Destroying volume: %r' % volume.id)
         self._connect().destroy_volume(volume)
 
     def clean_up(self, image_id, delete_snapshot=True, force=False):
@@ -256,7 +256,7 @@ def clean_up(self, image_id, delete_snapshot=True, force=False):
             Boolean: True, if resources are deleted else False
         """
         if not AWS_DELETE_RESOURCES and force:
-            log.info('Deleting resource is disabled by config.'
+            _log.info('Deleting resource is disabled by config.'
                      'Override by passing force=True.')
             return False
 
@@ -336,7 +336,7 @@ def create_volume(self, source):
         Args:
             source (str): File path of the source file
         """
-        log.info('Start creating the volume from source: %r' % source)
+        _log.info('Start creating the volume from source: %r' % source)
         return self._create_volume(source)
 
     def create_snapshot(self, source):
@@ -351,7 +351,7 @@ def create_snapshot(self, source):
         """
         self.volume = self._create_volume(source)
 
-        log.info('Start creating snapshot from volume: %r' % self.volume.id)
+        _log.info('Start creating snapshot from volume: %r' % self.volume.id)
         snapshot = self._create_snapshot(self.volume)
 
         self._remove_volume(self.volume)
@@ -384,11 +384,11 @@ def create_image(self, source):
         """
 
         snapshot = self.create_snapshot(source)
-        log.debug('Finished create snapshot: %r' % snapshot.id)
+        _log.debug('Finished create snapshot: %r' % snapshot.id)
 
-        log.info('Start to register the image '
+        _log.info('Start to register the image '
                  'from the snapshot: %r' % snapshot.id)
         image = self.register_image(snapshot)
-        log.debug('Finish registering the image with id: %r' % image.id)
+        _log.debug('Finish registering the image with id: %r' % image.id)
 
         return image
diff --git a/fedimg/services/ec2/ec2initiate.py b/fedimg/services/ec2/ec2initiate.py
@@ -38,7 +38,7 @@
 from fedimg.utils import get_virt_types_from_url, get_source_from_image
 from fedimg.utils import get_image_name_from_image, get_file_arch
 
-LOG = logging.getLogger(__name__)
+_log = logging.getLogger(__name__)
 
 
 def main(image_urls, access_id, secret_key, regions, volume_types=None,
@@ -116,10 +116,10 @@ def main(image_urls, access_id, secret_key, regions, volume_types=None,
                     *[regions, virt_types, volume_types])
             for region, virt_type, volume_type in combinations:
                 uploader.set_region(region)
-                LOG.debug('(uploader) Region is set to: %r' % region)
+                _log.debug('(uploader) Region is set to: %r' % region)
 
                 uploader.set_image_virt_type(virt_type)
-                LOG.debug('(uploader) Virtualization type '
+                _log.debug('(uploader) Virtualization type '
                           'is set to: %r' % virt_type)
 
                 image_name = get_image_name_from_image(
@@ -131,7 +131,7 @@ def main(image_urls, access_id, secret_key, regions, volume_types=None,
                 uploader.set_image_name(image_name)
 
                 uploader.set_image_volume_type(volume_type)
-                LOG.debug('(uploader) Volume type is set to: %r' % volume_type)
+                _log.debug('(uploader) Volume type is set to: %r' % volume_type)
 
                 uploader.set_availability_zone_for_region()
 
@@ -157,7 +157,7 @@ def main(image_urls, access_id, secret_key, regions, volume_types=None,
                     region_image_mapping=[(region, image.id)]
                 ))
         except Exception as e:
-            LOG.debug(e.message)
+            _log.debug(e.message)
             #TODO: Implement the clean up of the images if failed.
             # uploader.clean_up(image_id=image.id, delete_snapshot=True)
 

diff --git a/fedimg/uploader.py b/fedimg/uploader.py
@@ -33,7 +33,7 @@
 from fedimg.config import AWS_BASE_REGION, AWS_REGIONS
 
 
-LOG = logging.getLogger(__name__)
+_log = logging.getLogger(__name__)
 
 
 def upload(pool, urls, *args, **kwargs):
@@ -53,7 +53,7 @@ def upload(pool, urls, *args, **kwargs):
     push_notifications = kwargs.get('push_notifications')
 
     if 'aws' in active_services:
-        LOG.info('Starting to process AWS EC2Service.')
+        _log.info('Starting to process AWS EC2Service.')
         images_metadata = ec2main(
             urls,
             AWS_ACCESS_ID,
@@ -73,4 +73,4 @@ def upload(pool, urls, *args, **kwargs):
                 push_notifications=push_notifications,
                 compose_id=compose_id
             )
-        LOG.info('AWS EC2Service process is completed.')
+        _log.info('AWS EC2Service process is completed.')
diff --git a/fedimg/utils.py b/fedimg/utils.py
@@ -24,7 +24,7 @@
 Utility functions for fedimg.
 """
 import logging
-log = logging.getLogger("fedmsg")
+_log = logging.getLogger(__name__)
 
 import functools
 import os
@@ -99,12 +99,12 @@ def get_value_from_dict(_dict, *keys):
 
 
 def external_run_command(command):
-    log.debug("Starting the command: %r" % command)
+    _log.debug("Starting the command: %r" % command)
     ret = subprocess.Popen(' '.join(command), stdin=subprocess.PIPE, shell=True,
                            stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                            close_fds=True)
     out, err = ret.communicate()
-    log.debug("Finished executing the command: %r" % command)
+    _log.debug("Finished executing the command: %r" % command)
     retcode = ret.returncode
     return out, err, retcode
 
@@ -126,7 +126,7 @@ def get_source_from_image(image_url):
     file_name = get_file_name_image(image_url)
     file_path = os.path.join(tmpdir, file_name)
 
-    log.info("[PREP] Preparing temporary directory for download: %r" % tmpdir)
+    _log.info("[PREP] Preparing temporary directory for download: %r" % tmpdir)
     output, error, retcode = external_run_command([
         'wget',
         image_url,

diff --git a/fedmsg.d/logging.py b/fedmsg.d/logging.py
@@ -20,6 +20,11 @@
             }
         ),
         loggers=dict(
+            fedimg={
+                "level": "DEBUG",
+                "propagate": False,
+                "handlers": ["console"],
+            },
             fedmsg={
                 "level": "INFO",
                 "propagate": False,