Skip to content
This repository has been archived by the owner on Aug 9, 2024. It is now read-only.

Commit

Permalink
Migrate fedimg from koji-based to compose-based
Browse files Browse the repository at this point in the history
  • Loading branch information
@sayanchowdhury
sayanchowdhury committed Jun 7, 2016
1 parent df6919e commit 335d223
Show file tree
Hide file tree
Showing 5 changed files with 66 additions and 138 deletions.
129 changes: 22 additions & 107 deletions fedimg/consumers.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,11 @@

import fedmsg.consumers
import fedmsg.encoding
import fedfind.release
import koji

import fedimg.uploader
from fedimg.util import get_rawxz_url
from fedimg.util import get_rawxz_urls, safeget


class KojiConsumer(fedmsg.consumers.FedmsgConsumer):
Expand All @@ -42,8 +43,7 @@ class KojiConsumer(fedmsg.consumers.FedmsgConsumer):
# build.state.change topic. That means we have to handle both cases like
# this, at least for now.
topic = [
'org.fedoraproject.prod.buildsys.task.state.change', # scratch tasks
'org.fedoraproject.prod.buildsys.build.state.change', # full builds (pungi4)
'org.fedoraproject.prod.pungi.compose.status.change',
]

config_key = 'kojiconsumer'
Expand All @@ -56,118 +56,33 @@ def __init__(self, *args, **kwargs):

log.info("Super happy fedimg ready and reporting for duty.")

def _get_upload_urls(self, builds):
""" Takes a list of koji createImage task IDs and returns a list of
URLs to .raw.xz image files that should be uploaded. """

for build in builds:
log.info('Got Koji build {0}'.format(build))

# Create a Koji connection to the Fedora Koji instance
koji_session = koji.ClientSession(fedimg.KOJI_SERVER)

rawxz_files = [] # list of full URLs of files

# Get all of the .raw.xz URLs for the builds
if len(builds) == 1:
task_result = koji_session.getTaskResult(builds[0])
url = get_rawxz_url(task_result)
if url:
rawxz_files.append(url)
elif len(builds) >= 2:
koji_session.multicall = True
for build in builds:
koji_session.getTaskResult(build)
results = koji_session.multiCall()
for result in results:
if not result: continue
url = get_rawxz_url(result[0])
if url:
rawxz_files.append(url)

# We only want to upload:
# 64 bit: base, atomic, bigdata
# Not uploading 32 bit, vagrant, experimental, or other images.
upload_files = [] # files that will actually be uploaded
for url in rawxz_files:
u = url.lower()
if u.find('x86_64') > -1 and u.find('vagrant') == -1:
if (u.find('fedora-cloud-base') > -1
or u.find('fedora-cloud-atomic') > -1
or u.find('fedora-cloud-bigdata') > -1
or u.find('fedora-atomic') > -1):
upload_files.append(url)
log.info('Image {0} will be uploaded'.format(url))

return upload_files

def consume(self, msg):
""" This is called when we receive a message matching our topics. """

log.info('Received %r %r' % (msg['topic'], msg['body']['msg_id']))

if msg['topic'].endswith('.task.state.change'):
# Scratch tasks.. the old way.
return self._consume_scratch_task(msg)
elif msg['topic'].endswith('.build.state.change'):
# Full builds from pungi4.. the new way.
return self._consume_full_build(msg)
else:
log.error("Unhandled message type received: %r %r" % (
msg['topic'], msg['body']['msg_id']))

def _consume_full_build(self, msg):
""" This is called when we receive a message matching the newer pungi4
full build topic.
"""

builds = list() # These will be the Koji task IDs to upload, if any.

msg = msg['body']['msg']
if msg['owner'] != 'releng':
log.debug("Dropping message. Owned by %r" % msg['owner'])
return

if msg['instance'] != 'primary':
log.info("Dropping message. From %r instance." % msg['instance'])
return

# Don't upload *any* images if one of them fails.
if msg['new'] != 1:
log.info("Dropping message. State is %r" % msg['new'])
return

# Create a Koji connection to the Fedora Koji instance to query.
koji_session = koji.ClientSession(fedimg.KOJI_SERVER)
children = koji_session.getTaskChildren(msg['task_id'])
for child in children:
if child["method"] == "createImage":
builds.append(child["id"])

if len(builds) > 0:
fedimg.uploader.upload(self.upload_pool,
self._get_upload_urls(builds))
STATUS_F = ('FINISHED_INCOMPLETE', 'FINISHED',)

msg_info = msg['body']['msg']
if msg_info['status'] not in STATUS_F:
continue

def _consume_scratch_task(self, msg):
""" This is called when we receive a message matching the older scratch
build topic.
"""
location = msg_info['location']
compose_id = msg_info['compose_id']
cmetadata = fedfind.release.get_release_cid(compose_id)

builds = list() # These will be the Koji task IDs to upload, if any.
images_meta = safeget(cmetadata, 'images', 'payload', 'images',
'CloudImages', 'x86_64')

msg_info = msg["body"]["msg"]["info"]
if images_meta is None:
continue

# If the build method is "image", we check to see if the child
# task's method is "createImage".
if msg_info["method"] == "image":
if isinstance(msg_info["children"], list):
for child in msg_info["children"]:
if child["method"] == "createImage":
# We only care about the image if the build
# completed successfully (with state code 2).
if child["state"] == 2:
builds.append(child["id"])
self.upload_urls = get_rawxz_urls(location, images_meta)
compose_meta = {
'compose_id': compose_id,
}

if len(builds) > 0:
if len(self.upload_urls) > 0:
fedimg.uploader.upload(self.upload_pool,
self._get_upload_urls(builds))
self.upload_urls,
compose_meta)
3 changes: 2 additions & 1 deletion fedimg/messenger.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
"""


def message(topic, image_url, dest, status, extra=None):
def message(topic, image_url, dest, status, compose, extra=None):
""" Takes a message topic, image name, an upload destination (ex.
"EC2-eu-west-1"), and a status (ex. "failed"). Can also take an optional
dictionary of addiitonal bits of information, such as an AMI ID for an
Expand All @@ -44,4 +44,5 @@ def message(topic, image_url, dest, status, extra=None):
'destination': dest,
'status': status,
'extra': extra,
'compose': compose
})
38 changes: 24 additions & 14 deletions fedimg/services/ec2.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ def __init__(self, raw_url, virt_type='hvm', vol_type='standard'):
attrs = line.strip().split('|')

# old configuration
if len(attrs)==6:
if len(attrs)==6:

info = {'region': attrs[0],
'driver': region_to_driver(attrs[0]),
Expand All @@ -104,10 +104,10 @@ def __init__(self, raw_url, virt_type='hvm', vol_type='standard'):
'arch': attrs[3],
'ami': attrs[4],
'aki': attrs[5]}

# new configuration
elif len(attrs)==4:

info = {'region': attrs[0],
'driver': region_to_driver(attrs[0]),
'arch': attrs[1],
Expand Down Expand Up @@ -164,7 +164,7 @@ def _clean_up(self, driver, delete_images=False):
driver.destroy_node(self.test_node)
self.test_node = None

def upload(self):
def upload(self, compose_meta):
""" Registers the image in each EC2 region. """

log.info('EC2 upload process started')
Expand All @@ -174,7 +174,8 @@ def upload(self):
self.destination = 'EC2 ({region})'.format(region=ami['region'])

fedimg.messenger.message('image.upload', self.build_name,
self.destination, 'started')
self.destination, 'started',
compose=compose_meta)

try:
# Connect to the region through the appropriate libcloud driver
Expand Down Expand Up @@ -303,7 +304,8 @@ def upload(self):

fedimg.messenger.message('image.upload', self.build_name,
self.destination, 'failed',
extra={'data': data})
extra={'data': data},
compose=compose_meta)

raise EC2UtilityException(
"Problem writing image to utility instance volume. "
Expand Down Expand Up @@ -428,7 +430,8 @@ def upload(self):
self.destination, 'completed',
extra={'id': image.id,
'virt_type': self.virt_type,
'vol_type': self.vol_type})
'vol_type': self.vol_type},
compose=compose_meta)

# Now, we'll spin up a node of the AMI to test:

Expand All @@ -453,7 +456,8 @@ def upload(self):
self.destination, 'started',
extra={'id': self.images[0].id,
'virt_type': self.virt_type,
'vol_type': self.vol_type})
'vol_type': self.vol_type},
compose=compose_meta)

# Actually deploy the test instance
try:
Expand All @@ -473,7 +477,8 @@ def upload(self):
self.destination, 'failed',
extra={'id': self.images[0].id,
'virt_type': self.virt_type,
'vol_type': self.vol_type})
'vol_type': self.vol_type},
compose=compose_meta)

raise EC2AMITestException("Failed to boot test node %r." % e)

Expand Down Expand Up @@ -515,7 +520,8 @@ def upload(self):
extra={'id': self.images[0].id,
'virt_type': self.virt_type,
'vol_type': self.vol_type,
'data': data})
'data': data},
compose=compose_meta)

raise EC2AMITestException("Tests on AMI failed.\n"
"output: %s" % data)
Expand All @@ -527,7 +533,8 @@ def upload(self):
self.destination, 'completed',
extra={'id': self.images[0].id,
'virt_type': self.virt_type,
'vol_type': self.vol_type})
'vol_type': self.vol_type},
compose=compose_meta)

# Let this EC2Service know that the AMI test passed, so
# it knows how to proceed.
Expand Down Expand Up @@ -589,7 +596,8 @@ def upload(self):

fedimg.messenger.message('image.upload',
self.build_name,
alt_dest, 'started')
alt_dest, 'started',
compose=compose_meta)

# Connect to the libcloud EC2 driver for the region we
# want to copy into
Expand Down Expand Up @@ -651,7 +659,8 @@ def upload(self):
ami['region']))
fedimg.messenger.message('image.upload',
self.build_name,
alt_dest, 'failed')
alt_dest, 'failed',
compose=compose_meta)
break

# Now cycle through and make all of the copied AMIs public
Expand Down Expand Up @@ -696,6 +705,7 @@ def upload(self):
alt_dest, 'completed',
extra={'id': image.id,
'virt_type': self.virt_type,
'vol_type': self.vol_type})
'vol_type': self.vol_type},
compose=compose_meta)

return 0
4 changes: 2 additions & 2 deletions fedimg/uploader.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
from fedimg.util import virt_types_from_url


def upload(pool, urls):
def upload(pool, urls, compose_meta):
""" Takes a list (urls) of one or more .raw.xz image files and
sends them off to cloud services for registration. The upload
jobs threadpool must be passed as `pool`."""
Expand All @@ -44,4 +44,4 @@ def upload(pool, urls):
services.append(EC2Service(url, virt_type=vt,
vol_type='gp2'))

results = pool.map(lambda s: s.upload(), services)
results = pool.map(lambda s: s.upload(compose_meta), services)
30 changes: 16 additions & 14 deletions fedimg/util.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,22 +47,15 @@ def get_file_arch(file_name):
return None


def get_rawxz_url(task_result):
""" Returns the URL of the raw.xz file produced by the Koji task whose
output files are passed as a list via the task_result argument. """
# There should only be one item in this list
rawxz_list = [f for f in task_result['files'] if f.endswith('.raw.xz')]
def get_rawxz_urls(location, images):
""" Iterates through all the images metadata and returns the url of .raw.xz
files.
"""
rawxz_list = [f['path'] for f in images if f['path'].endswith('.raw.xz')]
if not rawxz_list:
return None
file_name = rawxz_list[0]

task_id = task_result['task_id']

# extension to base URL to exact file directory
koji_url_extension = "/{}/{}".format(str(task_id)[-4:], str(task_id))
full_file_location = fedimg.BASE_KOJI_TASK_URL + koji_url_extension
return []

return full_file_location + "/{}".format(file_name)
return map((lambda path: location+path), rawxz_list)


def virt_types_from_url(url):
Expand Down Expand Up @@ -99,3 +92,12 @@ def ssh_connection_works(username, ip, keypath):
pass
ssh.close()
return works


def safeget(dct, *keys):
for key in keys:
try:
dct = dct[key]
except KeyError:
return None
return dct

0 comments on commit 335d223

Please sign in to comment.