Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use context managers for templating changes in Bundle easyblock #3547

Open
wants to merge 1 commit into
base: 5.0.x
Choose a base branch
from
Open

Use context managers for templating changes in Bundle easyblock #3547

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
110 changes: 52 additions & 58 deletions easybuild/easyblocks/generic/bundle.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,26 +90,27 @@ def __init__(self, *args, **kwargs):
self.cfg = self.cfg.copy()

# disable templating to avoid premature resolving of template values
self.cfg.enable_templating = False

# list of checksums for patches (must be included after checksums for sources)
checksums_patches = []

if self.cfg['sanity_check_components'] and self.cfg['sanity_check_all_components']:
raise EasyBuildError("sanity_check_components and sanity_check_all_components cannot be enabled together")

# backup and reset general sanity checks from main body of ec, if component-specific sanity checks are enabled
# necessary to avoid:
# - duplicating the general sanity check across all components running sanity checks
# - general sanity checks taking precedence over those defined in a component's easyblock
self.backup_sanity_paths = self.cfg['sanity_check_paths']
self.backup_sanity_cmds = self.cfg['sanity_check_commands']
if self.cfg['sanity_check_components'] or self.cfg['sanity_check_all_components']:
# reset general sanity checks, to be restored later
self.cfg['sanity_check_paths'] = {}
self.cfg['sanity_check_commands'] = {}

for comp in self.cfg['components']:
with self.cfg.disable_templating():
# list of checksums for patches (must be included after checksums for sources)
checksums_patches = []

if self.cfg['sanity_check_components'] and self.cfg['sanity_check_all_components']:
raise EasyBuildError("sanity_check_components and sanity_check_all_components"
"cannot be enabled together")

# backup and reset general sanity checks from main body of ec,
# if component-specific sanity checks are enabled necessary to avoid:
# - duplicating the general sanity check across all components running sanity checks
# - general sanity checks taking precedence over those defined in a component's easyblock
self.backup_sanity_paths = self.cfg['sanity_check_paths']
self.backup_sanity_cmds = self.cfg['sanity_check_commands']
if self.backup_sanity_paths or self.backup_sanity_cmds:
# reset general sanity checks, to be restored later
self.cfg['sanity_check_paths'] = {}
self.cfg['sanity_check_commands'] = {}
components = self.cfg['components']

for comp in components:
comp_name, comp_version, comp_specs = comp[0], comp[1], {}
if len(comp) == 3:
comp_specs = comp[2]
Expand Down Expand Up @@ -161,36 +162,31 @@ def __init__(self, *args, **kwargs):
for key in comp_specs:
comp_cfg[key] = comp_specs[key]

# enable resolving of templates for component-specific EasyConfig instance,
# but don't require that all template values can be resolved at this point;
# this is important to ensure that template values like %(name)s and %(version)s
# Don't require that all template values can be resolved at this point but still resolve them.
# This is important to ensure that template values like %(name)s and %(version)s
# are correctly resolved with the component name/version before values are copied over to self.cfg
comp_cfg.enable_templating = True
comp_cfg.expect_resolved_template_values = False

# 'sources' is strictly required
comp_sources = comp_cfg['sources']
if comp_sources:
# If per-component source URLs are provided, attach them directly to the relevant sources
comp_source_urls = comp_cfg['source_urls']
if comp_source_urls:
for source in comp_sources:
if isinstance(source, str):
self.cfg.update('sources', [{'filename': source, 'source_urls': comp_source_urls[:]}])
elif isinstance(source, dict):
# Update source_urls in the 'source' dict to use the one for the components
# (if it doesn't already exist)
if 'source_urls' not in source:
source['source_urls'] = comp_source_urls[:]
self.cfg.update('sources', [source])
else:
raise EasyBuildError("Source %s for component %s is neither a string nor a dict, cannot "
"process it.", source, comp_cfg['name'])
else:
# add component sources to list of sources
self.cfg.update('sources', comp_sources)
else:
with comp_cfg.allow_unresolved_templates():
comp_sources = comp_cfg['sources']
if not comp_sources:
raise EasyBuildError("No sources specification for component %s v%s", comp_name, comp_version)
# If per-component source URLs are provided, attach them directly to the relevant sources
comp_source_urls = comp_cfg['source_urls']
if comp_source_urls:
for source in comp_sources:
if isinstance(source, str):
self.cfg.update('sources', [{'filename': source, 'source_urls': comp_source_urls[:]}])
elif isinstance(source, dict):
# Update source_urls in the 'source' dict to use the one for the components
# (if it doesn't already exist)
if 'source_urls' not in source:
source['source_urls'] = comp_source_urls[:]
self.cfg.update('sources', [source])
else:
raise EasyBuildError("Source %s for component %s is neither a string nor a dict, cannot "
"process it.", source, comp_cfg['name'])
else:
# add component sources to list of sources
self.cfg.update('sources', comp_sources)

comp_checksums = comp_cfg['checksums']
if comp_checksums:
Expand All @@ -202,23 +198,21 @@ def __init__(self, *args, **kwargs):
# add per-component checksums for patches to list of checksums for patches
checksums_patches.extend(comp_checksums[src_cnt:])

comp_patches = comp_cfg['patches']
with comp_cfg.allow_unresolved_templates():
comp_patches = comp_cfg['patches']
if comp_patches:
self.cfg.update('patches', comp_patches)

comp_cfg.expect_resolved_template_values = True

self.comp_instances.append((comp_cfg, comp_cfg.easyblock(comp_cfg, logfile=self.logfile)))

self.cfg.update('checksums', checksums_patches)

# restore general sanity checks if using component-specific sanity checks
if self.cfg['sanity_check_components'] or self.cfg['sanity_check_all_components']:
if self.cfg.get('sanity_check_components', resolve=False) or self.cfg.get('sanity_check_all_components',
resolve=False):
self.cfg['sanity_check_paths'] = self.backup_sanity_paths
self.cfg['sanity_check_commands'] = self.backup_sanity_cmds

self.cfg.enable_templating = True

def check_checksums(self):
"""
Check whether a SHA256 checksum is available for all sources & patches (incl. extensions).
Expand Down Expand Up @@ -278,9 +272,10 @@ def install_step(self):

comp.src = []

# find match entries in self.src for this component
comp.cfg.expect_resolved_template_values = False
for source in comp.cfg['sources']:
# find matching entries in self.src for this component
with comp.cfg.allow_unresolved_templates():
comp_sources = comp.cfg['sources']
for source in comp_sources:
if isinstance(source, str):
comp_src_fn = source
elif isinstance(source, dict):
Expand All @@ -303,7 +298,6 @@ def install_step(self):

# location of first unpacked source is used to determine where to apply patch(es)
comp.src[-1]['finalpath'] = comp.cfg['start_dir']
comp.cfg.expect_resolved_template_values = True

# check if sanity checks are enabled for the component
if self.cfg['sanity_check_all_components'] or comp.cfg['name'] in self.cfg['sanity_check_components']:
Expand Down