Skip to content

Commit

Permalink
Merge pull request #332 from grycap/fix-folders
Browse files Browse the repository at this point in the history 
Fix #331
  • Loading branch information
@srisco
srisco authored Sep 6, 2019
2 parents f6fbc3c + fbb7a0c commit e9016fa
Show file tree
Hide file tree
Showing 3 changed files with 48 additions and 54 deletions.
66 changes: 29 additions & 37 deletions scar/providers/aws/functioncode.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,27 +22,6 @@
from scar.utils import FileUtils, lazy_property, GitHubUtils, \
GITHUB_USER, GITHUB_SUPERVISOR_PROJECT

_SUPERVISOR_ZIP_PATH = FileUtils.join_paths(FileUtils.get_tmp_dir(), 'faas.zip')


def _download_faas_supervisor_zip(supervisor_version: str) -> None:
supervisor_zip_url = GitHubUtils.get_source_code_url(GITHUB_USER, GITHUB_SUPERVISOR_PROJECT,
supervisor_version)
with open(_SUPERVISOR_ZIP_PATH, "wb") as thezip:
thezip.write(get_file(supervisor_zip_url))


def _extract_handler_code(scar_tmp_folder_path: str, handler_name: str) -> None:
function_handler_dest = FileUtils.join_paths(scar_tmp_folder_path, f"{handler_name}.py")
file_path = ""
with ZipFile(_SUPERVISOR_ZIP_PATH) as thezip:
for file in thezip.namelist():
if file.endswith("function_handler.py"):
file_path = FileUtils.join_paths(FileUtils.get_tmp_dir(), file)
thezip.extract(file, FileUtils.get_tmp_dir())
break
FileUtils.copy_file(file_path, function_handler_dest)


_INIT_SCRIPT_NAME = "init_script.sh"

Expand All @@ -53,34 +32,47 @@ class FunctionPackager():
@lazy_property
def udocker(self):
"""Udocker client"""
udocker = Udocker(self.aws, self.scar_tmp_folder_path, _SUPERVISOR_ZIP_PATH)
udocker = Udocker(self.aws, self.scar_tmp_function_folder_path, self._supervisor_zip_path)
return udocker

def __init__(self, aws_properties, supervisor_version):
self.aws = aws_properties
self.supervisor_version = supervisor_version
self.scar_tmp_folder = FileUtils.create_tmp_dir()
self.scar_tmp_folder_path = self.scar_tmp_folder.name
self.scar_tmp_function_folder = FileUtils.create_tmp_dir()
self.scar_tmp_function_folder_path = self.scar_tmp_function_folder.name
self._supervisor_zip_path = FileUtils.join_paths(self.aws.lambdaf.tmp_folder_path, 'faas.zip')

self.package_args = {}

@exception(logger)
def create_zip(self):
"""Creates the lambda function deployment package."""
self._clean_tmp_folders()
_download_faas_supervisor_zip(self.supervisor_version)
_extract_handler_code(self.scar_tmp_folder_path,
self.aws.lambdaf.name)
self._download_faas_supervisor_zip()
self._extract_handler_code()
self._manage_udocker_images()
self._add_init_script()
self._add_extra_payload()
self._zip_scar_folder()
self._check_code_size()
# self._clean_tmp_folders()

def _clean_tmp_folders(self):
FileUtils.delete_file(_SUPERVISOR_ZIP_PATH)
FileUtils.delete_file(self.aws.lambdaf.zip_file_path)
def _download_faas_supervisor_zip(self) -> None:
supervisor_zip_url = GitHubUtils.get_source_code_url(
GITHUB_USER,
GITHUB_SUPERVISOR_PROJECT,
self.supervisor_version)
with open(self._supervisor_zip_path, "wb") as thezip:
thezip.write(get_file(supervisor_zip_url))

def _extract_handler_code(self) -> None:
function_handler_dest = FileUtils.join_paths(self.scar_tmp_function_folder_path, f"{self.aws.lambdaf.name}.py")
file_path = ""
with ZipFile(self._supervisor_zip_path) as thezip:
for file in thezip.namelist():
if file.endswith("function_handler.py"):
file_path = FileUtils.join_paths(self.aws.lambdaf.tmp_folder_path, file)
thezip.extract(file, self.aws.lambdaf.tmp_folder_path)
break
FileUtils.copy_file(file_path, function_handler_dest)

def _manage_udocker_images(self):
if hasattr(self.aws.lambdaf, "image") and \
Expand All @@ -99,26 +91,26 @@ def _add_init_script(self):
self.aws.lambdaf.init_script = FileUtils.join_paths(self.aws.config_path,
self.aws.lambdaf.init_script)
FileUtils.copy_file(self.aws.lambdaf.init_script,
FileUtils.join_paths(self.scar_tmp_folder_path, _INIT_SCRIPT_NAME))
FileUtils.join_paths(self.scar_tmp_function_folder_path, _INIT_SCRIPT_NAME))
self.aws.lambdaf.environment['Variables']['INIT_SCRIPT_PATH'] = \
f"/var/task/{_INIT_SCRIPT_NAME}"

def _add_extra_payload(self):
if hasattr(self.aws.lambdaf, "extra_payload"):
logger.info("Adding extra payload from {0}".format(self.aws.lambdaf.extra_payload))
FileUtils.copy_dir(self.aws.lambdaf.extra_payload, self.scar_tmp_folder_path)
FileUtils.copy_dir(self.aws.lambdaf.extra_payload, self.scar_tmp_function_folder_path)
self.aws.lambdaf.environment['Variables']['EXTRA_PAYLOAD'] = "/var/task"

def _zip_scar_folder(self):
FileUtils.zip_folder(self.aws.lambdaf.zip_file_path,
self.scar_tmp_folder_path,
self.scar_tmp_function_folder_path,
"Creating function package")

def _check_code_size(self):
# Check if the code size fits within the AWS limits
if hasattr(self.aws, "s3") and hasattr(self.aws.s3, "deployment_bucket"):
AWSValidator.validate_s3_code_size(self.scar_tmp_folder_path,
AWSValidator.validate_s3_code_size(self.scar_tmp_function_folder_path,
self.aws.lambdaf.max_s3_payload_size)
else:
AWSValidator.validate_function_code_size(self.scar_tmp_folder_path,
AWSValidator.validate_function_code_size(self.scar_tmp_function_folder_path,
self.aws.lambdaf.max_payload_size)
34 changes: 18 additions & 16 deletions scar/providers/aws/lambdafunction.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,32 +49,34 @@ def __init__(self, aws_properties, supervisor_version):
self._initialize_properties(aws_properties)

def _initialize_properties(self, aws_properties):
self.aws.lambdaf.environment = {'Variables' : {}}
self.aws.lambdaf.zip_file_path = FileUtils.join_paths(FileUtils.get_tmp_dir(), 'function.zip')
self.aws.lambdaf.environment = {'Variables': {}}
self.aws.lambdaf.invocation_type = "RequestResponse"
self.aws.lambdaf.log_type = "Tail"
self.aws.lambdaf.layers = []
self.aws.lambdaf.tmp_folder = FileUtils.create_tmp_dir()
self.aws.lambdaf.tmp_folder_path = self.aws.lambdaf.tmp_folder.name
self.aws.lambdaf.zip_file_path = FileUtils.join_paths(self.aws.lambdaf.tmp_folder_path, 'function.zip')
if hasattr(self.aws.lambdaf, "name"):
self.aws.lambdaf.handler = "{0}.lambda_handler".format(self.aws.lambdaf.name)
if not hasattr(self.aws.lambdaf, "asynchronous"):
self.aws.lambdaf.asynchronous = False
self._set_default_call_parameters()

def _set_default_call_parameters(self):
self.asynchronous_call_parameters = {"invocation_type" : "Event",
"log_type" : "None",
"asynchronous" : "True"}
self.request_response_call_parameters = {"invocation_type" : "RequestResponse",
"log_type" : "Tail",
"asynchronous" : "False"}
self.asynchronous_call_parameters = {"invocation_type": "Event",
"log_type": "None",
"asynchronous": "True"}
self.request_response_call_parameters = {"invocation_type": "RequestResponse",
"log_type": "Tail",
"asynchronous": "False"}

def _get_creations_args(self):
return {'FunctionName' : self.aws.lambdaf.name,
'Runtime' : self.aws.lambdaf.runtime,
'Role' : self.aws.iam.role,
'Handler' : self.aws.lambdaf.handler,
'Code' : self.aws.lambdaf.code,
'Environment' : self.aws.lambdaf.environment,
return {'FunctionName': self.aws.lambdaf.name,
'Runtime': self.aws.lambdaf.runtime,
'Role': self.aws.iam.role,
'Handler': self.aws.lambdaf.handler,
'Code': self.aws.lambdaf.code,
'Environment': self.aws.lambdaf.environment,
'Description': self.aws.lambdaf.description,
'Timeout': self.aws.lambdaf.time,
'MemorySize': self.aws.lambdaf.memory,
Expand Down Expand Up @@ -178,7 +180,7 @@ def _set_function_code(self):
FunctionPackager(self.aws, self.supervisor_version).create_zip()
if hasattr(self.aws, "s3") and hasattr(self.aws.s3, 'deployment_bucket'):
self._upload_to_S3()
self.aws.lambdaf.code = {"S3Bucket": self.aws.s3.deployment_bucket, "S3Key" : self.aws.s3.file_key}
self.aws.lambdaf.code = {"S3Bucket": self.aws.s3.deployment_bucket, "S3Key": self.aws.s3.file_key}
else:
self.aws.lambdaf.code = {"ZipFile": FileUtils.read_file(self.aws.lambdaf.zip_file_path, mode="rb")}

Expand Down Expand Up @@ -252,7 +254,7 @@ def _get_invocation_payload(self):
# Check for defined commands
# This overrides any other function payload
if hasattr(self.aws.lambdaf, "c_args"):
payload = { "cmd_args" : json.dumps(self.aws.lambdaf.c_args) }
payload = {"cmd_args" : json.dumps(self.aws.lambdaf.c_args)}
return json.dumps(payload)

def _invoke_lambda_function(self):
Expand Down
2 changes: 1 addition & 1 deletion scar/version.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,4 +12,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.

__version__ = '3.2.1'
__version__ = '3.2.2'

0 comments on commit e9016fa

Please sign in to comment.