[AWS Batch] Eliminate the need for access and secret keys in the conf…

…iguration

CHANGELOG.md

@@ -6,7 +6,9 @@
 - 
 
 ### Changed
-- 
+- [AWS Lambda] Eliminate the need for access and secret keys in the configuration
+- [AWS Batch] Eliminate the need for access and secret keys in the configuration
+- [AWS S3] Eliminate the need for access and secret keys in the configuration
 
 ### Fixed
 - [AWS Lambda] Fixed runtime deletion with "lithops runtime delete"

docs/index.rst

@@ -1,12 +1,6 @@
 What is Lithops?
 ****************
 
-.. image:: source/images/lithops_logo_readme.png
-   :alt: Lithops
-   :align: center
-
-|
-
 **Lithops is a Python multi-cloud serverless computing framework. It allows to run unmodified local python code at massive scale in the main serverless computing platforms.**
 
 Lithops delivers the user’s code into the cloud without requiring knowledge of how it is deployed and run.
@@ -28,6 +22,18 @@ analytics, to name a few.
 Lithops abstracts away the underlying cloud-specific APIs for accessing storage and provides an intuitive and easy to use interface to process high volumes of data.
 
 
+Use any Cloud
+*************
+**Lithops provides an extensible backend architecture that is designed to work with different compute and storage services available on Cloud providers and on-premise backends.**
+
+In this sense, you can code your application in Python and run it unmodified wherever your data is located at: IBM Cloud, AWS, Azure, Google Cloud and Alibaba Aliyun...
+
+.. image:: source/images/multicloud.jpg
+   :alt: Available backends
+   :align: center
+
+|
+
 Quick Start
 ***********
 
@@ -50,17 +56,6 @@ You're ready to execute a simple example!
         fut = fexec.call_async(hello, 'World')
         print(fut.result())
 
-Use any Cloud
-*************
-**Lithops provides an extensible backend architecture that is designed to work with different compute and storage services available on Cloud providers and on-premise backends.**
-
-In this sense, you can code your application in Python and run it unmodified wherever your data is located at: IBM Cloud, AWS, Azure, Google Cloud and Alibaba Aliyun...
-
-.. image:: source/images/multicloud.jpg
-   :alt: Available backends
-   :align: center
-
-|
 
 Additional resources
 ********************

docs/source/compute_backends.rst

@@ -19,7 +19,6 @@ Compute Backends
    compute_config/oracle_functions.md
    compute_config/aliyun_functions.md
    compute_config/openwhisk.md
-   compute_config/ibm_cf.md
 
 **Serverless (CaaS) Backends:**
 

docs/source/compute_config/azure_vms.md

@@ -1,4 +1,4 @@
-# Azure Virtual Machines (Beta)
+# Azure Virtual Machines
 
 The Azure Virtual Machines client of Lithops can provide a truely serverless user experience on top of Azure VMs where Lithops creates new Virtual Machines (VMs) dynamically in runtime and scale Lithops jobs against them. Alternatively Lithops can start and stop an existing VM instances.
 

docs/source/compute_config/oracle_functions.md

@@ -1,4 +1,4 @@
-# Oracle Functions (beta)
+# Oracle Functions
 
 Lithops with *Oracle Functions* as serverless compute backend.
 

lithops/serverless/backends/aws_batch/aws_batch.py

@@ -45,34 +45,43 @@ def __init__(self, aws_batch_config, internal_storage):
         self.name = 'aws_batch'
         self.type = utils.BackendType.BATCH.value
         self.aws_batch_config = aws_batch_config
-
-        self.user_key = aws_batch_config['access_key_id'][-4:]
-        self.package = f'lithops_v{__version__.replace(".", "-")}_{self.user_key}'
-        self.region_name = aws_batch_config['region']
+        self.region = aws_batch_config['region']
 
         self._env_type = self.aws_batch_config['env_type']
         self._queue_name = f'{self.package}_{self._env_type.replace("_", "-")}_queue'
         self._compute_env_name = f'{self.package}_{self._env_type.replace("_", "-")}_env'
 
         logger.debug('Creating Boto3 AWS Session and Batch Client')
-        self.aws_session = boto3.Session(aws_access_key_id=aws_batch_config['access_key_id'],
-                                         aws_secret_access_key=aws_batch_config['secret_access_key'],
-                                         aws_session_token=aws_batch_config.get('session_token'),
-                                         region_name=self.region_name)
-        self.batch_client = self.aws_session.client('batch', region_name=self.region_name)
+        self.aws_session = boto3.Session(
+            aws_access_key_id=aws_batch_config.get('access_key_id'),
+            aws_secret_access_key=aws_batch_config.get('secret_access_key'),
+            aws_session_token=aws_batch_config.get('session_token'),
+            region_name=self.region
+        )
+        self.batch_client = self.aws_session.client('batch', region_name=self.region)
 
         self.internal_storage = internal_storage
 
         if 'account_id' in self.aws_batch_config:
             self.account_id = self.aws_batch_config['account_id']
         else:
-            sts_client = self.aws_session.client('sts', region_name=self.region_name)
+            sts_client = self.aws_session.client('sts', region_name=self.region)
             self.account_id = sts_client.get_caller_identity()["Account"]
 
-        self.ecr_client = self.aws_session.client('ecr', region_name=self.region_name)
+        sts_client = self.aws_session.client('sts', region_name=self.region)
+        caller_id = sts_client.get_caller_identity()
+
+        if ":" in caller_id["UserId"]:  # SSO user
+            self.user_key = caller_id["UserId"].split(":")[1]
+        else:  # IAM user
+            self.user_key = caller_id["UserId"][-4:].lower()
+
+        self.ecr_client = self.aws_session.client('ecr', region_name=self.region)
+        package = f'lithops_v{__version__.replace(".", "")}_{self.user_key}'
+        self.package = f"{package}_{self.namespace}" if self.namespace else package
 
         msg = COMPUTE_CLI_MSG.format('AWS Batch')
-        logger.info("{} - Region: {}".format(msg, self.region_name))
+        logger.info(f"{msg} - Region: {self.region}")
 
     def _get_default_runtime_image_name(self):
         python_version = utils.CURRENT_PY_VERSION.replace('.', '')
@@ -81,7 +90,7 @@ def _get_default_runtime_image_name(self):
 
     def _get_full_image_name(self, runtime_name):
         full_image_name = runtime_name if ':' in runtime_name else f'{runtime_name}:latest'
-        registry = f'{self.account_id}.dkr.ecr.{self.region_name}.amazonaws.com'
+        registry = f'{self.account_id}.dkr.ecr.{self.region}.amazonaws.com'
         full_image_name = '/'.join([registry, self.package.replace('-', '.'), full_image_name]).lower()
         repo_name = full_image_name.split('/', 1)[1:].pop().split(':')[0]
         return full_image_name, registry, repo_name
@@ -585,7 +594,7 @@ def invoke(self, runtime_name, runtime_memory, payload):
 
     def get_runtime_key(self, runtime_name, runtime_memory, version=__version__):
         jobdef_name = self._format_jobdef_name(runtime_name, runtime_memory, version)
-        runtime_key = os.path.join(self.name, version, self.region_name, jobdef_name)
+        runtime_key = os.path.join(self.name, version, self.region, jobdef_name)
         return runtime_key
 
     def get_runtime_info(self):

lithops/serverless/backends/aws_batch/config.py

@@ -76,9 +76,6 @@ def load_config(config_data):
     if 'aws' not in config_data:
         raise Exception("'aws' section is mandatory in the configuration")
 
-    if not {'access_key_id', 'secret_access_key'}.issubset(set(config_data['aws'])):
-        raise Exception("'access_key_id' and 'secret_access_key' are mandatory under the 'aws' section of the configuration")
-
     if not config_data['aws_batch']:
         raise Exception("'aws_batch' section is mandatory in the configuration")
 

lithops/serverless/backends/aws_lambda/aws_lambda.py

@@ -57,7 +57,7 @@ def __init__(self, lambda_config, internal_storage):
         self.lambda_config = lambda_config
         self.internal_storage = internal_storage
         self.user_agent = lambda_config['user_agent']
-        self.region_name = lambda_config['region']
+        self.region = lambda_config['region']
         self.role_arn = lambda_config['execution_role']
         self.namespace = lambda_config.get('namespace')
 
@@ -67,43 +67,43 @@ def __init__(self, lambda_config, internal_storage):
             aws_access_key_id=lambda_config.get('access_key_id'),
             aws_secret_access_key=lambda_config.get('secret_access_key'),
             aws_session_token=lambda_config.get('session_token'),
-            region_name=self.region_name
+            region_name=self.region
         )
 
         self.lambda_client = self.aws_session.client(
-            'lambda', region_name=self.region_name,
+            'lambda', region_name=self.region,
             config=botocore.client.Config(
                 user_agent_extra=self.user_agent
             )
         )
 
         self.credentials = self.aws_session.get_credentials()
         self.session = URLLib3Session()
-        self.host = f'lambda.{self.region_name}.amazonaws.com'
+        self.host = f'lambda.{self.region}.amazonaws.com'
 
         if 'account_id' in self.lambda_config:
             self.account_id = self.lambda_config['account_id']
         else:
-            sts_client = self.aws_session.client('sts', region_name=self.region_name)
+            sts_client = self.aws_session.client('sts', region_name=self.region)
             self.account_id = sts_client.get_caller_identity()["Account"]
 
-        sts_client = self.aws_session.client('sts', region_name=self.region_name)
+        sts_client = self.aws_session.client('sts', region_name=self.region)
         caller_id = sts_client.get_caller_identity()
 
         if ":" in caller_id["UserId"]:  # SSO user
             self.user_key = caller_id["UserId"].split(":")[1]
         else:  # IAM user
             self.user_key = caller_id["UserId"][-4:].lower()
 
-        self.ecr_client = self.aws_session.client('ecr', region_name=self.region_name)
+        self.ecr_client = self.aws_session.client('ecr', region_name=self.region)
         package = f'lithops_v{__version__.replace(".", "")}_{self.user_key}'
         self.package = f"{package}_{self.namespace}" if self.namespace else package
 
         msg = COMPUTE_CLI_MSG.format('AWS Lambda')
         if self.namespace:
-            logger.info(f"{msg} - Region: {self.region_name} - Namespace: {self.namespace}")
+            logger.info(f"{msg} - Region: {self.region} - Namespace: {self.namespace}")
         else:
-            logger.info(f"{msg} - Region: {self.region_name}")
+            logger.info(f"{msg} - Region: {self.region}")
 
     def _format_function_name(self, runtime_name, runtime_memory, version=__version__):
         name = f'{runtime_name}-{runtime_memory}-{version}'
@@ -357,7 +357,7 @@ def build_runtime(self, runtime_name, runtime_file, extra_args=[]):
         finally:
             os.remove(LITHOPS_FUNCTION_ZIP)
 
-        registry = f'{self.account_id}.dkr.ecr.{self.region_name}.amazonaws.com'
+        registry = f'{self.account_id}.dkr.ecr.{self.region}.amazonaws.com'
 
         res = self.ecr_client.get_authorization_token()
         if res['ResponseMetadata']['HTTPStatusCode'] != 200:
@@ -474,7 +474,7 @@ def _deploy_container_runtime(self, runtime_name, memory, timeout):
         except botocore.exceptions.ClientError:
             raise Exception(f'Runtime "{runtime_name}" is not deployed to ECR')
 
-        registry = f'{self.account_id}.dkr.ecr.{self.region_name}.amazonaws.com'
+        registry = f'{self.account_id}.dkr.ecr.{self.region}.amazonaws.com'
         image_uri = f'{registry}/{repo_name}@{image_digest}'
 
         env_vars = {t['name']: t['value'] for t in self.lambda_config['env_vars']}
@@ -628,7 +628,7 @@ def invoke(self, runtime_name, runtime_memory, payload):
         headers = {'Host': self.host, 'X-Amz-Invocation-Type': 'Event', 'User-Agent': self.user_agent}
         url = f'https://{self.host}/2015-03-31/functions/{function_name}/invocations'
         request = AWSRequest(method="POST", url=url, data=json.dumps(payload, default=str), headers=headers)
-        SigV4Auth(self.credentials, "lambda", self.region_name).add_auth(request)
+        SigV4Auth(self.credentials, "lambda", self.region).add_auth(request)
 
         invoked = False
         while not invoked:
@@ -674,7 +674,7 @@ def get_runtime_key(self, runtime_name, runtime_memory, version=__version__):
         in order to know which runtimes are installed and which not.
         """
         action_name = self._format_function_name(runtime_name, runtime_memory, version)
-        runtime_key = os.path.join(self.name, version, self.region_name, action_name)
+        runtime_key = os.path.join(self.name, version, self.region, action_name)
 
         return runtime_key