♻️ Replace r clone ⚠️🚨

.env-devel

@@ -32,6 +32,8 @@ AUTOSCALING_NODES_MONITORING=null
 AUTOSCALING_POLL_INTERVAL=10
 AUTOSCALING_SSM_ACCESS=null
 
+AWS_S3_CLI_S3='{"S3_ACCESS_KEY":"12345678", "S3_BUCKET_NAME":"simcore", "S3_ENDPOINT": "http://172.17.0.1:9001", "S3_SECRET_KEY": "12345678", "S3_REGION": "us-east-1"}'
+
 CATALOG_BACKGROUND_TASK_REST_TIME=60
 CATALOG_DEV_FEATURES_ENABLED=0
 CATALOG_HOST=catalog

packages/settings-library/src/settings_library/aws_s3_cli.py

@@ -0,0 +1,11 @@
+from pydantic import Field
+
+from .base import BaseCustomSettings
+from .s3 import S3Settings
+
+
+class AwsS3CliSettings(BaseCustomSettings):
+    AWS_S3_CLI_S3: S3Settings = Field(
+        default=None,
+        description="These settings intentionally do not use auto_default_from_env=True because we might want to turn them off if RClone is enabled.",
+    )

packages/simcore-sdk/src/simcore_sdk/node_data/data_manager.py

@@ -10,6 +10,7 @@
 from servicelib.archiving_utils import unarchive_dir
 from servicelib.logging_utils import log_context
 from servicelib.progress_bar import ProgressBarData
+from settings_library.aws_s3_cli import AwsS3CliSettings
 from settings_library.r_clone import RCloneSettings
 
 from ..node_ports_common import filemanager
@@ -41,6 +42,7 @@ async def _push_directory(
     r_clone_settings: RCloneSettings,
     exclude_patterns: set[str] | None = None,
     progress_bar: ProgressBarData,
+    aws_s3_cli_settings: AwsS3CliSettings | None,
 ) -> None:
     s3_object = __create_s3_object_key(project_id, node_uuid, source_path)
     with log_context(
@@ -56,6 +58,7 @@ async def _push_directory(
             io_log_redirect_cb=io_log_redirect_cb,
             progress_bar=progress_bar,
             exclude_patterns=exclude_patterns,
+            aws_s3_cli_settings=aws_s3_cli_settings,
         )
 
 
@@ -68,6 +71,7 @@ async def _pull_directory(
     io_log_redirect_cb: LogRedirectCB,
     r_clone_settings: RCloneSettings,
     progress_bar: ProgressBarData,
+    aws_s3_cli_settings: AwsS3CliSettings | None,
     save_to: Path | None = None,
 ) -> None:
     save_to_path = destination_path if save_to is None else save_to
@@ -84,6 +88,7 @@ async def _pull_directory(
             io_log_redirect_cb=io_log_redirect_cb,
             r_clone_settings=r_clone_settings,
             progress_bar=progress_bar,
+            aws_s3_cli_settings=aws_s3_cli_settings,
         )
 
 
@@ -116,6 +121,7 @@ async def _pull_legacy_archive(
                 io_log_redirect_cb=io_log_redirect_cb,
                 r_clone_settings=None,
                 progress_bar=sub_prog,
+                aws_s3_cli_settings=None,
             )
             _logger.info("completed pull of %s.", destination_path)
 
@@ -195,6 +201,7 @@ async def push(
     r_clone_settings: RCloneSettings,
     exclude_patterns: set[str] | None = None,
     progress_bar: ProgressBarData,
+    aws_s3_cli_settings: AwsS3CliSettings | None,
 ) -> None:
     """pushes and removes the legacy archive if present"""
 
@@ -207,6 +214,7 @@ async def push(
         exclude_patterns=exclude_patterns,
         io_log_redirect_cb=io_log_redirect_cb,
         progress_bar=progress_bar,
+        aws_s3_cli_settings=aws_s3_cli_settings,
     )
     archive_exists = await _state_metadata_entry_exists(
         user_id=user_id,
@@ -236,6 +244,7 @@ async def pull(
     io_log_redirect_cb: LogRedirectCB,
     r_clone_settings: RCloneSettings,
     progress_bar: ProgressBarData,
+    aws_s3_cli_settings: AwsS3CliSettings | None,
 ) -> None:
     """restores the state folder"""
 
@@ -274,6 +283,7 @@ async def pull(
             io_log_redirect_cb=io_log_redirect_cb,
             r_clone_settings=r_clone_settings,
             progress_bar=progress_bar,
+            aws_s3_cli_settings=aws_s3_cli_settings,
         )
         return
 

packages/simcore-sdk/src/simcore_sdk/node_ports_common/_utils.py

@@ -0,0 +1,7 @@
+from abc import abstractmethod
+
+
+class BaseLogParser:
+    @abstractmethod
+    async def __call__(self, logs: str) -> None:
+        ...